diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0afd96e1c3f..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2.1 -jobs: - build: - docker: - - image: alpine:3.7 - steps: - - run: - name: The First Step - command: | - echo 'Fake step!' - diff --git a/.drone.star b/.drone.star index a90a3027b23..f025f8df415 100644 --- a/.drone.star +++ b/.drone.star @@ -6,7 +6,7 @@ load('scripts/drone/pipelines/pr.star', 'pr_pipelines') load('scripts/drone/pipelines/main.star', 'main_pipelines') -load('scripts/drone/pipelines/release.star', 'release_pipelines', 'test_release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline') +load('scripts/drone/pipelines/release.star', 'release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline') load('scripts/drone/version.star', 'version_branch_pipelines') load('scripts/drone/pipelines/cron.star', 'cronjobs') load('scripts/drone/vault.star', 'secrets') @@ -17,4 +17,4 @@ def main(ctx): publish_image_pipelines('public') + publish_image_pipelines('security') + \ publish_artifacts_pipelines('security') + publish_artifacts_pipelines('public') + \ publish_npm_pipelines('public') + publish_packages_pipeline() + \ - test_release_pipelines() + version_branch_pipelines() + cronjobs(edition=edition) + secrets() + version_branch_pipelines() + cronjobs(edition=edition) + secrets() diff --git a/.drone.yml b/.drone.yml index 50d442ab33c..f85642fef3e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,7 +11,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -111,7 +111,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -187,22 +187,22 @@ steps: image: grafana/build-container:1.4.9 name: package - commands: - - ./e2e/start-server + - ./scripts/grafana-server/start-server depends_on: - package detach: true environment: PORT: 3001 image: grafana/build-container:1.4.9 - name: end-to-end-tests-server + name: grafana-server - commands: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat @@ -210,8 +210,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat @@ -219,8 +219,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat @@ -228,8 +228,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-various-suite - commands: - apt-get update @@ -276,14 +276,14 @@ steps: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci-pr.conf.js depends_on: - - end-to-end-tests-server + - grafana-server environment: GRAFANA_MISC_STATS_API_KEY: from_secret: grafana_misc_stats_api_key - HOST: end-to-end-tests-server + HOST: grafana-server PORT: 3001 failure: always - image: hugohaggmark/docker-puppeteer + image: grafana/docker-puppeteer:1.0.0 name: test-a11y-frontend - commands: - ./scripts/ci-reference-docs-lint.sh ci @@ -356,7 +356,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -417,7 +417,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -518,7 +518,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -617,22 +617,22 @@ steps: image: grafana/build-container:1.4.9 name: package - commands: - - ./e2e/start-server + - ./scripts/grafana-server/start-server depends_on: - package detach: true environment: PORT: 3001 image: grafana/build-container:1.4.9 - name: end-to-end-tests-server + name: grafana-server - commands: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat @@ -640,8 +640,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat @@ -649,8 +649,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat @@ -658,8 +658,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-various-suite - commands: - apt-get update @@ -723,14 +723,14 @@ steps: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json depends_on: - - end-to-end-tests-server + - grafana-server environment: GRAFANA_MISC_STATS_API_KEY: from_secret: grafana_misc_stats_api_key - HOST: end-to-end-tests-server + HOST: grafana-server PORT: 3001 failure: ignore - image: hugohaggmark/docker-puppeteer + image: grafana/docker-puppeteer:1.0.0 name: test-a11y-frontend - commands: - ./scripts/ci-frontend-metrics.sh | ./bin/grabpl publish-metrics $${GRAFANA_MISC_STATS_API_KEY} @@ -808,7 +808,7 @@ steps: - commands: - ./bin/grabpl upload-cdn --edition oss --bucket "grafana-static-assets" depends_on: - - end-to-end-tests-server + - grafana-server environment: GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key @@ -864,7 +864,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -932,7 +932,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: initialize @@ -944,6 +944,7 @@ steps: - gcloud auth activate-service-account --key-file=gcpkey.json - rm gcpkey.json - cp C:\App\nssm-2.24.zip . + - .\grabpl.exe gen-version --build-id $$env:DRONE_BUILD_NUMBER - .\grabpl.exe windows-installer --edition oss --packages-bucket grafana-downloads --build-id $$env:DRONE_BUILD_NUMBER - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] @@ -1014,7 +1015,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1097,7 +1098,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1223,22 +1224,22 @@ steps: - name: docker path: /var/run/docker.sock - commands: - - ./e2e/start-server + - ./scripts/grafana-server/start-server depends_on: - package detach: true environment: PORT: 3001 image: grafana/build-container:1.4.9 - name: end-to-end-tests-server + name: grafana-server - commands: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat @@ -1246,8 +1247,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat @@ -1255,8 +1256,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat @@ -1264,8 +1265,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-various-suite - commands: - apt-get update @@ -1311,7 +1312,7 @@ steps: - commands: - ./bin/grabpl upload-cdn --edition oss --bucket "$${PRERELEASE_BUCKET}/artifacts/static-assets" depends_on: - - end-to-end-tests-server + - grafana-server environment: GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key @@ -1320,7 +1321,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition oss --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads/${DRONE_TAG} + - ./bin/grabpl upload-packages --edition oss --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -1401,7 +1402,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1522,7 +1523,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1606,7 +1607,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: initialize @@ -1618,10 +1619,11 @@ steps: - gcloud auth activate-service-account --key-file=gcpkey.json - rm gcpkey.json - cp C:\App\nssm-2.24.zip . + - .\grabpl.exe gen-version ${DRONE_TAG} - .\grabpl.exe windows-installer --edition oss ${DRONE_TAG} - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://%PRERELEASE_BUCKET%/artifacts/downloads/oss/release/ - - gsutil cp "$$fname.sha256" gs://%PRERELEASE_BUCKET%/artifacts/downloads/oss/release/ + - gsutil cp $$fname gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/oss/release/ + - gsutil cp "$$fname.sha256" gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/oss/release/ depends_on: - initialize environment: @@ -1664,7 +1666,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -1824,24 +1826,24 @@ steps: - name: docker path: /var/run/docker.sock - commands: - - ./e2e/start-server + - ./scripts/grafana-server/start-server depends_on: - package detach: true environment: PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz PORT: 3001 - RUNDIR: e2e/tmp-grafana-enterprise + RUNDIR: scripts/grafana-server/tmp-grafana-enterprise image: grafana/build-container:1.4.9 - name: end-to-end-tests-server + name: grafana-server - commands: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat @@ -1849,8 +1851,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat @@ -1858,8 +1860,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat @@ -1867,8 +1869,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-various-suite - commands: - apt-get update @@ -1914,7 +1916,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets - commands: - - ./bin/grabpl upload-packages --edition enterprise --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads/${DRONE_TAG} + - ./bin/grabpl upload-packages --edition enterprise --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads depends_on: - package environment: @@ -1957,7 +1959,7 @@ steps: image: grafana/grafana-ci-deploy:1.3.1 name: upload-cdn-assets-enterprise2 - commands: - - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads-enterprise2/${DRONE_TAG} + - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads-enterprise2 depends_on: - package-enterprise2 environment: @@ -2004,7 +2006,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2177,7 +2179,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2305,7 +2307,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -2337,10 +2339,11 @@ steps: - gcloud auth activate-service-account --key-file=gcpkey.json - rm gcpkey.json - cp C:\App\nssm-2.24.zip . + - .\grabpl.exe gen-version ${DRONE_TAG} - .\grabpl.exe windows-installer --edition enterprise ${DRONE_TAG} - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://%PRERELEASE_BUCKET%/artifacts/downloads/enterprise/release/ - - gsutil cp "$$fname.sha256" gs://%PRERELEASE_BUCKET%/artifacts/downloads/enterprise/release/ + - gsutil cp $$fname gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/enterprise/release/ + - gsutil cp "$$fname.sha256" gs://%PRERELEASE_BUCKET%/artifacts/downloads/${DRONE_TAG}/enterprise/release/ depends_on: - initialize environment: @@ -2379,7 +2382,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2457,7 +2460,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2518,7 +2521,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2596,7 +2599,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2658,7 +2661,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2694,7 +2697,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2741,7 +2744,7 @@ steps: name: initialize - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -2776,7 +2779,8 @@ volumes: path: /var/run/docker.sock name: docker --- -depends_on: [] +depends_on: +- publish-artifacts-public kind: pipeline name: publish-packages node: @@ -2788,13 +2792,14 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl - commands: - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - ./bin/grabpl store-packages --edition oss --gcp-key /tmp/gcpkey.json ${DRONE_TAG} + - ./bin/grabpl store-packages --edition oss --packages-bucket grafana-downloads + --gcp-key /tmp/gcpkey.json ${DRONE_TAG} depends_on: - grabpl environment: @@ -2812,7 +2817,8 @@ steps: name: store-packages-oss - commands: - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - ./bin/grabpl store-packages --edition enterprise --gcp-key /tmp/gcpkey.json ${DRONE_TAG} + - ./bin/grabpl store-packages --edition enterprise --packages-bucket grafana-downloads + --gcp-key /tmp/gcpkey.json ${DRONE_TAG} depends_on: - grabpl environment: @@ -2841,1336 +2847,6 @@ volumes: --- depends_on: [] kind: pipeline -name: oss-build-publish-e2e-test-release -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - make gen-go - - ./bin/grabpl verify-version v7.3.0-test - - ./bin/grabpl gen-version v7.3.0-test - - yarn install --immutable - image: grafana/build-container:1.4.9 - name: initialize -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} - --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: build-backend -- commands: - - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --no-install-deps - --edition oss --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.4.9 - name: build-frontend -- commands: - - ./bin/grabpl build-plugins --jobs 8 --edition oss --no-install-deps --sign --signing-admin - depends_on: - - initialize - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: build-plugins -- commands: - - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . - depends_on: - - build-backend - image: grafana/build-container:1.4.9 - name: validate-scuemata -- commands: - - '# Make sure the git tree is clean.' - - '# Stashing changes, since packages that were produced in build-backend step are - needed.' - - git stash - - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . - - '# The above command generates Typescript files (*.gen.ts) from all appropriate - .cue files.' - - '# It is required that the generated Typescript be in sync with the input CUE - files.' - - '# ...Modulo eslint auto-fixes...:' - - yarn run eslint . --ext .gen.ts --fix - - '# If any filenames are emitted by the below script, run the generator command - `grafana-cli cue gen-ts` locally and commit the result.' - - ./scripts/clean-git-or-error.sh - - '# Un-stash changes.' - - git stash pop - depends_on: - - validate-scuemata - image: grafana/build-container:1.4.9 - name: ensure-cuetsified -- commands: - - ./bin/grabpl package --jobs 8 --edition oss --github-token $${GITHUB_TOKEN} --no-pull-enterprise - --sign v7.3.0-test - depends_on: - - build-plugins - - build-backend - - build-frontend - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition oss --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./e2e/start-server - depends_on: - - package - detach: true - environment: - PORT: 3001 - image: grafana/build-container:1.4.9 - name: end-to-end-tests-server -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - image: google/cloud-sdk:367.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - yarn storybook:build - - ./bin/grabpl verify-storybook - depends_on: - - build-frontend - environment: - NODE_OPTIONS: --max_old_space_size=4096 - image: grafana/build-container:1.4.9 - name: build-storybook -- commands: - - ./bin/grabpl upload-cdn --edition oss --bucket "grafana-static-assets" - depends_on: - - end-to-end-tests-server - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets -- commands: - - ./bin/grabpl upload-packages --edition oss --packages-bucket grafana-downloads-test - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages -- commands: - - echo Testing release - depends_on: - - build-storybook - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - GCP_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: store-storybook -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -depends_on: [] -kind: pipeline -name: oss-test-test-release -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - make gen-go - - ./bin/grabpl verify-version v7.3.0-test - - ./bin/grabpl gen-version v7.3.0-test - - yarn install --immutable - image: grafana/build-container:1.4.9 - name: initialize -- commands: - - |- - echo -e "unknwon - referer - errorstring - eror - iam - wan" > words_to_ignore.txt - - codespell -I words_to_ignore.txt docs/ - - rm words_to_ignore.txt - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: codespell -- commands: - - ./bin/grabpl shellcheck - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: shellcheck -- commands: - - ./bin/grabpl lint-backend --edition oss - depends_on: - - initialize - environment: - CGO_ENABLED: "1" - image: grafana/build-container:1.4.9 - name: lint-backend -- commands: - - yarn run prettier:check - - yarn run lint - - yarn run i18n:compile - - yarn run typecheck - depends_on: - - initialize - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.4.9 - name: lint-frontend -- commands: - - ./bin/grabpl test-backend --edition oss - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend -- commands: - - ./bin/grabpl integration-tests --edition oss - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-integration -- commands: - - yarn run ci:test-frontend - depends_on: - - initialize - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.4.9 - name: test-frontend -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -depends_on: [] -kind: pipeline -name: oss-integration-tests-test-release -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - make gen-go - - ./bin/grabpl verify-version v7.3.0-test - - ./bin/grabpl gen-version v7.3.0-test - - yarn install --immutable - image: grafana/build-container:1.4.9 - name: initialize -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - ./bin/grabpl integration-tests --database postgres - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.4.9 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - ./bin/grabpl integration-tests --database mysql - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.4.9 - name: mysql-integration-tests -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -depends_on: -- oss-build-publish-e2e-test-release -- oss-test-test-release -- oss-integration-tests-test-release -kind: pipeline -name: oss-windows-test-release -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/windows/grabpl.exe - -OutFile grabpl.exe - image: grafana/ci-wix:0.1.1 - name: initialize -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition oss --packages-bucket grafana-downloads-test - v7.3.0-test - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://grafana-downloads-test/oss/release/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads-test/oss/release/ - depends_on: - - initialize - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: enterprise-build-e2e-publish-test-release -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout main - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl verify-version v7.3.0-test - - ./bin/grabpl gen-version v7.3.0-test - - yarn install --immutable - depends_on: - - clone-enterprise - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: initialize -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} - --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: build-backend -- commands: - - ./bin/grabpl build-frontend --jobs 8 --github-token $${GITHUB_TOKEN} --no-install-deps - --edition enterprise --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - NODE_OPTIONS: --max_old_space_size=8192 - image: grafana/build-container:1.4.9 - name: build-frontend -- commands: - - ./bin/grabpl build-plugins --jobs 8 --edition enterprise --no-install-deps --sign - --signing-admin - depends_on: - - initialize - environment: - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: build-plugins -- commands: - - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root . - depends_on: - - build-backend - image: grafana/build-container:1.4.9 - name: validate-scuemata -- commands: - - '# Make sure the git tree is clean.' - - '# Stashing changes, since packages that were produced in build-backend step are - needed.' - - git stash - - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . - - '# The above command generates Typescript files (*.gen.ts) from all appropriate - .cue files.' - - '# It is required that the generated Typescript be in sync with the input CUE - files.' - - '# ...Modulo eslint auto-fixes...:' - - yarn run eslint . --ext .gen.ts --fix - - '# If any filenames are emitted by the below script, run the generator command - `grafana-cli cue gen-ts` locally and commit the result.' - - ./scripts/clean-git-or-error.sh - - '# Un-stash changes.' - - git stash pop - depends_on: - - validate-scuemata - image: grafana/build-container:1.4.9 - name: ensure-cuetsified -- commands: - - ./bin/grabpl build-backend --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} - --no-pull-enterprise v7.3.0-test - depends_on: - - initialize - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: build-backend-enterprise2 -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise --github-token $${GITHUB_TOKEN} - --no-pull-enterprise --sign v7.3.0-test - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.4.9 - name: copy-packages-for-docker -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json - - ./bin/grabpl build-docker --edition enterprise --shouldSave --ubuntu - depends_on: - - copy-packages-for-docker - environment: - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk - name: package-docker-images-ubuntu - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./e2e/start-server - depends_on: - - package - detach: true - environment: - PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz - PORT: 3001 - RUNDIR: e2e/tmp-grafana-enterprise - image: grafana/build-container:1.4.9 - name: end-to-end-tests-server -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-dashboards-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite smoke-tests-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-smoke-tests-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite panels-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-panels-suite -- commands: - - apt-get install -y netcat - - ./bin/grabpl e2e-tests --port 3001 --suite various-suite --tries 3 - depends_on: - - package - environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 - name: end-to-end-tests-various-suite -- commands: - - apt-get update - - apt-get install -yq zip - - ls -lah ./e2e - - find ./e2e -type f -name "*.mp4" - - printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json - - gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json - - find ./e2e -type f -name "*spec.ts.mp4" | zip e2e/videos.zip -@ - - gsutil cp e2e/videos.zip gs://$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - export E2E_ARTIFACTS_VIDEO_ZIP=https://storage.googleapis.com/$${E2E_TEST_ARTIFACTS_BUCKET}/${DRONE_BUILD_NUMBER}/artifacts/videos/videos.zip - - 'echo "E2E Test artifacts uploaded to: $${E2E_ARTIFACTS_VIDEO_ZIP}"' - - 'curl -X POST https://api.github.com/repos/${DRONE_REPO}/statuses/${DRONE_COMMIT_SHA} - -H "Authorization: token $${GITHUB_TOKEN}" -d "{\"state\":\"success\",\"target_url\":\"$${E2E_ARTIFACTS_VIDEO_ZIP}\", - \"description\": \"Click on the details to download e2e recording videos\", \"context\": - \"e2e_artifacts\"}"' - depends_on: - - end-to-end-tests-dashboards-suite - - end-to-end-tests-panels-suite - - end-to-end-tests-smoke-tests-suite - - end-to-end-tests-various-suite - environment: - E2E_TEST_ARTIFACTS_BUCKET: releng-pipeline-artifacts-dev - GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY: - from_secret: gcp_upload_artifacts_key - GITHUB_TOKEN: - from_secret: github_token - image: google/cloud-sdk:367.0.0 - name: e2e-tests-artifacts-upload - when: - status: - - success - - failure -- commands: - - ./bin/grabpl upload-cdn --edition enterprise --bucket "grafana-static-assets" - depends_on: - - package - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets -- commands: - - ./bin/grabpl upload-packages --edition enterprise --packages-bucket grafana-downloads-test - depends_on: - - package - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages -- commands: - - ./bin/grabpl package --jobs 8 --edition enterprise2 --github-token $${GITHUB_TOKEN} - --no-pull-enterprise --sign v7.3.0-test - depends_on: - - build-plugins - - build-backend - - build-frontend - - build-backend-enterprise2 - environment: - GITHUB_TOKEN: - from_secret: github_token - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_API_KEY: - from_secret: grafana_api_key - image: grafana/build-container:1.4.9 - name: package-enterprise2 -- commands: - - ./bin/grabpl upload-cdn --edition enterprise2 --bucket "grafana-static-assets" - depends_on: - - package-enterprise2 - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-cdn-assets-enterprise2 -- commands: - - ./bin/grabpl upload-packages --edition enterprise2 --packages-bucket grafana-downloads-test - depends_on: - - package-enterprise2 - environment: - GCP_GRAFANA_UPLOAD_KEY: - from_secret: gcp_key - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/grafana-ci-deploy:1.3.1 - name: upload-packages-enterprise2 -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - disable: true -depends_on: [] -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: enterprise-test-test-release -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout main - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl verify-version v7.3.0-test - - ./bin/grabpl gen-version v7.3.0-test - - yarn install --immutable - depends_on: - - clone-enterprise - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: initialize -- commands: - - |- - echo -e "unknwon - referer - errorstring - eror - iam - wan" > words_to_ignore.txt - - codespell -I words_to_ignore.txt docs/ - - rm words_to_ignore.txt - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: codespell -- commands: - - ./bin/grabpl shellcheck - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: shellcheck -- commands: - - ./bin/grabpl lint-backend --edition enterprise - depends_on: - - initialize - environment: - CGO_ENABLED: "1" - image: grafana/build-container:1.4.9 - name: lint-backend -- commands: - - yarn run prettier:check - - yarn run lint - - yarn run i18n:compile - - yarn run typecheck - depends_on: - - initialize - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.4.9 - name: lint-frontend -- commands: - - ./bin/grabpl test-backend --edition enterprise - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend -- commands: - - ./bin/grabpl integration-tests --edition enterprise - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-integration -- commands: - - yarn run ci:test-frontend - depends_on: - - initialize - environment: - TEST_MAX_WORKERS: 50% - image: grafana/build-container:1.4.9 - name: test-frontend -- commands: - - ./bin/grabpl lint-backend --edition enterprise2 - depends_on: - - initialize - environment: - CGO_ENABLED: "1" - image: grafana/build-container:1.4.9 - name: lint-backend-enterprise2 -- commands: - - ./bin/grabpl test-backend --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-enterprise2 -- commands: - - ./bin/grabpl integration-tests --edition enterprise2 - depends_on: - - initialize - image: grafana/build-container:1.4.9 - name: test-backend-integration-enterprise2 -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - disable: true -depends_on: [] -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: enterprise-integration-tests-test-release -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: -- environment: - PGDATA: /var/lib/postgresql/data/pgdata - POSTGRES_DB: grafanatest - POSTGRES_PASSWORD: grafanatest - POSTGRES_USER: grafanatest - image: postgres:12.3-alpine - name: postgres - volumes: - - name: postgres - path: /var/lib/postgresql/data/pgdata -- environment: - MYSQL_DATABASE: grafana_tests - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_USER: grafana - image: mysql:5.6.48 - name: mysql - volumes: - - name: mysql - path: /var/lib/mysql -- environment: {} - image: redis:6.2.1-alpine - name: redis -- environment: {} - image: memcached:1.6.9-alpine - name: memcached -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout main - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - - make gen-go - - ./bin/grabpl verify-version v7.3.0-test - - ./bin/grabpl gen-version v7.3.0-test - - yarn install --immutable - depends_on: - - clone-enterprise - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.4.9 - name: initialize -- commands: - - apt-get update - - apt-get install -yq postgresql-client - - dockerize -wait tcp://postgres:5432 -timeout 120s - - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql - - go clean -testcache - - ./bin/grabpl integration-tests --database postgres - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: postgres - PGPASSWORD: grafanatest - POSTGRES_HOST: postgres - image: grafana/build-container:1.4.9 - name: postgres-integration-tests -- commands: - - apt-get update - - apt-get install -yq default-mysql-client - - dockerize -wait tcp://mysql:3306 -timeout 120s - - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root - -prootpass - - go clean -testcache - - ./bin/grabpl integration-tests --database mysql - depends_on: - - grabpl - environment: - GRAFANA_TEST_DB: mysql - MYSQL_HOST: mysql - image: grafana/build-container:1.4.9 - name: mysql-integration-tests -- commands: - - dockerize -wait tcp://redis:6379/0 -timeout 120s - - ./bin/grabpl integration-tests - depends_on: - - grabpl - environment: - REDIS_URL: redis://redis:6379/0 - image: grafana/build-container:1.4.9 - name: redis-integration-tests -- commands: - - dockerize -wait tcp://memcached:11211 -timeout 120s - - ./bin/grabpl integration-tests - depends_on: - - grabpl - environment: - MEMCACHED_HOSTS: memcached:11211 - image: grafana/build-container:1.4.9 - name: memcached-integration-tests -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker -- name: postgres - temp: - medium: memory -- name: mysql - temp: - medium: memory ---- -clone: - disable: true -depends_on: -- enterprise-build-e2e-publish-test-release -- enterprise-test-test-release -- enterprise-integration-tests-test-release -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: enterprise-windows-test-release -platform: - arch: amd64 - os: windows - version: "1809" -services: [] -steps: -- commands: - - echo $env:DRONE_RUNNER_NAME - image: mcr.microsoft.com/windows:1809 - name: identify-runner -- commands: - - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/windows/grabpl.exe - -OutFile grabpl.exe - - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout main - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: clone -- commands: - - cp -r grafana-enterprise C:\App\grafana-enterprise - - rm -r -force grafana-enterprise - - cp grabpl.exe C:\App\grabpl.exe - - rm -force grabpl.exe - - C:\App\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\App\grafana-enterprise - - cp C:\App\grabpl.exe grabpl.exe - depends_on: - - clone - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/ci-wix:0.1.1 - name: initialize -- commands: - - $$gcpKey = $$env:GCP_KEY - - '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) - > gcpkey.json' - - dos2unix gcpkey.json - - gcloud auth activate-service-account --key-file=gcpkey.json - - rm gcpkey.json - - cp C:\App\nssm-2.24.zip . - - .\grabpl.exe windows-installer --edition enterprise --packages-bucket grafana-downloads-test - v7.3.0-test - - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0] - - gsutil cp $$fname gs://grafana-downloads-test/enterprise/release/ - - gsutil cp "$$fname.sha256" gs://grafana-downloads-test/enterprise/release/ - depends_on: - - initialize - environment: - GCP_KEY: - from_secret: gcp_key - GITHUB_TOKEN: - from_secret: github_token - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - image: grafana/ci-wix:0.1.1 - name: build-windows-installer -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -depends_on: -- oss-build-publish-e2e-test-release -- oss-test-test-release -- oss-integration-tests-test-release -- oss-windows-test-release -- enterprise-build-e2e-publish-test-release -- enterprise-test-test-release -- enterprise-integration-tests-test-release -- enterprise-windows-test-release -kind: pipeline -name: publish-test-release -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.15 - name: identify-runner -- commands: - - make gen-go - - ./bin/grabpl verify-version v7.3.0-test - image: grafana/build-container:1.4.9 - name: initialize -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - ./bin/grabpl store-packages --edition oss --gcp-key /tmp/gcpkey.json --deb-db-bucket - grafana-testing-aptly-db --deb-repo-bucket grafana-testing-repo --packages-bucket - grafana-downloads-test --rpm-repo-bucket grafana-testing-repo --simulate-release - v7.3.0-test - depends_on: - - grabpl - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.1 - name: store-packages-oss -- commands: - - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json - - ./bin/grabpl store-packages --edition enterprise --gcp-key /tmp/gcpkey.json --deb-db-bucket - grafana-testing-aptly-db --deb-repo-bucket grafana-testing-repo --packages-bucket - grafana-downloads-test --rpm-repo-bucket grafana-testing-repo --simulate-release - v7.3.0-test - depends_on: - - grabpl - environment: - GCP_KEY: - from_secret: gcp_key - GPG_KEY_PASSWORD: - from_secret: gpg_key_password - GPG_PRIV_KEY: - from_secret: gpg_priv_key - GPG_PUB_KEY: - from_secret: gpg_pub_key - GRAFANA_COM_API_KEY: - from_secret: grafana_api_key - image: grafana/grafana-ci-deploy:1.3.1 - name: store-packages-enterprise -trigger: - event: - - custom -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -depends_on: -- oss-build-publish-e2e-test-release -- oss-test-test-release -- oss-integration-tests-test-release -- oss-windows-test-release -- enterprise-build-e2e-publish-test-release -- enterprise-test-test-release -- enterprise-integration-tests-test-release -- enterprise-windows-test-release -- publish-test-release -kind: pipeline -name: notify-test-release -platform: - arch: amd64 - os: linux -steps: -- image: plugins/slack - name: slack - settings: - channel: grafana-ci-notifications - template: |- - Build {{build.number}} failed for commit: : {{build.link}} - Branch: - Author: {{build.author}} - webhook: - from_secret: slack_webhook -trigger: - event: - - custom - status: - - failure -type: docker ---- -depends_on: [] -kind: pipeline name: oss-build-publish-e2e-release-branch node: type: no-parallel @@ -4181,7 +2857,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4304,22 +2980,22 @@ steps: - name: docker path: /var/run/docker.sock - commands: - - ./e2e/start-server + - ./scripts/grafana-server/start-server depends_on: - package detach: true environment: PORT: 3001 image: grafana/build-container:1.4.9 - name: end-to-end-tests-server + name: grafana-server - commands: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat @@ -4327,8 +3003,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat @@ -4336,8 +3012,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat @@ -4345,8 +3021,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-various-suite - commands: - apt-get update @@ -4392,7 +3068,7 @@ steps: - commands: - ./bin/grabpl upload-cdn --edition oss --bucket "grafana-static-assets" depends_on: - - end-to-end-tests-server + - grafana-server environment: GCP_GRAFANA_UPLOAD_KEY: from_secret: gcp_key @@ -4441,7 +3117,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4555,7 +3231,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4632,7 +3308,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/windows/grabpl.exe -OutFile grabpl.exe image: grafana/ci-wix:0.1.1 name: initialize @@ -4680,7 +3356,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -4834,24 +3510,24 @@ steps: - name: docker path: /var/run/docker.sock - commands: - - ./e2e/start-server + - ./scripts/grafana-server/start-server depends_on: - package detach: true environment: PACKAGE_FILE: dist/grafana-enterprise-*linux-amd64.tar.gz PORT: 3001 - RUNDIR: e2e/tmp-grafana-enterprise + RUNDIR: scripts/grafana-server/tmp-grafana-enterprise image: grafana/build-container:1.4.9 - name: end-to-end-tests-server + name: grafana-server - commands: - apt-get install -y netcat - ./bin/grabpl e2e-tests --port 3001 --suite dashboards-suite --tries 3 depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-dashboards-suite - commands: - apt-get install -y netcat @@ -4859,8 +3535,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-smoke-tests-suite - commands: - apt-get install -y netcat @@ -4868,8 +3544,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-panels-suite - commands: - apt-get install -y netcat @@ -4877,8 +3553,8 @@ steps: depends_on: - package environment: - HOST: end-to-end-tests-server - image: cypress/included:9.2.0 + HOST: grafana-server + image: cypress/included:9.3.1 name: end-to-end-tests-various-suite - commands: - apt-get update @@ -5017,7 +3693,7 @@ services: [] steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -5182,7 +3858,7 @@ services: steps: - commands: - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/grabpl + - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/grabpl - chmod +x bin/grabpl image: byrnedo/alpine-curl:0.1.8 name: grabpl @@ -5302,7 +3978,7 @@ steps: name: identify-runner - commands: - $$ProgressPreference = "SilentlyContinue" - - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.2/windows/grabpl.exe + - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v2.8.7/windows/grabpl.exe -OutFile grabpl.exe - git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git" - cd grafana-enterprise @@ -5499,6 +4175,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: 81173294ad474a12cbb622d9e963e208ddd2a2ac42a30dde6396794923b158a0 +hmac: 19229313440a0d96416f34c4c7ac5d8d00f30478e71a0a6072ae106145e1794c ... diff --git a/.eslintignore b/.eslintignore index 4608870d6e9..1fc48c822a9 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,6 +6,7 @@ devenv data dist e2e/tmp +scripts/grafana-server/tmp public/lib/monaco deployment_tools_config.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e0ffc40fc45..92f2242843c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -30,18 +30,19 @@ go.sum @grafana/backend-platform /scripts/drone/ @grafana/grafana-release-eng # Cloud Datasources backend code -/pkg/tsdb/cloudwatch @grafana/cloud-datasources @grafana/observability-squad +/pkg/tsdb/cloudwatch @grafana/cloud-datasources /pkg/tsdb/azuremonitor @grafana/cloud-datasources /pkg/tsdb/cloudmonitoring @grafana/cloud-datasources # Observability backend code -/pkg/tsdb/influxdb @grafana/observability-squad -/pkg/tsdb/elasticsearch @grafana/observability-squad -/pkg/tsdb/graphite @grafana/observability-squad -/pkg/tsdb/jaeger @grafana/observability-squad -/pkg/tsdb/loki @grafana/observability-squad -/pkg/tsdb/zipkin @grafana/observability-squad -/pkg/tsdb/tempo @grafana/observability-squad +/pkg/tsdb/prometheus @grafana/observability-metrics +/pkg/tsdb/influxdb @grafana/observability-metrics +/pkg/tsdb/elasticsearch @grafana/observability-logs-and-traces +/pkg/tsdb/graphite @grafana/observability-metrics +/pkg/tsdb/jaeger @grafana/observability-logs-and-traces +/pkg/tsdb/loki @grafana/observability-logs-and-traces +/pkg/tsdb/zipkin @grafana/observability-logs-and-traces +/pkg/tsdb/tempo @grafana/observability-logs-and-traces # BI backend code /pkg/tsdb/mysql @grafana/grafana-bi-squad @@ -89,21 +90,24 @@ go.sum @grafana/backend-platform /packages/grafana-ui/src/components/TimeSeries @grafana/grafana-bi-squad /packages/grafana-ui/src/components/uPlot @grafana/grafana-bi-squad /packages/grafana-ui/src/utils/storybook @grafana/plugins-platform-frontend -/packages/jaeger-ui-components/ @grafana/observability-squad +/packages/jaeger-ui-components/ @grafana/observability-logs-and-traces /plugins-bundled @grafana/plugins-platform-frontend # public folder /public/app/core/components/TimePicker @grafana/grafana-bi-squad /public/app/core/components/Layers @grafana/grafana-edge-squad /public/app/features/canvas/ @grafana/grafana-edge-squad /public/app/features/dimensions/ @grafana/grafana-edge-squad +/public/app/features/geo/ @grafana/grafana-edge-squad /public/app/features/live/ @grafana/grafana-edge-squad -/public/app/features/explore/ @grafana/observability-squad +/public/app/features/explore/ @grafana/observability-experience-squad /public/app/features/plugins @grafana/plugins-platform-frontend +/public/app/core/components/TransformersUI/spatial @grafana/grafana-edge-squad /public/app/plugins/panel/alertlist @grafana/alerting-squad /public/app/plugins/panel/barchart @grafana/grafana-bi-squad /public/app/plugins/panel/heatmap @grafana/grafana-bi-squad /public/app/plugins/panel/histogram @grafana/grafana-bi-squad -/public/app/plugins/panel/nodeGraph @grafana/observability-squad +/public/app/plugins/panel/logs @grafana/observability-logs-and-traces +/public/app/plugins/panel/nodeGraph @grafana/observability-logs-and-traces /public/app/plugins/panel/piechart @grafana/grafana-bi-squad /public/app/plugins/panel/state-timeline @grafana/grafana-bi-squad /public/app/plugins/panel/status-history @grafana/grafana-bi-squad @@ -134,21 +138,21 @@ lerna.json @grafana/frontend-ops *.mdx @marcusolsson @jessover9000 @grafana/plugins-platform-frontend # Core datasources -/public/app/plugins/datasource/cloudwatch @grafana/cloud-datasources @grafana/observability-squad -/public/app/plugins/datasource/elasticsearch @grafana/observability-squad +/public/app/plugins/datasource/cloudwatch @grafana/cloud-datasources +/public/app/plugins/datasource/elasticsearch @grafana/observability-logs-and-traces /public/app/plugins/datasource/grafana-azure-monitor-datasource @grafana/cloud-datasources -/public/app/plugins/datasource/graphite @grafana/observability-squad -/public/app/plugins/datasource/influxdb @grafana/observability-squad -/public/app/plugins/datasource/jaeger @grafana/observability-squad -/public/app/plugins/datasource/loki @grafana/observability-squad +/public/app/plugins/datasource/graphite @grafana/observability-metrics +/public/app/plugins/datasource/influxdb @grafana/observability-metrics +/public/app/plugins/datasource/jaeger @grafana/observability-logs-and-traces +/public/app/plugins/datasource/loki @grafana/observability-logs-and-traces /public/app/plugins/datasource/mssql @grafana/grafana-bi-squad /public/app/plugins/datasource/mysql @grafana/grafana-bi-squad /public/app/plugins/datasource/opentsdb @grafana/backend-platform /public/app/plugins/datasource/postgres @grafana/grafana-bi-squad -/public/app/plugins/datasource/prometheus @grafana/observability-squad +/public/app/plugins/datasource/prometheus @grafana/observability-metrics /public/app/plugins/datasource/cloud-monitoring @grafana/cloud-datasources -/public/app/plugins/datasource/zipkin @grafana/observability-squad -/public/app/plugins/datasource/tempo @grafana/observability-squad +/public/app/plugins/datasource/zipkin @grafana/observability-logs-and-traces +/public/app/plugins/datasource/tempo @grafana/observability-logs-and-traces /public/app/plugins/datasource/alertmanager @grafana/alerting-squad # Cloud middleware diff --git a/.github/renovate.json5 b/.github/renovate.json5 index faeff874218..ffd81f5e49b 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -4,10 +4,12 @@ ], "enabledManagers": ["npm"], "ignoreDeps": [ + "@grafana/slate-react", // should be updated when the `slate` package is updated "@types/systemjs", "@types/d3-force", // we should bump this once we move to esm modules "@types/d3-interpolate", // we should bump this once we move to esm modules "@types/d3-scale-chromatic", // we should bump this once we move to esm modules + "@types/grafana__slate-react", // should be updated when the `slate` package is updated "@types/react-icons", // jaeger-ui-components is being refactored to use @grafana/ui icons instead "d3", "d3-force", // we should bump this once we move to esm modules @@ -21,7 +23,9 @@ "slate", "slate-plain-serializer", "systemjs", + "copy-webpack-plugin", // try to upgrade with newer yarn release. Not working with 3.1.1 "ts-loader", // we should remove ts-loader and use babel-loader instead + "typescript", // >4.5.x doesn't seem to work on windows with pnp currently. probably similar to https://github.com/yarnpkg/berry/issues/3722 "ora" // we should bump this once we move to esm modules ], "ignorePaths": ["packages/grafana-toolkit/package.json", "emails/**", "plugins-bundled/**", "**/mocks/**"], @@ -29,8 +33,14 @@ "packageRules": [ { "matchUpdateTypes": ["patch"], + "excludePackagePatterns": ["@storybook"], "extends": ["schedule:monthly"], "groupName": "Monthly patch updates" + }, + { + "matchPackagePatterns": ["@storybook"], + "extends": ["schedule:monthly"], + "groupName": "Storybook updates" } ], "pin": { diff --git a/.github/workflows/detect-breaking-changes-build.yml b/.github/workflows/detect-breaking-changes-build.yml index 33e9f66519b..c6ceb52d9dd 100644 --- a/.github/workflows/detect-breaking-changes-build.yml +++ b/.github/workflows/detect-breaking-changes-build.yml @@ -3,27 +3,30 @@ name: Levitate / Detect breaking changes on: pull_request jobs: - build: - name: Detect + buildPR: + name: Build PR runs-on: ubuntu-latest - env: - GITHUB_STEP_NUMBER: 7 + defaults: + run: + working-directory: './pr' steps: - uses: actions/checkout@v2 + with: + path: './pr' - - name: Setup environment - uses: actions/setup-node@v2.5.1 + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn config get cacheFolder)" + + - name: Restore yarn cache + uses: actions/cache@v2 + id: yarn-cache with: - node-version: '16' - - - name: Get link for the Github Action job - id: job - uses: actions/github-script@v5 - with: - script: | - const script = require('./.github/workflows/scripts/pr-get-job-link.js') - await script({github, context, core}) + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: yarn-cache-folder-${{ hashFiles('**/yarn.lock', '.yarnrc.yml') }} + restore-keys: | + yarn-cache-folder- - name: Install dependencies run: yarn install --immutable @@ -31,24 +34,108 @@ jobs: - name: Build packages run: yarn packages:build - - name: Detect breaking changes - id: breaking-changes - run: ./scripts/check-breaking-changes.sh - env: - FORCE_COLOR: 3 - GITHUB_JOB_LINK: ${{ steps.job.outputs.link }} + - name: Zip built packages + run: zip -r ./pr_built_packages.zip ./packages/**/dist - - name: Persisting the check output - run: | - mkdir -p ./levitate - echo "{ \"exit_code\": ${{ steps.breaking-changes.outputs.is_breaking }}, \"message\": \"${{ steps.breaking-changes.outputs.message }}\", \"job_link\": \"${{ steps.job.outputs.link }}#step:${GITHUB_STEP_NUMBER}:1\" }" > ./levitate/result.json - - - name: Upload check output as artifact + - name: Upload build output as artifact uses: actions/upload-artifact@v2 with: - name: levitate - path: levitate/ + name: buildPr + path: './pr/pr_built_packages.zip' + + buildMain: + name: Build Main + runs-on: ubuntu-latest + defaults: + run: + working-directory: './main' + + steps: + - uses: actions/checkout@v2 + with: + path: './main' + ref: 'main' + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn config get cacheFolder)" + + - name: Restore yarn cache + uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: yarn-cache-folder-${{ hashFiles('**/yarn.lock', '.yarnrc.yml') }} + restore-keys: | + yarn-cache-folder- - - name: Exit - run: exit ${{ steps.breaking-changes.outputs.is_breaking }} - shell: bash + - name: Install dependencies + run: yarn install --immutable + + - name: Build packages + run: yarn packages:build + + - name: Zip built packages + run: zip -r ./main_built_packages.zip ./packages/**/dist + + - name: Upload build output as artifact + uses: actions/upload-artifact@v2 + with: + name: buildMain + path: './main/main_built_packages.zip' + + Detect: + name: Detect breaking changes + runs-on: ubuntu-latest + needs: ['buildPR', 'buildMain'] + env: + GITHUB_STEP_NUMBER: 7 + + steps: + - uses: actions/checkout@v2 + + - name: Get built packages from pr + uses: actions/download-artifact@v2 + with: + name: buildPr + + - name: Get built packages from main + uses: actions/download-artifact@v2 + with: + name: buildMain + + - name: Unzip artifact from pr + run: unzip pr_built_packages.zip -d ./pr && rm pr_built_packages.zip + + - name: Unzip artifact from main + run: unzip main_built_packages.zip -d ./main && rm main_built_packages.zip + + - name: Get link for the Github Action job + id: job + uses: actions/github-script@v5 + with: + script: | + const script = require('./.github/workflows/scripts/pr-get-job-link.js') + await script({github, context, core}) + + - name: Detect breaking changes + id: breaking-changes + run: ./scripts/check-breaking-changes.sh + env: + FORCE_COLOR: 3 + GITHUB_JOB_LINK: ${{ steps.job.outputs.link }} + + - name: Persisting the check output + run: | + mkdir -p ./levitate + echo "{ \"exit_code\": ${{ steps.breaking-changes.outputs.is_breaking }}, \"message\": \"${{ steps.breaking-changes.outputs.message }}\", \"job_link\": \"${{ steps.job.outputs.link }}#step:${GITHUB_STEP_NUMBER}:1\" }" > ./levitate/result.json + + - name: Upload check output as artifact + uses: actions/upload-artifact@v2 + with: + name: levitate + path: levitate/ + + - name: Exit + run: exit ${{ steps.breaking-changes.outputs.is_breaking }} + shell: bash diff --git a/.github/workflows/milestone.yml b/.github/workflows/milestone.yml new file mode 100644 index 00000000000..d4238e61a2a --- /dev/null +++ b/.github/workflows/milestone.yml @@ -0,0 +1,21 @@ +name: Close Milestone +on: + workflow_dispatch: + inputs: + version_input: + description: 'The version to be released please respect: major.minor.patch or major.minor.patch-beta format. example: 7.4.3 or 7.4.3-beta1' + required: true +jobs: + call-remove-milestone: + uses: grafana/grafana/.github/workflows/remove-milestone.yml@main + with: + version_call: ${{ github.event.inputs.version_input }} + secrets: + token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} + call-close-milestone: + uses: grafana/grafana/.github/workflows/close-milestone.yml@main + with: + version_call: ${{ github.event.inputs.version_input }} + secrets: + token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} + needs: call-remove-milestone \ No newline at end of file diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 69252f7167a..4e26de337b6 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -3,7 +3,9 @@ on: pull_request_target: types: - opened + - reopened - synchronize + - ready_for_review - labeled - unlabeled issues: @@ -14,6 +16,7 @@ on: jobs: main: runs-on: ubuntu-latest + if: github.event.pull_request.draft == false steps: - name: Checkout Actions uses: actions/checkout@v2 diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index ed4c15a2076..3bf5a7b9ead 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -6,19 +6,6 @@ on: description: 'The version to be released please respect: major.minor.patch or major.minor.patch-beta format. example: 7.4.3 or 7.4.3-beta1' required: true jobs: - call-remove-milestone: - uses: grafana/grafana/.github/workflows/remove-milestone.yml@main - with: - version_call: ${{ github.event.inputs.version_input }} - secrets: - token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} - call-close-milestone: - uses: grafana/grafana/.github/workflows/close-milestone.yml@main - with: - version_call: ${{ github.event.inputs.version_input }} - secrets: - token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} - needs: call-remove-milestone call-bump-version: uses: grafana/grafana/.github/workflows/bump-version.yml@main with: @@ -26,7 +13,6 @@ jobs: secrets: token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} metricsWriteAPIKey: ${{ secrets.GRAFANA_MISC_STATS_API_KEY }} - needs: call-close-milestone call-update-changelog: uses: grafana/grafana/.github/workflows/update-changelog.yml@main with: diff --git a/.gitignore b/.gitignore index 3201f07be2a..2ae86906ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ public/dist/tsconfig.tsbuildinfo /emails/dist /reports /e2e/tmp +/scripts/grafana-server/tmp vendor/ /docs/menu.yaml /requests @@ -138,6 +139,10 @@ compilation-stats.json /e2e/**/videos/* /e2e/benchmarks/**/results/* /e2e/benchmarks/**/results +/e2e/build_results.zip + +# grafana server +/scripts/grafana-server/server.log # a11y tests /pa11y-ci-results.json @@ -151,6 +156,7 @@ compilation-stats.json # auto generated Go files *_gen.go +!pkg/services/featuremgmt/toggles_gen.go # Auto-generated localisation files public/locales/_build/ diff --git a/.linguirc b/.linguirc index 383d2faee3b..3a6a7c4b82f 100644 --- a/.linguirc +++ b/.linguirc @@ -2,7 +2,8 @@ "locales": [ "en", "fr", - "es" + "es", + "pseudo-LOCALE" ], "catalogs": [ { @@ -11,14 +12,18 @@ "public/app" ], "exclude": [ + "**/*.d.ts", + "**/*.test.ts", "**/node_modules/**", "public/app/plugins" ] } ], "fallbackLocales": { + "pseudo-LOCALE": "en", "default": "en" }, + "pseudoLocale": "pseudo-LOCALE", "sourceLocale": "en", "format": "po", "formatOptions": { diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 00000000000..53a42214a46 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v16.13.2 diff --git a/.prettierignore b/.prettierignore index 9f8fae45f84..951a41382f5 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,6 +7,7 @@ public/vendor/ vendor/ /data/ e2e/tmp +scripts/grafana-server/tmp public/build/ public/sass/*.generated.scss devenv/ diff --git a/.yarn/sdks/typescript/package.json b/.yarn/sdks/typescript/package.json index fae7e416784..03b3aa3d057 100644 --- a/.yarn/sdks/typescript/package.json +++ b/.yarn/sdks/typescript/package.json @@ -1,6 +1,6 @@ { "name": "typescript", - "version": "4.5.4-sdk", + "version": "4.4.4-sdk", "main": "./lib/typescript.js", "type": "commonjs" } diff --git a/.yarnrc.yml b/.yarnrc.yml index ec1744db672..0a07e82236b 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -9,28 +9,28 @@ packageExtensions: "@mdx-js/loader@1.6.22": peerDependencies: react: 17.0.1 - "@storybook/addon-docs@6.4.9": + "@storybook/addon-docs@6.4.13": peerDependencies: - "@storybook/manager-webpack5": 6.4.9 - "@storybook/addon-essentials@6.4.9": + "@storybook/manager-webpack5": 6.4.13 + "@storybook/addon-essentials@6.4.13": peerDependencies: - "@storybook/components": 6.4.9 - "@storybook/core-events": 6.4.9 - "@storybook/manager-webpack5": 6.4.9 - "@storybook/theming": 6.4.9 - "@storybook/core-server@6.4.9": + "@storybook/components": 6.4.13 + "@storybook/core-events": 6.4.13 + "@storybook/manager-webpack5": 6.4.13 + "@storybook/theming": 6.4.13 + "@storybook/core-server@6.4.13": peerDependencies: "@babel/core": ^7.0.0 - "@storybook/core@6.4.9": + "@storybook/core@6.4.13": peerDependencies: "@babel/core": ^7.0.0 - "@storybook/manager-webpack5": 6.4.9 - "@storybook/csf-tools@6.4.9": + "@storybook/manager-webpack5": 6.4.13 + "@storybook/csf-tools@6.4.13": peerDependencies: "@babel/core": ^7.0.0 - "@storybook/react@6.4.9": + "@storybook/react@6.4.13": peerDependencies: - "@storybook/manager-webpack5": 6.4.9 + "@storybook/manager-webpack5": 6.4.13 doctrine@3.0.0: dependencies: assert: 2.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 80e0d42186e..94e10774dee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,41 @@ + + +# 8.3.4 (2022-01-17) + +### Features and enhancements + +- **Alerting:** Allow configuration of non-ready alertmanagers. [#43063](https://github.com/grafana/grafana/pull/43063), [@alexweav](https://github.com/alexweav) +- **Alerting:** Allow customization of Google chat message. [#43568](https://github.com/grafana/grafana/pull/43568), [@alexweav](https://github.com/alexweav) +- **Alerting:** Allow customization of Google chat message (#43568). [#43723](https://github.com/grafana/grafana/pull/43723), [@alexweav](https://github.com/alexweav) +- **AppPlugins:** Support app plugins with only default nav. [#43016](https://github.com/grafana/grafana/pull/43016), [@torkelo](https://github.com/torkelo) +- **InfluxDB:** InfluxQL: query editor: skip fields in metadata queries. [#42543](https://github.com/grafana/grafana/pull/42543), [@gabor](https://github.com/gabor) +- **Postgres/MySQL/MSSQL:** Cancel in-flight SQL query if user cancels query in grafana. [#43890](https://github.com/grafana/grafana/pull/43890), [@mdvictor](https://github.com/mdvictor) +- **Prometheus:** Forward oauth tokens after prometheus datasource migration. [#43686](https://github.com/grafana/grafana/pull/43686), [@MasslessParticle](https://github.com/MasslessParticle) + +### Bug fixes + +- **Azure Monitor:** Bug fix for variable interpolations in metrics dropdowns. [#43251](https://github.com/grafana/grafana/pull/43251), [@sarahzinger](https://github.com/sarahzinger) +- **Azure Monitor:** Improved error messages for variable queries. [#43213](https://github.com/grafana/grafana/pull/43213), [@sunker](https://github.com/sunker) +- **CloudMonitoring:** Fixes broken variable queries that use group bys. [#43914](https://github.com/grafana/grafana/pull/43914), [@sunker](https://github.com/sunker) +- **Configuration:** You can now see your expired API keys if you have no active ones. [#42452](https://github.com/grafana/grafana/pull/42452), [@ashharrison90](https://github.com/ashharrison90) +- **Elasticsearch:** Fix handling multiple datalinks for a single field. [#44029](https://github.com/grafana/grafana/pull/44029), [@Elfo404](https://github.com/Elfo404) +- **Export:** Fix error being thrown when exporting dashboards using query variables that reference the default datasource. [#44034](https://github.com/grafana/grafana/pull/44034), [@ashharrison90](https://github.com/ashharrison90) +- **ImportDashboard:** Fixes issue with importing dashboard and name ending up in uid. [#43451](https://github.com/grafana/grafana/pull/43451), [@torkelo](https://github.com/torkelo) +- **Login:** Page no longer overflows on mobile. [#43739](https://github.com/grafana/grafana/pull/43739), [@ashharrison90](https://github.com/ashharrison90) +- **Plugins:** Set backend metadata property for core plugins. [#43349](https://github.com/grafana/grafana/pull/43349), [@marefr](https://github.com/marefr) +- **Prometheus:** Fill missing steps with null values. [#43622](https://github.com/grafana/grafana/pull/43622), [@ivanahuckova](https://github.com/ivanahuckova) +- **Prometheus:** Fix interpolation of $\_\_rate_interval variable. [#44035](https://github.com/grafana/grafana/pull/44035), [@ivanahuckova](https://github.com/ivanahuckova) +- **Prometheus:** Interpolate variables with curly brackets syntax. [#42927](https://github.com/grafana/grafana/pull/42927), [@ivanahuckova](https://github.com/ivanahuckova) +- **Prometheus:** Respect the http-method data source setting. [#42753](https://github.com/grafana/grafana/pull/42753), [@gabor](https://github.com/gabor) +- **Table:** Fixes issue with field config applied to wrong fields when hiding columns. [#43376](https://github.com/grafana/grafana/pull/43376), [@torkelo](https://github.com/torkelo) +- **Toolkit:** Fix bug with rootUrls not being properly parsed when signing a private plugin. [#43014](https://github.com/grafana/grafana/pull/43014), [@dessen-xu](https://github.com/dessen-xu) +- **Variables:** Fix so data source variables are added to adhoc configuration. [#43881](https://github.com/grafana/grafana/pull/43881), [@hugohaggmark](https://github.com/hugohaggmark) + +### Plugin development fixes & changes + +- **Toolkit:** Revert build config so tslib is bundled with plugins to prevent plugins from crashing. [#43556](https://github.com/grafana/grafana/pull/43556), [@mckn](https://github.com/mckn) + + # 8.3.3 (2021-12-10) @@ -1222,6 +1260,16 @@ Issue [#33352](https://github.com/grafana/grafana/issues/33352) + + +# 7.5.13 (2022-01-18) + +### Bug fixes + +- **[v7.5.x] Alerting:** Fix NoDataFound for alert rules using AND operator (#41305). [#44066](https://github.com/grafana/grafana/pull/44066), [@armandgrillet](https://github.com/armandgrillet) + + + # 7.5.11 (2021-10-05) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea19d6993c8..de09cda0031 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,6 +80,6 @@ Before we can accept your pull request, you need to [sign our CLA](https://grafa ## Where do I go from here? - Set up your [development environment](contribute/developer-guide.md). -- Learn how to [contribute documentation](contribute/documentation.md). +- Learn how to [contribute documentation](contribute/README.md). - Get started [developing plugins](https://grafana.com/docs/grafana/latest/developers/plugins/) for Grafana. -- Look through the resources in the [contribute](https://github.com/grafana/grafana/tree/main/contribute) folder. +- Look through the resources in the [contribute](contribute) folder. diff --git a/GOVERNANCE.md b/GOVERNANCE.md index d85c66e0825..a9c56deb7a2 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -79,7 +79,7 @@ The current team members are: - Dominik Prokop ([Grafana Labs](https://grafana.com/)) - Emil Tullstedt ([Grafana Labs](https://grafana.com/)) - Erik Sundell ([Grafana Labs](https://grafana.com/)) -- Fredrik Enestad ([Soundtrack Your Brand](https://www.soundtrackyourbrand.com/)) +- Fredrik Enestad ([Embark Studios](https://www.embark-studios.com/)) - Hugo Häggmark ([Grafana Labs](https://grafana.com/)) - Ivana Huckova ([Grafana Labs](https://grafana.com/)) - Jeroen Op 't Eynde ([Grafana Labs](https://grafana.com/)) diff --git a/LICENSING.md b/LICENSING.md index 5c5d7d8f7a3..7506fcdc946 100644 --- a/LICENSING.md +++ b/LICENSING.md @@ -21,7 +21,7 @@ grafana-mixin/ cue/ ``` -The following directories and their subdirectories are the original upstream licenses: +The following directories and their subdirectories are licensed under their original upstream licenses: ``` public/vendor/ diff --git a/conf/defaults.ini b/conf/defaults.ini index 207401eda1a..03aad2655a1 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -259,7 +259,6 @@ cookie_samesite = lax allow_embedding = false # Set to true if you want to enable http strict transport security (HSTS) response header. -# This is only sent when HTTPS is enabled in this configuration. # HSTS tells browsers that the site should only be accessed using HTTPS. strict_transport_security = false @@ -294,8 +293,8 @@ content_security_policy_template = """script-src 'self' 'unsafe-eval' 'unsafe-in [snapshots] # snapshot sharing options external_enabled = true -external_snapshot_url = https://snapshots-origin.raintank.io -external_snapshot_name = Publish to snapshot.raintank.io +external_snapshot_url = https://snapshots.raintank.io +external_snapshot_name = Publish to snapshots.raintank.io # Set to true to enable this Grafana instance act as an external snapshot server and allow unauthenticated requests for # creating and deleting snapshots. @@ -564,6 +563,7 @@ jwk_set_file = cache_ttl = 60m expected_claims = {} key_file = +auto_sign_up = false #################################### Auth LDAP ########################### [auth.ldap] @@ -865,6 +865,11 @@ max_annotations_to_keep = # Enable the Explore section enabled = true +#################################### Query History ############################# +[query_history] +# Enable the Query history +enabled = false + #################################### Internal Grafana Metrics ############ # Metrics available at HTTP API Url /metrics [metrics] diff --git a/conf/sample.ini b/conf/sample.ini index 87ded5d82e2..2a7355fea4d 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -259,7 +259,6 @@ ;allow_embedding = false # Set to true if you want to enable http strict transport security (HSTS) response header. -# This is only sent when HTTPS is enabled in this configuration. # HSTS tells browsers that the site should only be accessed using HTTPS. ;strict_transport_security = false @@ -294,8 +293,8 @@ [snapshots] # snapshot sharing options ;external_enabled = true -;external_snapshot_url = https://snapshots-origin.raintank.io -;external_snapshot_name = Publish to snapshot.raintank.io +;external_snapshot_url = https://snapshots.raintank.io +;external_snapshot_name = Publish to snapshots.raintank.io # Set to true to enable this Grafana instance act as an external snapshot server and allow unauthenticated requests for # creating and deleting snapshots. @@ -548,6 +547,7 @@ ;cache_ttl = 60m ;expected_claims = {"aud": ["foo", "bar"]} ;key_file = /path/to/key/file +;auto_sign_up = false #################################### Auth LDAP ########################## [auth.ldap] @@ -848,6 +848,11 @@ # Enable the Explore section ;enabled = true +#################################### Query History ############################# +[query_history] +# Enable the Query history +;enabled = false + #################################### Internal Grafana Metrics ########################## # Metrics available at HTTP API Url /metrics [metrics] diff --git a/contribute/style-guides/e2e-core.md b/contribute/style-guides/e2e-core.md index 813ec9fe574..fb2c24e0bfc 100644 --- a/contribute/style-guides/e2e-core.md +++ b/contribute/style-guides/e2e-core.md @@ -16,8 +16,8 @@ BASE_URL=http://172.0.10.2:3333 yarn e2e The above commands use some utils scripts under [_\/e2e_](../../e2e) that can also be used for more control. -- `./e2e/start-server` This creates a fresh new grafana server working dir, setup's config and starts the server. It will also kill any previously started server that is still running using pid file at _\/e2e/tmp/pid_. -- `./e2e/wait-for-grafana` waits for `$HOST` and `$PORT` to be available. Per default localhost and 3001. +- `./scripts/grafana-server/start-server` This creates a fresh new grafana server working dir, setup's config and starts the server. It will also kill any previously started server that is still running using pid file at _\/scripts/grafana-server/tmp/pid_. +- `./scripts/grafana-server/wait-for-grafana` waits for `$HOST` and `$PORT` to be available. Per default localhost and 3001. - `./e2e/run-suite ` Starts cypress in different modes. ## Test suites diff --git a/devenv/datasources.yaml b/devenv/datasources.yaml index 883c2ef2c3b..94b419e3cca 100644 --- a/devenv/datasources.yaml +++ b/devenv/datasources.yaml @@ -255,6 +255,17 @@ datasources: secureJsonData: password: Password! + - name: gdev-mssql-tls + type: mssql + url: localhost:1434 + database: grafana + user: grafana + jsonData: + encrypt: "true" + tlsSkipVerify: true + secureJsonData: + password: Password! + - name: gdev-mssql-ds-tests type: mssql url: localhost:1433 diff --git a/devenv/datasources_docker.yaml b/devenv/datasources_docker.yaml index 922a1ccc1c5..833960cc4ae 100644 --- a/devenv/datasources_docker.yaml +++ b/devenv/datasources_docker.yaml @@ -42,7 +42,7 @@ datasources: version: Flux organization: myorg defaultBucket: mybucket - + - name: gdev-influxdb-influxql type: influxdb access: proxy @@ -220,6 +220,17 @@ datasources: secureJsonData: password: Password! + - name: gdev-mssql-tls + type: mssql + url: localhost:1434 + database: grafana + user: grafana + jsonData: + encrypt: "true" + tlsSkipVerify: true + secureJsonData: + password: Password! + - name: gdev-mssql-ds-tests type: mssql url: mssqltests:1433 @@ -259,4 +270,4 @@ datasources: type: loki access: proxy url: http://loki:3100 - editable: false \ No newline at end of file + editable: false diff --git a/devenv/docker/blocks/mssql_tls/build/Dockerfile b/devenv/docker/blocks/mssql_tls/build/Dockerfile new file mode 100644 index 00000000000..1c6399f14d7 --- /dev/null +++ b/devenv/docker/blocks/mssql_tls/build/Dockerfile @@ -0,0 +1,21 @@ +FROM mcr.microsoft.com/mssql/server:2019-CU8-ubuntu-18.04 + +WORKDIR /usr/setup +COPY setup.sh setup.sql.template entrypoint.sh ./ +COPY mssql.conf /var/opt/mssql/mssql.conf + +USER root + +RUN chmod +x setup.sh +RUN chown -R mssql ./ +RUN mkdir -p /home/mssql +RUN chown -R mssql /home/mssql + +USER mssql + +RUN touch ~/.rnd +RUN openssl req -x509 -nodes -newkey rsa:2048 -subj '/CN=mssql_tls' -keyout /var/opt/mssql/mssql.key -out /var/opt/mssql/mssql.pem -days 365 +RUN chmod 440 /var/opt/mssql/mssql.key +RUN chmod 440 /var/opt/mssql/mssql.pem + +CMD /bin/bash ./entrypoint.sh diff --git a/devenv/docker/blocks/mssql_tls/build/entrypoint.sh b/devenv/docker/blocks/mssql_tls/build/entrypoint.sh new file mode 100644 index 00000000000..de5aef5124a --- /dev/null +++ b/devenv/docker/blocks/mssql_tls/build/entrypoint.sh @@ -0,0 +1,2 @@ +#start SQL Server and run setup script +/usr/setup/setup.sh & /opt/mssql/bin/sqlservr \ No newline at end of file diff --git a/devenv/docker/blocks/mssql_tls/build/mssql.conf b/devenv/docker/blocks/mssql_tls/build/mssql.conf new file mode 100644 index 00000000000..830ea13f9ff --- /dev/null +++ b/devenv/docker/blocks/mssql_tls/build/mssql.conf @@ -0,0 +1,5 @@ +[network] +tlscert = /var/opt/mssql/mssql.pem +tlskey = /var/opt/mssql/mssql.key +tlsprotocols = 1.2 +forceencryption = 1 diff --git a/devenv/docker/blocks/mssql_tls/build/setup.sh b/devenv/docker/blocks/mssql_tls/build/setup.sh new file mode 100644 index 00000000000..5d22062bd2f --- /dev/null +++ b/devenv/docker/blocks/mssql_tls/build/setup.sh @@ -0,0 +1,13 @@ +#/bin/bash +set -eo pipefail + +#wait for the SQL Server to come up +sleep 15s + +cat /usr/setup/setup.sql.template | awk '{ + gsub(/%%DB%%/,"'$MSSQL_DATABASE'"); + gsub(/%%USER%%/,"'$MSSQL_USER'"); + gsub(/%%PWD%%/,"'$MSSQL_PASSWORD'") +}1' > /usr/setup/setup.sql + +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i /usr/setup/setup.sql diff --git a/devenv/docker/blocks/mssql_tls/build/setup.sql.template b/devenv/docker/blocks/mssql_tls/build/setup.sql.template new file mode 100644 index 00000000000..5ff4e194df3 --- /dev/null +++ b/devenv/docker/blocks/mssql_tls/build/setup.sql.template @@ -0,0 +1,26 @@ +CREATE LOGIN %%USER%% WITH PASSWORD = '%%PWD%%' +GO + +CREATE DATABASE %%DB%% +ON +( NAME = %%DB%%, + FILENAME = '/var/opt/mssql/data/%%DB%%.mdf', + SIZE = 500MB, + MAXSIZE = 1000MB, + FILEGROWTH = 100MB ) +LOG ON +( NAME = %%DB%%_log, + FILENAME = '/var/opt/mssql/data/%%DB%%_log.ldf', + SIZE = 500MB, + MAXSIZE = 1000MB, + FILEGROWTH = 100MB ); +GO + +USE %%DB%%; +GO + +CREATE USER %%USER%% FOR LOGIN %%USER%%; +GO + +EXEC sp_addrolemember 'db_owner', '%%USER%%'; +GO diff --git a/devenv/docker/blocks/mssql_tls/docker-compose.yaml b/devenv/docker/blocks/mssql_tls/docker-compose.yaml new file mode 100644 index 00000000000..ab8599666c5 --- /dev/null +++ b/devenv/docker/blocks/mssql_tls/docker-compose.yaml @@ -0,0 +1,18 @@ + mssql_tls: + build: + context: docker/blocks/mssql_tls/build + environment: + ACCEPT_EULA: Y + MSSQL_SA_PASSWORD: Password! + MSSQL_PID: Developer + MSSQL_DATABASE: grafana + MSSQL_USER: grafana + MSSQL_PASSWORD: Password! + ports: + - "1434:1433" + + fake-mssql-tls-data: + image: grafana/fake-data-gen + environment: + FD_DATASOURCE: mssql_tls + FD_PORT: 1434 diff --git a/devenv/docker/blocks/mysql_tests/Dockerfile b/devenv/docker/blocks/mysql_tests/Dockerfile index 63bc031a682..fb19cacd6cc 100644 --- a/devenv/docker/blocks/mysql_tests/Dockerfile +++ b/devenv/docker/blocks/mysql_tests/Dockerfile @@ -1,4 +1,5 @@ ARG mysql_version=5.6 FROM mysql:${mysql_version} ADD setup.sql /docker-entrypoint-initdb.d +RUN chown -R mysql:mysql /docker-entrypoint-initdb.d/ CMD ["mysqld"] diff --git a/devenv/docker/blocks/postgres_tests/Dockerfile b/devenv/docker/blocks/postgres_tests/Dockerfile index f8deb7d6815..46423b06ce0 100644 --- a/devenv/docker/blocks/postgres_tests/Dockerfile +++ b/devenv/docker/blocks/postgres_tests/Dockerfile @@ -1,4 +1,5 @@ ARG postgres_version=9.3 FROM postgres:${postgres_version} ADD setup.sql /docker-entrypoint-initdb.d +RUN chown -R postgres:postgres /docker-entrypoint-initdb.d/ CMD ["postgres"] diff --git a/docs/README.md b/docs/README.md index 03257e72aa5..9d6abafad4d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,7 +20,7 @@ If you have the grafana/website repo checked out in the same directory as the gr Edit content in the `sources` directory. -### [Contributing](https://github.com/grafana/grafana/blob/main/contribute/documentation.md) +### [Contributing](/contribute/documentation/README.md) ### Using `relref` for internal links diff --git a/docs/sources/administration/configuration.md b/docs/sources/administration/configuration.md index d802fe3ecc8..088d8fdc9d3 100644 --- a/docs/sources/administration/configuration.md +++ b/docs/sources/administration/configuration.md @@ -546,7 +546,7 @@ mitigate the risk of [Clickjacking](https://owasp.org/www-community/attacks/Clic ### strict_transport_security -Set to `true` if you want to enable HTTP `Strict-Transport-Security` (HSTS) response header. This is only sent when HTTPS is enabled in this configuration. HSTS tells browsers that the site should only be accessed using HTTPS. +Set to `true` if you want to enable HTTP `Strict-Transport-Security` (HSTS) response header. Only use this when HTTPS is enabled in your configuration, or when there is another upstream system that ensures your application does HTTPS (like a frontend load balancer). HSTS tells browsers that the site should only be accessed using HTTPS. ### strict_transport_security_max_age_seconds @@ -586,11 +586,11 @@ Set to `false` to disable external snapshot publish endpoint (default `true`). ### external_snapshot_url -Set root URL to a Grafana instance where you want to publish external snapshots (defaults to https://snapshots-origin.raintank.io). +Set root URL to a Grafana instance where you want to publish external snapshots (defaults to https://snapshots.raintank.io). ### external_snapshot_name -Set name for external snapshot button. Defaults to `Publish to snapshot.raintank.io`. +Set name for external snapshot button. Defaults to `Publish to snapshots.raintank.io`. ### public_mode @@ -631,7 +631,7 @@ Path to the default home dashboard. If this value is empty, then Grafana uses St Set to `false` to prohibit users from being able to sign up / create user accounts. Default is `false`. The admin user can still create -users from the [Grafana Admin Pages](/reference/admin). +users from the [Grafana Admin Pages]({{< relref "../manage-users/server-admin/server-admin-manage-users.md" >}}). ### allow_org_create @@ -890,8 +890,6 @@ Email server settings. Enable this to allow Grafana to send email. Default is `false`. -If the password contains `#` or `;`, then you have to wrap it with triple quotes. Example: """#password;""" - ### host Default is `localhost:25`. @@ -902,7 +900,7 @@ In case of SMTP auth, default is `empty`. ### password -In case of SMTP auth, default is `empty`. +In case of SMTP auth, default is `empty`. If the password contains `#` or `;`, then you have to wrap it with triple quotes. Example: """#password;""" ### cert_file diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index 92838238730..910e0512ef0 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -147,8 +147,8 @@ Since not all datasources have the same configuration settings we only have the | -------------------------- | ------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | tlsAuth | boolean | _HTTP\*_, MySQL | Enable TLS authentication using client cert configured in secure json data | | tlsAuthWithCACert | boolean | _HTTP\*_, MySQL, PostgreSQL | Enable TLS authentication using CA cert | -| tlsSkipVerify | boolean | _HTTP\*_, MySQL, PostgreSQL | Controls whether a client verifies the server's certificate chain and host name. | -| serverName | string | _HTTP\*_ | Optional. Controls the server name used for certificate common name/subject alternative name verification. Defaults to using the data source URL. | +| tlsSkipVerify | boolean | _HTTP\*_, MySQL, PostgreSQL, MSSQL | Controls whether a client verifies the server's certificate chain and host name. | +| serverName | string | _HTTP\*_, MSSQL | Optional. Controls the server name used for certificate common name/subject alternative name verification. Defaults to using the data source URL. | | timeout | string | _HTTP\*_ | Request timeout in seconds. Overrides dataproxy.timeout option | | graphiteVersion | string | Graphite | Graphite version | | timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL and MSSQL | Lowest interval/step value that should be used for this data source. | @@ -179,7 +179,7 @@ Since not all datasources have the same configuration settings we only have the | tsdbResolution | string | OpenTSDB | Resolution | | sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | | tlsConfigurationMethod | string | PostgreSQL | SSL Certificate configuration, either by 'file-path' or 'file-content' | -| sslRootCertFile | string | PostgreSQL | SSL server root certificate file, must be readable by the Grafana user | +| sslRootCertFile | string | PostgreSQL, MSSQL | SSL server root certificate file, must be readable by the Grafana user | | sslCertFile | string | PostgreSQL | SSL client certificate file, must be readable by the Grafana user | | sslKeyFile | string | PostgreSQL | SSL client key file, must be readable by _only_ the Grafana user | | encrypt | string | MSSQL | Connection SSL encryption handling. 'disable', 'false' or 'true' | diff --git a/docs/sources/alerting/unified-alerting/alerting-rules/create-cortex-loki-managed-rule.md b/docs/sources/alerting/unified-alerting/alerting-rules/create-cortex-loki-managed-rule.md index 6af1a5c9c7f..49197362786 100644 --- a/docs/sources/alerting/unified-alerting/alerting-rules/create-cortex-loki-managed-rule.md +++ b/docs/sources/alerting/unified-alerting/alerting-rules/create-cortex-loki-managed-rule.md @@ -11,11 +11,13 @@ Grafana allows you to create alerting rules for an external Cortex or Loki insta ## Before you begin -For Cortex and Loki data sources to work with Grafana 8.0 alerting, enable the ruler API by configuring their respective services. +- Verify that you have write permission to the Prometheus data source. Otherwise, you will not be able to create or update Cortex managed alerting rules. -**Loki** - The `local` rule storage type, default for the Loki data source, supports only viewing of rules. To edit rules, configure one of the other rule storage types. +- For Cortex and Loki data sources, enable the ruler API by configuring their respective services. -**Cortex** - When configuring a Grafana Prometheus data source to point to Cortex, use the [legacy `/api/prom` prefix](https://cortexmetrics.io/docs/api/#path-prefixes), not `/prometheus`. The Prometheus data source supports both Cortex and Prometheus, and Grafana expects that both the [Query API](https://cortexmetrics.io/docs/api/#querier--query-frontend) and [Ruler API](https://cortexmetrics.io/docs/api/#ruler) are under the same URL. You cannot provide a separate URL for the Ruler API. + - **Loki** - The `local` rule storage type, default for the Loki data source, supports only viewing of rules. To edit rules, configure one of the other rule storage types. + + - **Cortex** - use the [legacy `/api/prom` prefix](https://cortexmetrics.io/docs/api/#path-prefixes), not `/prometheus`. The Prometheus data source supports both Cortex and Prometheus, and Grafana expects that both the [Query API](https://cortexmetrics.io/docs/api/#querier--query-frontend) and [Ruler API](https://cortexmetrics.io/docs/api/#ruler) are under the same URL. You cannot provide a separate URL for the Ruler API. > **Note:** If you do not want to manage alerting rules for a particular Loki or Prometheus data source, go to its settings and clear the **Manage alerts via Alerting UI** checkbox. diff --git a/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md b/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md index 10c8e04a3ce..2acc19046d1 100644 --- a/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md +++ b/docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md @@ -20,7 +20,7 @@ Grafana allows you to create alerting rules that query one or more data sources, 1. In Step 2, add queries and expressions to evaluate. - Keep the default name or hover over and click the edit icon to change the name. - For queries, select a data source from the drop-down. - - Add one or more [queries]({{< relref "../../../panels/queries.md" >}}) or [expressions]({{< relref "../../../panels/expressions.md" >}}). + - Add one or more [queries]({{< relref "../../../panels/query-a-data-source/add-a-query.md" >}}) or [expressions]({{< relref "../../../panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions.md" >}}). - For each expression, select either **Classic condition** to create a single alert rule, or choose from **Math**, **Reduce**, **Resample** options to generate separate alert for each series. For details on these options, see [Single and multi dimensional rule](#single-and-multi-dimensional-rule). - Click **Run queries** to verify that the query is successful. 1. In Step 3, add conditions. @@ -57,7 +57,7 @@ To generate a separate alert for each series, create a multi-dimensional rule. U #### Rule with classic condition -For more information, see [expressions documentation]({{< relref "../../../panels/expressions.md" >}}). +For more information, see [expressions documentation]({{< relref "../../../panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions.md" >}}). ### No data and error handling diff --git a/docs/sources/auth/github.md b/docs/sources/auth/github.md index 5cf1bbb5dfa..06c02185c2a 100644 --- a/docs/sources/auth/github.md +++ b/docs/sources/auth/github.md @@ -16,7 +16,7 @@ settings page). When you create the application you will need to specify a callback URL. Specify this as callback: ```bash -http://:/login/github +http://:/grafana/login/github ``` This callback URL must match the full HTTP address that you use in your diff --git a/docs/sources/auth/jwt.md b/docs/sources/auth/jwt.md index c484baac9fa..aa1cda58c3e 100644 --- a/docs/sources/auth/jwt.md +++ b/docs/sources/auth/jwt.md @@ -44,8 +44,13 @@ username_claim = sub # Specify a claim to use as an email to sign in. email_claim = sub + +# auto-create users if they are not already matched +# auto_sign_up = true ``` +If `auto_sign_up` is enabled, then the `sub` claim is used as the "external Auth ID". The `name` claim is used as the user's full name if it is present. + ## Signature verification JSON web token integrity needs to be verified so cryptographic signature is used for this purpose. So we expect that every token must be signed with some known cryptographic key. diff --git a/docs/sources/best-practices/best-practices-for-creating-dashboards.md b/docs/sources/best-practices/best-practices-for-creating-dashboards.md index a9d5292dc50..48befdb8ed7 100644 --- a/docs/sources/best-practices/best-practices-for-creating-dashboards.md +++ b/docs/sources/best-practices/best-practices-for-creating-dashboards.md @@ -49,6 +49,6 @@ Once you have a strategy or design guidelines, write them down to help maintain - Use the left and right Y-axes when displaying time series with different units or ranges. - Add documentation to dashboards and panels. - To add documentation to a dashboard, add a [Text panel visualization]({{< relref "../visualizations/text-panel.md" >}}) to the dashboard. Record things like the purpose of the dashboard, useful resource links, and any instructions users might need to interact with the dashboard. Check out this [Wikimedia example](https://grafana.wikimedia.org/d/000000066/resourceloader?orgId=1). - - To add documentation to a panel, [edit the panel settings]({{< relref "../panels/add-a-panel.md#edit-panel-settings" >}}) and add a description. Any text you add will appear if you hover your cursor over the small `i` in the top left corner of the panel. + - To add documentation to a panel, [edit the panel settings]({{< relref "../panels/working-with-panels/add-panel.md" >}}) and add a description. Any text you add will appear if you hover your cursor over the small `i` in the top left corner of the panel. - Reuse your dashboards and enforce consistency by using [templates and variables]({{< relref "../variables/_index.md" >}}). - Be careful with stacking graph data. The visualizations can be misleading, and hide important data. We recommend turning it off in most cases. diff --git a/docs/sources/best-practices/dashboard-management-maturity-levels.md b/docs/sources/best-practices/dashboard-management-maturity-levels.md index d3e44910be7..268cd83d7dd 100644 --- a/docs/sources/best-practices/dashboard-management-maturity-levels.md +++ b/docs/sources/best-practices/dashboard-management-maturity-levels.md @@ -46,7 +46,7 @@ How can you tell you are here? - Compare like to like: split service dashboards when the magnitude differs. Make sure aggregated metrics don't drown out important information. - Expressive charts with meaningful use of color and normalizing axes where you can. - - Example of meaningful color: Blue means it's good, red means it's bad. [Thresholds]({{< relref "../panels/thresholds.md" >}}) can help with that. + - Example of meaningful color: Blue means it's good, red means it's bad. [Thresholds]({{< relref "../panels/specify-thresholds/about-thresholds.md" >}}) can help with that. - Example of normalizing axes: When comparing CPU usage, measure by percentage rather than raw number, because machines can have a different number of cores. Normalizing CPU usage by the number of cores reduces cognitive load because the viewer can trust that at 100% all cores are being used, without having to know the number of CPUs. - Directed browsing cuts down on "guessing." - Template variables make it harder to “just browse” randomly or aimlessly. diff --git a/docs/sources/dashboards/time-range-controls.md b/docs/sources/dashboards/time-range-controls.md index 3bc88cf25e0..0aa3d7208c0 100644 --- a/docs/sources/dashboards/time-range-controls.md +++ b/docs/sources/dashboards/time-range-controls.md @@ -7,7 +7,7 @@ weight = 7 # Time range controls -Grafana provides several ways to manage the time ranges of the data being visualized, both at the dashboard level and the panel level. +Grafana provides several ways to manage the time ranges of the data being visualized, for dashboard, panels and also for alerting. This page describes supported time units and relative ranges, the common time controls, dashboard-wide time settings, and panel-specific time settings. @@ -34,11 +34,18 @@ Here are some examples: | This Year | `now/Y` | `now/Y` | | Previous fiscal year | `now-1y/fy` | `now-1y/fy` | +### Note about Grafana alerting + +For Grafana alerting, we do not support are the following syntaxes at this time. + +- now+n for future timestamps. +- now-1n/n for "start of n until end of n" since this is an absolute timestamp. + ## Common time range controls The dashboard and panel time controls have a common user interface (UI). - + The options are defined below. @@ -110,7 +117,7 @@ For more advanced time settings, click the **Dashboard settings** (gear) icon at ## Panel time overrides and timeshift -In [Query options]({{< relref "../panels/queries.md#query-options" >}}), you can override the relative time range for individual panels, causing them to be different than what is selected in the dashboard time picker in the upper right. This allows you to show metrics from different time periods or days at the same time. +In [Query options]({{< relref "../panels/reference-query-options.md" >}}), you can override the relative time range for individual panels, causing them to be different than what is selected in the dashboard time picker in the upper right. This allows you to show metrics from different time periods or days at the same time. > **Note:** Panel time overrides have no effect when the time range for the dashboard is absolute. diff --git a/docs/sources/datasources/influxdb/influxdb-flux.md b/docs/sources/datasources/influxdb/influxdb-flux.md index c0fa0eb5b80..2da265372dc 100644 --- a/docs/sources/datasources/influxdb/influxdb-flux.md +++ b/docs/sources/datasources/influxdb/influxdb-flux.md @@ -74,4 +74,4 @@ from(bucket: "grafana") |> yield(name: "mean") ``` -You can view the interpolated version of a query with the query inspector. For more information, refer to [Inspect a panel]({{< relref "../../panels/inspect-panel.md" >}}) and [Queries]({{< relref "../../panels/queries.md" >}}). +You can view the interpolated version of a query with the query inspector. For more information, refer to [Navigate the Query Inspector]({{< relref "../../panels/working-with-panels/navigate-inspector-panel.md" >}}). diff --git a/docs/sources/datasources/mssql.md b/docs/sources/datasources/mssql.md index 89b04ec23ac..6827b580cef 100644 --- a/docs/sources/datasources/mssql.md +++ b/docs/sources/datasources/mssql.md @@ -174,7 +174,7 @@ A time series query result is returned in a [wide data frame format]({{< relref > For backward compatibility, there's an exception to the above rule for queries that return three columns including a string column named metric. Instead of transforming the metric column into field labels, it becomes the field name, and then the series name is formatted as the value of the metric column. See the example with the metric column below. -You can optionally customize the default series name formatting using instructions in [Standard field options/Display name]({{< relref "../panels/standard-options.md#display-name" >}}). +You can optionally customize the default series name formatting using instructions in [Reference: Standard field definitions]({{< relref "../panels/reference-standard-field-definitions.md#display-name" >}}). **Example with `metric` column:** @@ -218,7 +218,7 @@ GROUP BY ORDER BY 1 ``` -Given the data frame result in the following example and using the graph panel, you will get two series named _value 10.0.1.1_ and _value 10.0.1.2_. To render the series with a name of _10.0.1.1_ and _10.0.1.2_ , use a [Standard field options/Display name]({{< relref "../panels/standard-options.md#display-name" >}}) value of `${__field.labels.hostname}`. +Given the data frame result in the following example and using the graph panel, you will get two series named _value 10.0.1.1_ and _value 10.0.1.2_. To render the series with a name of _10.0.1.1_ and _10.0.1.2_ , use a [Reference: Standard field definitions]({{< relref "../panels/reference-standard-field-definitions.md#display-name" >}}) display name value of `${__field.labels.hostname}`. Data frame result: diff --git a/docs/sources/datasources/mysql.md b/docs/sources/datasources/mysql.md index 8a98f686494..ab0b7982c96 100644 --- a/docs/sources/datasources/mysql.md +++ b/docs/sources/datasources/mysql.md @@ -186,7 +186,7 @@ A time series query result is returned in a [wide data frame format]({{< relref > For backward compatibility, there's an exception to the above rule for queries that return three columns including a string column named metric. Instead of transforming the metric column into field labels, it becomes the field name, and then the series name is formatted as the value of the metric column. See the example with the metric column below. -You can optionally customize the default series name formatting using instructions in [Standard field options/Display name]({{< relref "../panels/standard-options.md#display-name" >}}). +You can optionally customize the default series name formatting using instructions in [Reference: Standard field definitions]({{< relref "../panels/reference-standard-field-definitions.md#display-name" >}}). **Example with `metric` column:** @@ -228,7 +228,7 @@ GROUP BY time, hostname ORDER BY time ``` -Given the data frame result in the following example and using the graph panel, you will get two series named _value 10.0.1.1_ and _value 10.0.1.2_. To render the series with a name of _10.0.1.1_ and _10.0.1.2_ , use a [Standard field options/Display name]({{< relref "../panels/standard-options.md#display-name" >}}) value of `${__field.labels.hostname}`. +Given the data frame result in the following example and using the graph panel, you will get two series named _value 10.0.1.1_ and _value 10.0.1.2_. To render the series with a name of _10.0.1.1_ and _10.0.1.2_ , use a [Reference: Standard field definitions]({{< relref "../panels/reference-standard-field-definitions.md#display-name" >}}) display value of `${__field.labels.hostname}`. Data frame result: diff --git a/docs/sources/datasources/postgres.md b/docs/sources/datasources/postgres.md index 3800cc33d96..3e18046d530 100644 --- a/docs/sources/datasources/postgres.md +++ b/docs/sources/datasources/postgres.md @@ -12,7 +12,7 @@ Grafana ships with a built-in PostgreSQL data source plugin that allows you to q ## PostgreSQL settings -To access PostgreSQL settings, hover your mouse over the **Configuration** (gear) icon, then click **Data Sources**, and then click the Prometheus data source. +To access PostgreSQL settings, hover your mouse over the **Configuration** (gear) icon, then click **Data Sources**, and then click the PostgreSQL data source. | Name | Description | | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -191,7 +191,7 @@ A time series query result is returned in a [wide data frame format]({{< relref > For backward compatibility, there's an exception to the above rule for queries that return three columns including a string column named metric. Instead of transforming the metric column into field labels, it becomes the field name, and then the series name is formatted as the value of the metric column. See the example with the metric column below. -You can optionally customize the default series name formatting using instructions in [Standard field options/Display name]({{< relref "../panels/standard-options.md#display-name" >}}). +You can optionally customize the default series name formatting using instructions in [Reference: Standard field definitions]({{< relref "../panels/reference-standard-field-definitions.md#display-name" >}}). **Example with `metric` column:** @@ -233,7 +233,7 @@ GROUP BY time, hostname ORDER BY time ``` -Given the data frame result in the following example and using the graph panel, you will get two series named _value 10.0.1.1_ and _value 10.0.1.2_. To render the series with a name of _10.0.1.1_ and _10.0.1.2_ , use a [Standard field options/Display name]({{< relref "../panels/standard-options.md#display-name" >}}) value of `${__field.labels.hostname}`. +Given the data frame result in the following example and using the graph panel, you will get two series named _value 10.0.1.1_ and _value 10.0.1.2_. To render the series with a name of _10.0.1.1_ and _10.0.1.2_ , use a [Reference: Standard field definitions]({{< relref "../panels/reference-standard-field-definitions.md#display-name" >}}) display value of `${__field.labels.hostname}`. Data frame result: diff --git a/docs/sources/datasources/prometheus.md b/docs/sources/datasources/prometheus.md index ed3148eecfe..82f0dfd83c5 100644 --- a/docs/sources/datasources/prometheus.md +++ b/docs/sources/datasources/prometheus.md @@ -31,6 +31,7 @@ To access Prometheus settings, hover your mouse over the **Configuration** (gear | `Custom Query Parameters` | Add custom parameters to the Prometheus query URL. For example `timeout`, `partial_response`, `dedup`, or `max_source_resolution`. Multiple parameters should be concatenated together with an '&'. | | `Label name` | Add the name of the field in the label object. | | `URL` | If the link is external, then enter the full link URL. You can interpolate the value from the field with `${__value.raw }` macro. | +| `URL Label` | (Optional) Set a custom display label for the link URL. The link label defaults to the full external URL or the name of datasource and is overridden by this setting. | | `Internal link` | Select if the link is internal or external. In the case of an internal link, a data source selector allows you to select the target data source. Supports tracing data sources only. | ## Prometheus query editor diff --git a/docs/sources/developers/plugins/_index.md b/docs/sources/developers/plugins/_index.md index cd20612d542..64a4a13f582 100644 --- a/docs/sources/developers/plugins/_index.md +++ b/docs/sources/developers/plugins/_index.md @@ -17,6 +17,8 @@ Open the terminal, and run the following command in your [plugin directory]({{< npx @grafana/toolkit plugin:create my-grafana-plugin ``` +> **Note:** If running NPM 7+ the `npx` commands mentioned in this article may hang. The workaround is to use `npx --legacy-peer-deps `. + If you want a more guided introduction to plugin development, check out our tutorials: - [Build a panel plugin]({{< relref "/tutorials/build-a-panel-plugin.md" >}}) diff --git a/docs/sources/developers/plugins/data-frames.md b/docs/sources/developers/plugins/data-frames.md index 2eb1eaa073f..1c85e563c85 100644 --- a/docs/sources/developers/plugins/data-frames.md +++ b/docs/sources/developers/plugins/data-frames.md @@ -163,7 +163,7 @@ Dimensions: 5 fields by 2 rows +---------------------+------------------+------------------+------------------+------------------+ ``` -> **Note:** Not all panels support the wide time series data frame format. To keep full backward compatibility we have introduced a transformation that can be used to convert from the wide to the long format. Read more about how to use it here: [Prepare time series-transformation]({{< relref "../../panels/transformations/types-options.md#prepare-time-series" >}}). +> **Note:** Not all panels support the wide time series data frame format. To keep full backward compatibility we have introduced a transformation that can be used to convert from the wide to the long format. Read more about how to use it here: [Prepare time series-transformation]({{< relref "../../panels/reference-transformation-functions.md#prepare-time-series" >}}). ## Technical references diff --git a/docs/sources/developers/plugins/legacy/snapshot-mode.md b/docs/sources/developers/plugins/legacy/snapshot-mode.md index 6f98e5d2364..b8f6c96a058 100644 --- a/docs/sources/developers/plugins/legacy/snapshot-mode.md +++ b/docs/sources/developers/plugins/legacy/snapshot-mode.md @@ -7,7 +7,7 @@ aliases = ["/docs/grafana/latest/plugins/developing/snapshot-mode/"] {{< figure class="float-right" src="/static/img/docs/Grafana-snapshot-example.png" caption="A dashboard using snapshot data and not live data." >}} -Grafana has this great feature where you can [save a snapshot of your dashboard]({{< relref "../../../dashboards/json-model.md" >}}). Instead of sending a screenshot of a dashboard to someone, you can send them a working, interactive Grafana dashboard with the snapshot data embedded inside it. The snapshot can be saved on your Grafana server and is available to all your co-workers. Raintank also hosts a [snapshot server](http://snapshot.raintank.io/) if you want to send the snapshot to someone who does not have access to your Grafana server. +Grafana has this great feature where you can [save a snapshot of your dashboard]({{< relref "../../../dashboards/json-model.md" >}}). Instead of sending a screenshot of a dashboard to someone, you can send them a working, interactive Grafana dashboard with the snapshot data embedded inside it. The snapshot can be saved on your Grafana server and is available to all your co-workers. Raintank also hosts a [snapshot server](https://snapshots.raintank.io) if you want to send the snapshot to someone who does not have access to your Grafana server. {{< figure class="float-right" src="/static/img/docs/animated_gifs/snapshots.gif" caption="Selecting a snapshot" >}} @@ -72,4 +72,4 @@ loadLocationDataFromFile(reload) { It is really easy to forget to add this support but it enables a great feature and can be used to demo your panel. -If there is a panel plugin that you would like to be installed on the Raintank Snapshot server then please contact us via [Slack](https://raintank.slack.com) or [GitHub](https://github.com/grafana/grafana). +If there is a panel plugin that you would like to be installed on the Raintank Snapshot server then please contact us via [Slack](https://slack.grafana.com) or [GitHub](https://github.com/grafana/grafana). diff --git a/docs/sources/developers/plugins/migration-guide.md b/docs/sources/developers/plugins/migration-guide.md index 8261bc77b7f..a0f87d22cf9 100644 --- a/docs/sources/developers/plugins/migration-guide.md +++ b/docs/sources/developers/plugins/migration-guide.md @@ -12,7 +12,10 @@ This guide helps you identify the steps you need to take based on the Grafana ve ## Table of contents -- [From version 7.x.x to 8.0.0](#from-version-7xx-to-800) +- [From version 8.3.x to 8.4.x](#from-version-83x-to-84x) + - [Value Mapping Editor has been removed from @grafana-ui library](#value-mapping-editor-has-been-removed-from-grafana-ui-library) + - [Thresholds Editor has been removed from @grafana-ui library](#thresholds-editor-has-been-removed-from-grafana-ui-library) +- [From version 7.x.x to 8.x.x](#from-version-7xx-to-8xx) - [Backend plugin v1 support has been dropped](#backend-plugin-v1-support-has-been-dropped) - [1. Add dependency on grafana-plugin-sdk-go](#1-add-dependency-on-grafana-plugin-sdk-go) - [2. Update the way you bootstrap your plugin](#2-update-the-way-you-bootstrap-your-plugin) @@ -33,6 +36,18 @@ This guide helps you identify the steps you need to take based on the Grafana ve - [Migrate to data frames](#migrate-to-data-frames) - [Troubleshoot plugin migration](#troubleshoot-plugin-migration) +## From version 8.3.x to 8.4.x + +This section explains how to migrate Grafana v8.3.x plugins to the updated plugin system available in Grafana v8.4.x. Depending on your plugin, you need to perform one or more of the following steps. + +### Value Mapping Editor has been removed from @grafana-ui library + +Removed due to being an internal component. + +### Thresholds Editor has been removed from @grafana-ui library + +Removed due to being an internal component. + ## From version 7.x.x to 8.x.x This section explains how to migrate Grafana v7.x.x plugins to the updated plugin system available in Grafana v8.x.x. Depending on your plugin, you need to perform one or more of the following steps. We have documented the breaking changes in Grafana v8.x.x and the steps you need to take to upgrade your plugin. @@ -176,7 +191,7 @@ We strongly recommend that you not allow unsigned plugins in your Grafana instal To sign your plugin, see [Sign a plugin](https://grafana.com/docs/grafana/latest/developers/plugins/sign-a-plugin/#sign-a-plugin). -You can still run and develop an unsigned plugin by running your Grafana instance in [development mode](https://grafana.com/docs/grafana/latest/administration/configuration/#app_mode). Alternatively, you can use the [allow_loading_unsigned_plugins configuration setting.]({{< relref "../../administration/#allow_loading_unsigned_plugins" >}}) +You can still run and develop an unsigned plugin by running your Grafana instance in [development mode](https://grafana.com/docs/grafana/latest/administration/configuration/#app_mode). Alternatively, you can use the [allow_loading_unsigned_plugins configuration setting.](../../administration/configuration.md#allow_loading_unsigned_plugins) ### Update react-hook-form from v6 to v7 @@ -377,7 +392,7 @@ const themeColor = getColorForTheme(color, theme); ``` -## From version 6.x.x to 7.x.x +## From version 6.x.x to 7.0.0 ### What's new in Grafana 7.0? @@ -387,7 +402,7 @@ Plugins built using Angular still work for the foreseeable future, but we encour #### New data format -Along with the move to React, the new plugin platform introduced a new internal data format called [data frames]({{< relref "data-frames.md" >}}). +Along with the move to React, the new plugin platform introduced a new internal data format called [data frames](data-frames.md). Previously, data source plugins could send data either as time series or tables. With data frames, data sources can send any data in a table-like structure. This gives you more flexibility to visualize your data in Grafana. @@ -411,9 +426,9 @@ For plugins prior to Grafana 7.0, all options are considered _Display options_. While backend plugins were available as an experimental feature in previous versions of Grafana, the support has been greatly improved for Grafana 7. Backend plugins for Grafana 7.0 are backwards-compatible and will continue to work. However, the old backend plugin system has been deprecated, and we recommend that you use the new SDK for backend plugins. -Since Grafana 7.0 introduced [signing of backend plugins]({{< relref "../../plugins/plugin-signatures.md" >}}), community plugins won’t load by default if they’re unsigned. +Since Grafana 7.0 introduced [signing of backend plugins](../../plugins/plugin-signatures.md), community plugins won’t load by default if they’re unsigned. -To learn more, refer to [Backend plugins]({{< relref "backend" >}}). +To learn more, refer to [Backend plugins](backend/_index.md). ### Migrate a plugin from Angular to React @@ -501,8 +516,8 @@ async query(options: DataQueryRequest): Promise { } ``` -For more information, refer to [Data frames]({{< relref "data-frames.md">}}). +For more information, refer to [Data frames](data-frames.md). ### Troubleshoot plugin migration -As of Grafana 7.0, backend plugins can now be cryptographically signed to verify their origin. By default, Grafana ignores unsigned plugins. For more information, refer to [Allow unsigned plugins]({{< relref "../../plugins/plugin-signatures.md#allow-unsigned-plugins" >}}). +As of Grafana 7.0, backend plugins can now be cryptographically signed to verify their origin. By default, Grafana ignores unsigned plugins. For more information, refer to [Allow unsigned plugins](../../plugins/plugin-signatures.md#allow-unsigned-plugins). diff --git a/docs/sources/developers/plugins/sign-a-plugin.md b/docs/sources/developers/plugins/sign-a-plugin.md index 5e0623f0245..9138ae8644e 100644 --- a/docs/sources/developers/plugins/sign-a-plugin.md +++ b/docs/sources/developers/plugins/sign-a-plugin.md @@ -41,6 +41,8 @@ Public plugins need to be reviewed by the Grafana team before you can sign them. npx @grafana/toolkit plugin:sign ``` +> **Note:** If running NPM 7+ the `npx` commands mentioned in this article may hang. The workaround is to use `npx --legacy-peer-deps `. + ## Sign a private plugin 1. In your plugin directory, sign the plugin with the API key you just created. Grafana Toolkit creates a [MANIFEST.txt](#plugin-manifest) file in the `dist` directory of your plugin. diff --git a/docs/sources/enterprise/access-control/permissions.md b/docs/sources/enterprise/access-control/permissions.md index b5bb0df11ea..6bca31e0889 100644 --- a/docs/sources/enterprise/access-control/permissions.md +++ b/docs/sources/enterprise/access-control/permissions.md @@ -91,7 +91,6 @@ The following list contains fine-grained access control actions. | `licensing:update` | n/a | Update the license token. | | `licensing:delete` | n/a | Delete the license token. | | `licensing.reports:read` | n/a | Get custom permission reports. | -| `serviceaccounts:delete` | `serviceaccounts:*`
`serviceaccounts:id:*` | Delete one or more service accounts. | ## Scope definitions @@ -109,4 +108,3 @@ The following list contains fine-grained access control scopes. | `settings:*` | Restrict an action to a subset of settings. For example, `settings:*` matches all settings, `settings:auth.saml:*` matches all SAML settings, and `settings:auth.saml:enabled` matches the enable property on the SAML settings. | | `provisioners:*` | Restrict an action to a set of provisioners. For example, `provisioners:*` matches any provisioner, and `provisioners:accesscontrol` matches the fine-grained access control [provisioner]({{< relref "./provisioning.md" >}}). | | `datasources:*`
`datasources:id:*`
`datasources:uid:*`
`datasources:name:*` | Restrict an action to a set of data sources. For example, `datasources:*` matches any data source, and `datasources:name:postgres` matches the data source named `postgres`. | -| `serviceaccounts:*`
`serviceaccounts:id:*` | Restrict an action to a set of service accounts. For example, `serviceaccounts:*` matches any service account and `serviceaccounts:id:1` matches the service account whose ID is `1`. | diff --git a/docs/sources/enterprise/access-control/provisioning.md b/docs/sources/enterprise/access-control/provisioning.md index 6601812e545..31f89051184 100644 --- a/docs/sources/enterprise/access-control/provisioning.md +++ b/docs/sources/enterprise/access-control/provisioning.md @@ -122,6 +122,65 @@ roles: - name: 'Admin' ``` +### Assign your custom role to specific teams + +To assign roles to teams, add said teams to the `teams` section of your roles. To remove a specific assignment, remove it from the list. + +> **Note:** Assignments are updated if the version of the role is greater or equal to the one stored internally.
+> You don’t need to increment the version number of the role to update its assignments.
+> Assignments to built-in roles will be ignored. Use `addDefaultAssignments` and `removeDefaultAssignments` instead. + +In order for provisioning to succeed, specified teams must already exist. Additionally, since teams are local to an organization, the organization has to be specified in the assignment. + +For example, the following role is assigned to the `user editors` team and `user admins` team: + +```yaml +# config file version +apiVersion: 1 + +# Roles to insert/update in the database +roles: + - name: custom:users:writer + description: 'List/update other users in the organization' + version: 1 + global: true + permissions: + - action: 'org.users:read' + scope: 'users:*' + - action: 'org.users:write' + scope: 'users:*' + teams: + - name: 'user editors' + orgId: 1 + - name: 'user admins' + orgId: 1 +``` + +### Assign fixed roles to specific teams + +To assign a fixed role to teams, add said teams to the `teams` section of the associated entry. To remove a specific assignment, remove it from the list. + +> **Note:** Since fixed roles are global, the Global attribute has to be specified. A fixed role will never be updated through provisioning. + +In order for provisioning to succeed, specified teams must already exist. Additionally, since teams are local to an organization, the organization has to be specified in the assignment. + +For example, the following fixed role is assigned to the `user editors` team and `user admins` team: + +```yaml +# config file version +apiVersion: 1 + +# Roles to insert/update in the database +roles: + - name: fixed:users:writer + global: true + teams: + - name: 'user editors' + orgId: 1 + - name: 'user admins' + orgId: 1 +``` + ## Manage default built-in role assignments During startup, Grafana creates [default built-in role assignments]({{< relref "./roles#default-built-in-role-assignments" >}}) with [fixed roles]({{< relref "./roles#fixed-roles" >}}). You can remove and later restore those assignments with provisioning. @@ -237,6 +296,12 @@ roles: - name: 'Editor' # overwrite org id and assign role globally global: true + - name: fixed:users:writer + global: true + # list of teams the role should be assigned to + teams: + - name: 'user editors' + orgId: 1 ``` ## Supported settings diff --git a/docs/sources/enterprise/enterprise-configuration.md b/docs/sources/enterprise/enterprise-configuration.md index 459a21e527e..ebee7b16687 100644 --- a/docs/sources/enterprise/enterprise-configuration.md +++ b/docs/sources/enterprise/enterprise-configuration.md @@ -271,7 +271,7 @@ List of comma- or space-separated organizations. Each user must be a member of a ### org_mapping -List of comma- or space-separated Organization:OrgId mappings. +List of comma- or space-separated Organization:OrgId:Role mappings. Organization can be `*` meaning "All users". Role is optional and can have the following values: `Viewer`, `Editor` or `Admin`. ### role_values_editor diff --git a/docs/sources/enterprise/kms-integration/using-aws-kms-to-encrypt-database-secrets.md b/docs/sources/enterprise/kms-integration/using-aws-kms-to-encrypt-database-secrets.md index 0b49a16a217..b91655c753c 100644 --- a/docs/sources/enterprise/kms-integration/using-aws-kms-to-encrypt-database-secrets.md +++ b/docs/sources/enterprise/kms-integration/using-aws-kms-to-encrypt-database-secrets.md @@ -39,7 +39,6 @@ You can use an encryption key from AWS Key Management Service to encrypt secrets - `access_key_id`: The AWS Access Key ID that you previously generated. - `secret_access_key`: The AWS Secret Access Key you previously generated. - - `token`: (Optional) An AWS Session Token, which you must provide if you created temporary credentials. - `region`: The AWS region where you created the KMS key. The region is contained in the key’s ARN. For example: `arn:aws:kms:*us-east-2*:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab` An example of an AWS KMS provider section in the `grafana.ini` file is as follows: @@ -53,8 +52,6 @@ You can use an encryption key from AWS Key Management Service to encrypt secrets ;access_key_id = AKIAIOSFODNN7EXAMPLE # AWS secret access key ;secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY - # AWS session token, optional - ;token = AQoDYXdzEJr... # AWS region, for example eu-north-1 ;region = eu-north-1 ``` diff --git a/docs/sources/enterprise/query-caching.md b/docs/sources/enterprise/query-caching.md index bab8740d919..8014e173e6a 100644 --- a/docs/sources/enterprise/query-caching.md +++ b/docs/sources/enterprise/query-caching.md @@ -17,9 +17,9 @@ The following cache backends are available: in-memory, Redis, and Memcached. When a panel queries a cached data source, the time until this query fetches fresh data is determined by the panel's **interval.** This means that wider panels and dashboards with shorter time ranges fetch new data more frequently than narrower panels and dashboards with longer time ranges. -Interval is visible in a panel's [query options]({{< relref "../panels/queries.md#query-options" >}}). It is calculated like this: `(max data points) / time range`. Max data points are calculated based on the width of the panel. For example, a full-width panel on a dashboard with a time range of `last 7 days` will retrieve fresh data every 10 minutes. In this example, cached data for this panel will be served for up to 10 minutes before Grafana queries the data source again and returns new data. +Interval is visible in a panel's [query options]({{< relref "../panels/reference-query-options.md" >}}). It is calculated like this: `(max data points) / time range`. Max data points are calculated based on the width of the panel. For example, a full-width panel on a dashboard with a time range of `last 7 days` will retrieve fresh data every 10 minutes. In this example, cached data for this panel will be served for up to 10 minutes before Grafana queries the data source again and returns new data. -You can make a panel retrieve fresh data more frequently by increasing the **Max data points** setting in the panel's [query options]({{< relref "../panels/queries.md#query-options" >}}). +You can make a panel retrieve fresh data more frequently by increasing the **Max data points** setting in the panel's [query options]({{< relref "../panels/reference-query-options.md" >}}). ## Query caching benefits diff --git a/docs/sources/enterprise/reporting.md b/docs/sources/enterprise/reporting.md index 4ba8e2134f2..d38d3c8b492 100644 --- a/docs/sources/enterprise/reporting.md +++ b/docs/sources/enterprise/reporting.md @@ -61,7 +61,7 @@ You can configure report-specific template variables for the dashboard on the re > **Note:** Available in Grafana Enterprise v8+. -You can include dynamic dashboards with panels or rows, set to repeat by a variable, into reports. For detailed information about setting up repeating panels or rows in dashboards, refer to the [Repeat panels or rows]({{< relref "../panels/repeat-panels-or-rows.md" >}}) section. +You can include dynamic dashboards with panels or rows, set to repeat by a variable, into reports. For detailed information about setting up repeating panels or rows in dashboards, refer to the [Repeat panels or rows]({{< relref "../panels/add-panels-dynamically/" >}}) section. #### Caveats: diff --git a/docs/sources/enterprise/saml.md b/docs/sources/enterprise/saml.md index d2a7ad9f52a..3c94203521e 100644 --- a/docs/sources/enterprise/saml.md +++ b/docs/sources/enterprise/saml.md @@ -40,29 +40,29 @@ In terms of initiation: The table below describes all SAML configuration options. Continue reading below for details on specific options. Like any other Grafana configuration, you can apply these options as [environment variables]({{< relref "../administration/configuration.md#configure-with-environment-variables" >}}). -| Setting | Required | Description | Default | -| ---------------------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------- | ------------- | -| `enabled` | No | Whether SAML authentication is allowed | `false` | -| `single_logout` | No | Whether SAML Single Logout enabled | `false` | -| `allow_idp_initiated` | No | Whether SAML IdP-initiated login is allowed | `false` | -| `certificate` or `certificate_path` | Yes | Base64-encoded string or Path for the SP X.509 certificate | | -| `private_key` or `private_key_path` | Yes | Base64-encoded string or Path for the SP private key | | -| `signature_algorithm` | No | Signature algorithm used for signing requests to the IdP. Supported values are rsa-sha1, rsa-sha256, rsa-sha512. | | -| `idp_metadata`, `idp_metadata_path`, or `idp_metadata_url` | Yes | Base64-encoded string, Path or URL for the IdP SAML metadata XML | | -| `max_issue_delay` | No | Duration, since the IdP issued a response and the SP is allowed to process it | `90s` | -| `metadata_valid_duration` | No | Duration, for how long the SP metadata is valid | `48h` | -| `relay_state` | No | Relay state for IdP-initiated login. Should match relay state configured in IdP | | -| `assertion_attribute_name` | No | Friendly name or name of the attribute within the SAML assertion to use as the user name | `displayName` | -| `assertion_attribute_login` | No | Friendly name or name of the attribute within the SAML assertion to use as the user login handle | `mail` | -| `assertion_attribute_email` | No | Friendly name or name of the attribute within the SAML assertion to use as the user email | `mail` | -| `assertion_attribute_groups` | No | Friendly name or name of the attribute within the SAML assertion to use as the user groups | | -| `assertion_attribute_role` | No | Friendly name or name of the attribute within the SAML assertion to use as the user roles | | -| `assertion_attribute_org` | No | Friendly name or name of the attribute within the SAML assertion to use as the user organization | | -| `allowed_organizations` | No | List of comma- or space-separated organizations. User should be a member of at least one organization to log in. | | -| `org_mapping` | No | List of comma- or space-separated Organization:OrgId mappings | | -| `role_values_editor` | No | List of comma- or space-separated roles which will be mapped into the Editor role | | -| `role_values_admin` | No | List of comma- or space-separated roles which will be mapped into the Admin role | | -| `role_values_grafana_admin` | No | List of comma- or space-separated roles which will be mapped into the Grafana Admin (Super Admin) role | | +| Setting | Required | Description | Default | +| ---------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| `enabled` | No | Whether SAML authentication is allowed | `false` | +| `single_logout` | No | Whether SAML Single Logout enabled | `false` | +| `allow_idp_initiated` | No | Whether SAML IdP-initiated login is allowed | `false` | +| `certificate` or `certificate_path` | Yes | Base64-encoded string or Path for the SP X.509 certificate | | +| `private_key` or `private_key_path` | Yes | Base64-encoded string or Path for the SP private key | | +| `signature_algorithm` | No | Signature algorithm used for signing requests to the IdP. Supported values are rsa-sha1, rsa-sha256, rsa-sha512. | | +| `idp_metadata`, `idp_metadata_path`, or `idp_metadata_url` | Yes | Base64-encoded string, Path or URL for the IdP SAML metadata XML | | +| `max_issue_delay` | No | Duration, since the IdP issued a response and the SP is allowed to process it | `90s` | +| `metadata_valid_duration` | No | Duration, for how long the SP metadata is valid | `48h` | +| `relay_state` | No | Relay state for IdP-initiated login. Should match relay state configured in IdP | | +| `assertion_attribute_name` | No | Friendly name or name of the attribute within the SAML assertion to use as the user name | `displayName` | +| `assertion_attribute_login` | No | Friendly name or name of the attribute within the SAML assertion to use as the user login handle | `mail` | +| `assertion_attribute_email` | No | Friendly name or name of the attribute within the SAML assertion to use as the user email | `mail` | +| `assertion_attribute_groups` | No | Friendly name or name of the attribute within the SAML assertion to use as the user groups | | +| `assertion_attribute_role` | No | Friendly name or name of the attribute within the SAML assertion to use as the user roles | | +| `assertion_attribute_org` | No | Friendly name or name of the attribute within the SAML assertion to use as the user organization | | +| `allowed_organizations` | No | List of comma- or space-separated organizations. User should be a member of at least one organization to log in. | | +| `org_mapping` | No | List of comma- or space-separated Organization:OrgId:Role mappings. Organization can be `*` meaning "All users". Role is optional and can have the following values: `Viewer`, `Editor` or `Admin`. | | +| `role_values_editor` | No | List of comma- or space-separated roles which will be mapped into the Editor role | | +| `role_values_admin` | No | List of comma- or space-separated roles which will be mapped into the Admin role | | +| `role_values_grafana_admin` | No | List of comma- or space-separated roles which will be mapped into the Grafana Admin (Super Admin) role | | ### Enable SAML authentication @@ -181,15 +181,15 @@ role_values_grafana_admin = superadmin Organization mapping allows you to assign users to particular organization in Grafana depending on attribute value obtained from identity provider. -1. In configuration file, set [`assertion_attribute_org`]({{< relref "./enterprise-configuration.md#assertion-attribute-org" >}}) to the attribute name you store organization info in. -1. Set [`org_mapping`]({{< relref "./enterprise-configuration.md#org-mapping" >}}) option to the comma-separated list of `Organization:OrgId` pairs to map organization from IdP to Grafana organization specified by id. +1. In configuration file, set [`assertion_attribute_org`]({{< relref "./enterprise-configuration.md#assertion-attribute-org" >}}) to the attribute name you store organization info in. This attribute can be an array if you want a user to be in multiple organizations. +1. Set [`org_mapping`]({{< relref "./enterprise-configuration.md#org-mapping" >}}) option to the comma-separated list of `Organization:OrgId` pairs to map organization from IdP to Grafana organization specified by id. If you want users to have different roles in multiple organizations, you can set this option to a comma-separated list of `Organization:OrgId:Role` mappings. -For example, use following configuration to assign users from `Engineering` organization to the Grafana organization with id `2` and users from `Sales` - to the org with id `3`, based on `Org` assertion attribute value: +For example, use following configuration to assign users from `Engineering` organization to the Grafana organization with id `2` as Editor and users from `Sales` - to the org with id `3` as Admin, based on `Org` assertion attribute value: ```bash [auth.saml] assertion_attribute_org = Org -org_mapping = Engineering:2, Sales:3 +org_mapping = Engineering:2:Editor, Sales:3:Admin ``` You can specify multiple organizations both for the IdP and Grafana: @@ -197,6 +197,10 @@ You can specify multiple organizations both for the IdP and Grafana: - `org_mapping = Engineering:2, Sales:2` to map users from `Engineering` and `Sales` to `2` in Grafana. - `org_mapping = Engineering:2, Engineering:3` to assign `Engineering` to both `2` and `3` in Grafana. +You can use `*` as an Organization if you want all your users to be in some organizations with a default role: + +- `org_mapping = *:2:Editor` to map all users to `2` in Grafana as Editors. + ### Configure allowed organizations > Only available in Grafana v7.0+ @@ -223,7 +227,7 @@ assertion_attribute_org = Org role_values_editor = editor, developer role_values_admin = admin, operator role_values_grafana_admin = superadmin -org_mapping = Engineering:2, Sales:3 +org_mapping = Engineering:2:Editor, Engineering:3:Viewer, Sales:3:Editor, *:1:Editor allowed_organizations = Engineering, Sales ``` diff --git a/docs/sources/explore/query-management.md b/docs/sources/explore/query-management.md index f907c6cd525..f1a71b17581 100644 --- a/docs/sources/explore/query-management.md +++ b/docs/sources/explore/query-management.md @@ -7,8 +7,8 @@ weight = 10 # Query management in Explore To help with debugging queries, Explore allows you to investigate query requests and responses, as well as query statistics, via the Query inspector. -This functionality is similar to the panel inspector [Stats tab]({{< relref "../panels/inspect-panel.md#inspect-query-performance" >}}) and -[Query tab]({{< relref "../panels/inspect-panel.md##view-raw-request-and-response-to-data-source" >}}). +This functionality is similar to the panel inspector tasks [Inspect query performance]({{< relref "../panels/query-a-data-source/inspect-query-performance.md" >}}) and +[Inspect query request and response data]({{< relref "../panels/query-a-data-source/inspect-request-and-response-data.md" >}}). {{< figure src="/static/img/docs/v71/query_inspector_explore.png" class="docs-image--no-shadow" max-width= "550px" caption="Screenshot of the query inspector button in Explore" >}} diff --git a/docs/sources/getting-started/getting-started.md b/docs/sources/getting-started/getting-started.md index 2d88761f29c..74d46e89c2d 100644 --- a/docs/sources/getting-started/getting-started.md +++ b/docs/sources/getting-started/getting-started.md @@ -34,7 +34,7 @@ To create your first dashboard: 1. Click the **+** icon on the side menu. 1. On the dashboard, click **Add an empty panel**. 1. In the New dashboard/Edit panel view, go to the **Query** tab. -1. Configure your [query]({{< relref "../panels/queries.md" >}}) by selecting `-- Grafana --` from the [data source selector]({{< relref "../panels/queries.md/#data-source-selector" >}}). This generates the Random Walk dashboard. +1. Configure your [query]({{< relref "../panels/query-a-data-source/add-a-query" >}}) by selecting `-- Grafana --` from the data source selector. This generates the Random Walk dashboard. 1. Click the **Save** icon in the top right corner of your screen to save the dashboard. 1. Add a descriptive name, and then click **Save**. diff --git a/docs/sources/http_api/_index.md b/docs/sources/http_api/_index.md index 67b048d7bd8..6efe8f5126b 100644 --- a/docs/sources/http_api/_index.md +++ b/docs/sources/http_api/_index.md @@ -13,25 +13,27 @@ dashboards, creating users, and updating data sources. ## HTTP APIs +- [Admin API]({{< relref "admin.md" >}}) +- [Alerting Notification Channels API]({{< relref "alerting_notification_channels.md" >}}) +- [Alerting API]({{< relref "alerting.md" >}}) +- [Annotations API]({{< relref "annotations.md" >}}) - [Authentication API]({{< relref "auth.md" >}}) - [Dashboard API]({{< relref "dashboard.md" >}}) -- [Dashboard versions API]({{< relref "dashboard_versions.md" >}}) -- [Dashboard permissions API]({{< relref "dashboard_permissions.md" >}}) -- [Folder API]({{< relref "folder.md" >}}) -- [Folder permissions API]({{< relref "folder_permissions.md" >}}) -- [Folder/dashboard search API]({{< relref "folder_dashboard_search.md" >}}) +- [Dashboard Permissions API]({{< relref "dashboard_permissions.md" >}}) +- [Dashboard Versions API]({{< relref "dashboard_versions.md" >}}) - [Data source API]({{< relref "data_source.md" >}}) +- [Folder API]({{< relref "folder.md" >}}) +- [Folder Permissions API]({{< relref "folder_permissions.md" >}}) +- [Folder/Dashboard Search API]({{< relref "folder_dashboard_search.md" >}}) +- [Library Element API]({{< relref "library_element.md" >}}) - [Organization API]({{< relref "org.md" >}}) -- [Snapshot API]({{< relref "snapshot.md" >}}) -- [Annotations API]({{< relref "annotations.md" >}}) -- [Playlists API]({{< relref "playlist.md" >}}) -- [Alerting API]({{< relref "alerting.md" >}}) -- [Alert notification channels API]({{< relref "alerting_notification_channels.md" >}}) -- [User API]({{< relref "user.md" >}}) -- [Team API]({{< relref "team.md" >}}) -- [Admin API]({{< relref "admin.md" >}}) -- [Preferences API]({{< relref "preferences.md" >}}) - [Other API]({{< relref "other.md" >}}) +- [Playlists API]({{< relref "playlist.md" >}}) +- [Preferences API]({{< relref "preferences.md" >}}) +- [Short URL API]({{< relref "short_url.md" >}}) +- [Snapshot API]({{< relref "snapshot.md" >}}) +- [Team API]({{< relref "team.md" >}}) +- [User API]({{< relref "user.md" >}}) ## Grafana Enterprise HTTP APIs diff --git a/docs/sources/http_api/admin.md b/docs/sources/http_api/admin.md index 5789d0daaac..fe57a085f25 100644 --- a/docs/sources/http_api/admin.md +++ b/docs/sources/http_api/admin.md @@ -282,6 +282,46 @@ Content-Type: application/json } ``` +## Grafana Usage Report preview + +`GET /api/admin/usage-report-preview` + +Preview usage report to be sent to vendor. + +Only works with Basic Authentication (username and password). See [introduction](http://docs.grafana.org/http_api/admin/#admin-api) for an explanation. + +**Example Request**: + +```http +GET /api/admin/usage-report-preview +Accept: application/json +Content-Type: application/json +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "version": "8_4_0", + "metrics": { + "stats.active_admins.count": 1, + "stats.active_editors.count": 1, + "stats.active_sessions.count": 0, + "stats.active_users.count": 2, + "stats.active_viewers.count": 0, + "stats.admins.count": 1, + "stats.alert_rules.count": 0, + "stats.alerting.ds.other.count": 0, + "stats.alerts.count": 5, + "stats.annotations.count": 6, + "stats.api_keys.count": 1 + } +} +``` + ## Global Users `POST /api/admin/users` diff --git a/docs/sources/http_api/query_history.md b/docs/sources/http_api/query_history.md new file mode 100644 index 00000000000..44a7f21d2cb --- /dev/null +++ b/docs/sources/http_api/query_history.md @@ -0,0 +1,59 @@ ++++ +title = "Query History HTTP API " +description = "Grafana Query History HTTP API" +keywords = ["grafana", "http", "documentation", "api", "queryHistory"] +aliases = ["/docs/grafana/latest/http_api/query_history/"] ++++ + +# Query history API + +This API can be used to add queries to Query history. It requires that the user is logged in and that Query history feature is enabled in config file. + +## Add query to Query history + +`POST /api/query-history` + +Adds query to query history. + +**Example request:** + +```http +POST /api/query-history HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +{ + "dataSourceUid": "PE1C5CBDA0504A6A3", + "queries": [ + { + "refId": "A", + "key": "Q-87fed8e3-62ba-4eb2-8d2a-4129979bb4de-0", + "scenarioId": "csv_content", + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + } + } +] +} +``` + +JSON body schema: + +- **datasourceUid** – Data source uid. +- **queries** – JSON of query or queries. + +**Example response:** + +```http +HTTP/1.1 200 +Content-Type: application/json +{ + "message": "Query successfully added to query history", +} +``` + +Status codes: + +- **200** – OK +- **500** – Errors (invalid JSON, missing or invalid fields) diff --git a/docs/sources/http_api/serviceaccount.md b/docs/sources/http_api/serviceaccount.md deleted file mode 100644 index eec9127b4a2..00000000000 --- a/docs/sources/http_api/serviceaccount.md +++ /dev/null @@ -1,44 +0,0 @@ -+++ -title = "Service account HTTP API" -description = "Grafana Service account HTTP API" -keywords = ["grafana", "http", "documentation", "api", "service account"] -aliases = ["/docs/grafana/latest/http_api/serviceaccount/"] -+++ - -# Service account API - -This API allows you to interact programmatically with the [Service accounts]({{< relref "../manage-users/serviceaccount/_index.md" >}}). - -**> Note:** If you are using Grafana Enterprise and have [Fine-grained access control]({{< relref "../enterprise/access-control/_index.md" >}}) enabled, for some endpoints you need to have relevant permissions. Refer to specific resources to understand what permissions are required. - -## Delete a service account - -`DELETE /api/serviceaccounts/:serviceaccountId` - -#### Required permissions - -For details, see the [introduction]({{< ref "#user-api" >}}). - -| Action | Scope | -| ---------------------- | ------------------ | -| serviceaccounts:delete | serviceaccounts:\* | - -Deletes the given service account if it exists. - -**Example request**: - -```http -DELETE /api/serviceaccounts/1 HTTP/1.1 -Accept: application/json -Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk -``` - -**Example response**: - -```http -HTTP/1.1 200 -Content-Type: application/json - -{"message":"Service account deleted"} -``` diff --git a/docs/sources/image-rendering/troubleshooting.md b/docs/sources/image-rendering/troubleshooting.md index 20cdae5fdcd..0b789834ed6 100644 --- a/docs/sources/image-rendering/troubleshooting.md +++ b/docs/sources/image-rendering/troubleshooting.md @@ -55,34 +55,40 @@ ldd chrome-linux/chrome **Ubuntu:** -On Ubuntu 18.10 the following dependencies have been confirmed as needed for the image rendering to function. +On Ubuntu 18.10 the following dependencies are required for the image rendering to function. ```bash -libx11-6 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrender1 libxtst6 libglib2.0-0 libnss3 libcups2 libdbus-1-3 libxss1 libxrandr2 libgtk-3-0 libgtk-3-0 libasound2 libxcb-dri3-0 libgbm1 libxshmfence1 +libx11-6 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrender1 libxtst6 libglib2.0-0 libnss3 libcups2 libdbus-1-3 libxss1 libxrandr2 libgtk-3-0 libasound2 libxcb-dri3-0 libgbm1 libxshmfence1 ``` **Debian:** -On Debian 9 (Stretch) the following dependencies have been confirmed as needed for the image rendering to function. +On Debian 9 (Stretch) the following dependencies are required for the image rendering to function. ```bash -libx11 libcairo bcairo2 libcairo2 libxtst6 libxcomposite1 libx11-xcb1 libxcursor1 libxdamage1 libnss3 libcups libcups2 libXss libXss1 libxss1 libxrandr2 libasound2 libatk1.0-0 libatk-bridge2.0-0 libpangocairo-1.0-0 libgtk-3-0 libgbm1 libxshmfence1 +libx11 libcairo libcairo2 libxtst6 libxcomposite1 libx11-xcb1 libxcursor1 libxdamage1 libnss3 libcups libcups2 libxss libxss1 libxrandr2 libasound2 libatk1.0-0 libatk-bridge2.0-0 libpangocairo-1.0-0 libgtk-3-0 libgbm1 libxshmfence1 ``` -On Debian 10 (Buster) the following dependencies have been confirmed as needed for the image rendering to function. +On Debian 10 (Buster) the following dependencies are required for the image rendering to function. ```bash -libxdamage1 libxext6 libxi6 libxtst6 libnss3 libnss3 libcups2 libxss1 libxrandr2 libasound2 libatk1.0-0 libatk-bridge2.0-0 libpangocairo-1.0-0 libpango-1.0-0 libcairo2 libatspi2.0-0 libgtk3.0-cil libgdk3.0-cil libx11-xcb-dev libgbm1 libxshmfence1 +libxdamage1 libxext6 libxi6 libxtst6 libnss3 libcups2 libxss1 libxrandr2 libasound2 libatk1.0-0 libatk-bridge2.0-0 libpangocairo-1.0-0 libpango-1.0-0 libcairo2 libatspi2.0-0 libgtk3.0-cil libgdk3.0-cil libx11-xcb-dev libgbm1 libxshmfence1 ``` **Centos:** -On a minimal Centos installation, the following dependencies have been confirmed as needed for the image rendering to function: +On a minimal CentOS 7 installation, the following dependencies are required for the image rendering to function: ```bash libXcomposite libXdamage libXtst cups libXScrnSaver pango atk adwaita-cursor-theme adwaita-icon-theme at at-spi2-atk at-spi2-core cairo-gobject colord-libs dconf desktop-file-utils ed emacs-filesystem gdk-pixbuf2 glib-networking gnutls gsettings-desktop-schemas gtk-update-icon-cache gtk3 hicolor-icon-theme jasper-libs json-glib libappindicator-gtk3 libdbusmenu libdbusmenu-gtk3 libepoxy liberation-fonts liberation-narrow-fonts liberation-sans-fonts liberation-serif-fonts libgusb libindicator-gtk3 libmodman libproxy libsoup libwayland-cursor libwayland-egl libxkbcommon m4 mailx nettle patch psmisc redhat-lsb-core redhat-lsb-submod-security rest spax time trousers xdg-utils xkeyboard-config alsa-lib ``` +On a minimal CentOS 8 installation, the following dependencies are required for the image rendering to function: + +```bash +libXcomposite libXdamage libXtst cups libXScrnSaver pango atk adwaita-cursor-theme adwaita-icon-theme at at-spi2-atk at-spi2-core cairo-gobject colord-libs dconf desktop-file-utils ed emacs-filesystem gdk-pixbuf2 glib-networking gnutls gsettings-desktop-schemas gtk-update-icon-cache gtk3 hicolor-icon-theme jasper-libs json-glib libappindicator-gtk3 libdbusmenu libdbusmenu-gtk3 libepoxy liberation-fonts liberation-narrow-fonts liberation-sans-fonts liberation-serif-fonts libgusb libindicator-gtk3 libmodman libproxy libsoup libwayland-cursor libwayland-egl libxkbcommon m4 mailx nettle patch psmisc redhat-lsb-core redhat-lsb-submod-security rest spax time trousers xdg-utils xkeyboard-config alsa-lib libX11-xcb +``` + ## Certificate signed by internal certificate authorities In many cases, Grafana runs on internal servers and uses certificates that have not been signed by a CA ([Certificate Authority](https://en.wikipedia.org/wiki/Certificate_authority)) known to Chrome, and therefore cannot be validated. Chrome internally uses NSS ([Network Security Services](https://en.wikipedia.org/wiki/Network_Security_Services)) for cryptographic operations such as the validation of certificates. diff --git a/docs/sources/installation/upgrading.md b/docs/sources/installation/upgrading.md index 0c6f289d8c8..71cec6052d8 100755 --- a/docs/sources/installation/upgrading.md +++ b/docs/sources/installation/upgrading.md @@ -365,7 +365,7 @@ ORDER BY time There are two possible workarounds to resolve this problem: 1. In Grafana v8.0.3, use an alias of the string column selected as `metric`. for example, `hostname as metric`. -2. Use the [Standard field options/Display name]({{< relref "../panels/standard-options.md#display-name" >}}) to format the alias. For the preceding example query, you would use `${__field.labels.hostname}` option. +2. Use the [Standard field options/Display name]({{< relref "../panels/reference-standard-field-definitions.md#display-name" >}}) to format the alias. For the preceding example query, you would use `${__field.labels.hostname}` option. For more information, refer to the our relational databases documentation of [Postgres]({{< relref "../datasources/postgres.md#time-series-queries" >}}), [MySQL]({{< relref "../datasources/mysql.md#time-series-queries" >}}), [Microsoft SQL Server]({{< relref "../datasources/mssql.md#time-series-queries" >}}). diff --git a/docs/sources/live/configure-grafana-live.md b/docs/sources/live/configure-grafana-live.md index c7f88c949dc..04bb3903ffb 100644 --- a/docs/sources/live/configure-grafana-live.md +++ b/docs/sources/live/configure-grafana-live.md @@ -93,7 +93,7 @@ http { location / { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; + proxy_set_header Connection $connection_upgrade; proxy_set_header Host $http_host; proxy_pass http://grafana; } diff --git a/docs/sources/manage-users/serviceaccount/_index.md b/docs/sources/manage-users/serviceaccount/_index.md deleted file mode 100644 index 39ba3e854c0..00000000000 --- a/docs/sources/manage-users/serviceaccount/_index.md +++ /dev/null @@ -1,11 +0,0 @@ -+++ -title = "Service accounts" -weight = 100 -+++ - -# Service accounts - -A service account represents a Grafana service identity, which are users that are intended for programmatic use either internally for a Grafana feature or through an API key, such as timely reporting, automatic provisioning or systems interacting with Grafana. See [service account] for more information - -- [Service account]({{< relref "serviceaccount.md" >}}) - Describes service account in detail. -- [Service account API]({{< relref "../../http_api/serviceaccount.md" >}}) - Manage users or change passwords programmatically. diff --git a/docs/sources/manage-users/serviceaccount/serviceaccount.md b/docs/sources/manage-users/serviceaccount/serviceaccount.md deleted file mode 100644 index f999756976d..00000000000 --- a/docs/sources/manage-users/serviceaccount/serviceaccount.md +++ /dev/null @@ -1,20 +0,0 @@ -+++ -title = "What are service accounts" -weight = 200 -+++ - -What are service accounts? - -A service account is a special kind of user used by an application or compute workload, which are users that are intended for programmatic use either internally for a Grafana feature or through an API key. Applications use service accounts to make authorized API calls, authorized as the service account itself. - -For example, all reporting could be setup and run as a service account from your application, and that account can be given permissions to access the resources it needs. This way the service account is the identity of the service, and the service account's permissions control which resources the service can access. - -A service account is identified by its login name, which is unique to the entire suite of organizations. The name is set upon creation of the service account. - -Differences between a service account and a user account -Service accounts differ from user accounts in a few key ways: - -- Service accounts do not have passwords, and cannot log in via browsers or cookies. -- Service accounts are associated with private/public RSA key-pairs that are used for authentication to Grafana. -- You can let other users or service accounts impersonate a service account. -- Service accounts lives on a organizational level, but it does not restrict them from acting cross organizational, such as adding or editing user information. diff --git a/docs/sources/panels/_index.md b/docs/sources/panels/_index.md index e4409b825a0..a95f01b05be 100644 --- a/docs/sources/panels/_index.md +++ b/docs/sources/panels/_index.md @@ -1,26 +1,16 @@ +++ title = "Panels" -aliases = ["/docs/grafana/latest/features/panels/panels/"] -weight = 70 +description = "Panels" +aliases = ["/docs/grafana/latest/features/panels/panels/", "/docs/sources/panels/"] +weight = 72 +++ -# Panel overview +# About Grafana panels The _panel_ is the basic visualization building block in Grafana. Each panel has a query editor specific to the data source selected in the panel. The query editor allows you to extract the perfect visualization to display on the panel. There are a wide variety of styling and formatting options for each panel. Panels can be dragged and dropped and rearranged on the dashboard. They can also be resized. -## Move panels +Before you begin, ensure that you have configured a data source. For more information about data sources, refer to [Data Sources]({{< relref "../datasources" >}}). -You can drag and drop panels by clicking and holding the panel title, then dragging it to its new location. You can also easily resize panels by clicking the (-) and (+) icons. - -![](/static/img/docs/animated_gifs/drag_drop.gif) - -## Tips and shortcuts - -- Click the graph title and in the dropdown menu quickly duplicate the panel. -- Click the colored icon in the legend to change a series color or the y-axis. -- Click series name in the legend to hide series. -- Ctrl/Shift/Meta + click legend name to hide other series. -- Hover your cursor over a panel and press `e` to open the panel editor. -- Hover your cursor over a panel and press `v` to open the panel in full screen view. +{{< section >}} diff --git a/docs/sources/panels/add-a-panel.md b/docs/sources/panels/add-a-panel.md deleted file mode 100644 index dbf6b0fa19f..00000000000 --- a/docs/sources/panels/add-a-panel.md +++ /dev/null @@ -1,61 +0,0 @@ -+++ -title = "Add a panel" -weight = 100 -+++ - -# Add a panel - -Panels allow you to show your data in visual form. This topic walks you through the most basic steps to build a panel. - -## 1. Add a panel to a dashboard - -1. Navigate to the dashboard you want to add a panel to. -1. Click the **Add panel** icon. - - ![](/static/img/docs/panels/add-panel-icon-7-0.png) - -1. Click **Add an empty panel**. - -Grafana creates an empty time series panel with your default data source selected. - -## 2. Write a query - -Each panel needs at least one query to display a visualization. You write queries in the Query tab of the panel editor. For more information about the Query tab, refer to [Queries]({{< relref "queries.md" >}}). - -1. Choose a data source. In the first line of the Query tab, click the drop-down list to see all available data sources. This list includes all data sources you added. Refer to [Add a data source]({{< relref "../datasources/add-a-data-source.md" >}}) if you need instructions. -1. Write or construct a query in the query language of your data source. Options will vary. Refer to your specific [data source documentation]({{< relref "../datasources/_index.md" >}}) for specific guidelines. - -## 3. Choose a visualization - -![](/static/img/docs/panel-editor/select-visualization-8-0.png) - -When you choose a visualization, Grafana will offer visualization suggestions based on your query. You can select from the available suggestions or you can pick a visualization from the complete list of supported visualizations. When you select a visualization, Grafana will show a preview of your query results with that visualization applied. -For more information about individual visualizations, refer to [Visualizations options]({{< relref "../visualizations/_index.md" >}}). - -## 4. (Optional) Edit panel settings - -While not required, most visualizations need some adjustment before they properly display the information that you need. Options are defined in the linked topics below. - -- [Panel options]({{< relref "./panel-options.md" >}}) -- [Visualization-specific options]({{< relref "../visualizations/_index.md" >}}) -- [Standard options]({{< relref "./standard-options.md" >}}) -- [Thresholds]({{< relref "./thresholds.md" >}}) -- [Value mappings]({{< relref "./value-mappings.md" >}}) -- [Data links]({{< relref "../linking/data-links.md" >}}) -- [Field overrides]({{< relref "./field-overrides.md" >}}) - -## 5. Apply changes and save - -Save the dashboard. Either press Ctrl/Cmd+S or click **Save** in the upper right corner of the screen. - -Your options vary depending on the changes you made and whether or not it is a new dashboard. We recommend you add a note to describe your changes before you click **Save**. Notes are very helpful if you need to revert the dashboard to a previous version. - -## What next? - -Our Grafana Fundamentals tutorial is a great place to start, or you can learn more about Grafana by reading one of the documentation topics linked below: - -- Learn more about [panel editor]({{< relref "panel-editor.md" >}}) options. -- Add more [queries]({{< relref "queries.md" >}}). -- [Transform]({{< relref "transformations/_index.md" >}}) your data. -- Set up an [alert]({{< relref "../alerting/_index.md" >}}). -- Create [templates and variables]({{< relref "../variables/_index.md" >}}). diff --git a/docs/sources/panels/add-panels-dynamically/_index.md b/docs/sources/panels/add-panels-dynamically/_index.md new file mode 100644 index 00000000000..a8dcdd584ba --- /dev/null +++ b/docs/sources/panels/add-panels-dynamically/_index.md @@ -0,0 +1,12 @@ ++++ +title = "Enable template variables to add panels dynamically" +keywords = ["dynamic", "variable", "panel"] +aliases = ["/docs/grafana/latest/panels/repeat-panels-or-rows/", "/docs/sources/panels/add-panels-dynamically/"] +weight = 800 ++++ + +# Enable template variables to add panels dynamically + +Harness the power of Grafana by configuring the system to automatically and dynamically add panels based on criteria that you define. + +{{< section >}} diff --git a/docs/sources/panels/add-panels-dynamically/about-repeating-panels-rows.md b/docs/sources/panels/add-panels-dynamically/about-repeating-panels-rows.md new file mode 100644 index 00000000000..ed6f96cf089 --- /dev/null +++ b/docs/sources/panels/add-panels-dynamically/about-repeating-panels-rows.md @@ -0,0 +1,16 @@ ++++ +title = "About repeating panels or rows" +weight = 10 +aliases = ["/docs/sources/panels/add-panels-dynamically/about-repeating-panels-rows/"] ++++ + +# About repeating panels and rows + +You can configure Grafana to dynamically add panels or rows to a dashboard. A dynamic panel (or row) is a panel that the system creates based on the value of a variable. Variables dynamically change your queries across all panels in a dashboard. + +## Grafana Play examples + +You can see examples in the following dashboards: + +- [Prometheus repeat](https://play.grafana.org/d/000000036/prometheus-repeat) +- [Repeated Rows Dashboard](https://play.grafana.org/d/000000153/repeat-rows). diff --git a/docs/sources/panels/add-panels-dynamically/configure-repeating-panels.md b/docs/sources/panels/add-panels-dynamically/configure-repeating-panels.md new file mode 100644 index 00000000000..a23bced89fb --- /dev/null +++ b/docs/sources/panels/add-panels-dynamically/configure-repeating-panels.md @@ -0,0 +1,28 @@ ++++ +title = "Configure repeating panels" +weight = 20 +aliases = ["/docs/sources/panels/add-panels-dynamically/configure-repeating-panels/"] ++++ + +# Configure repeating panels + +For queries that return multiple values for a variable, you can configure Grafana to dynamically add panels based on those values. + +> **Note:** Repeating panels require variables to have one or more items selected; you cannot repeat a panel zero times to hide it. + +## Before you begin + +- Ensure that the query includes a multi-value variable. + +**To configure repeating panels**: + +1. Edit the panel you want to repeat. + +1. On the display options pane, expand **Panel options > Repeat options**. + +1. Select a `direction`. + + - Choose `horizontal` to arrange panels side-by-side. Grafana adjusts the width of a repeated panel. Currently, you cannot mix other panels on a row with a repeated panel. + - Choose `vertical` to arrange panels in a column. The width of repeated panels is the same as the original, repeated panel. + +1. To propagate changes to all panels, reload the dashboard. diff --git a/docs/sources/panels/add-panels-dynamically/configure-repeating-rows.md b/docs/sources/panels/add-panels-dynamically/configure-repeating-rows.md new file mode 100644 index 00000000000..5eaad350ff6 --- /dev/null +++ b/docs/sources/panels/add-panels-dynamically/configure-repeating-rows.md @@ -0,0 +1,26 @@ ++++ +title = "Configure repeating rows" +weight = 30 +aliases = ["/docs/sources/panels/add-panels-dynamically/configure-repeating-rows/"] ++++ + +# Configure repeating rows + +As seen above with the panels you can also repeat rows if you have variables set with `Multi-value` or +`Include all value` selection option. + +## Before you begin + +- Ensure that the query includes a multi-value variable. + +**To configure repeating rows**: + +1. On the dashboard home page, click **Add panel**. + +1. On the **Add a panel** dialog box, click **Add a new row**. + +1. Hover over the row title and click the cog icon. + +1. On the `Row Options` configuration panel, select the variable for which you want to add repeating rows. + +> To help provide context to dashboard users, add the variable to the row title. diff --git a/docs/sources/panels/field-overrides.md b/docs/sources/panels/field-overrides.md deleted file mode 100644 index 624cf5f174f..00000000000 --- a/docs/sources/panels/field-overrides.md +++ /dev/null @@ -1,110 +0,0 @@ -+++ -title = "Field overrides" -keywords = ["grafana", "field options", "documentation", "format fields", "overrides", "override fields"] -aliases = ["/docs/grafana/latest/panels/field-options/", "/docs/grafana/latest/panels/field-options/configure-specific-fields/"] -weight = 700 -+++ - -# Field overrides - -Overrides allow you to change the settings for one or more fields. Field options for overrides are exactly the same as the field options available in a particular visualization. The only difference is that you choose which fields to apply them to. - -For example, you could change the number of decimal places shown in all numeric fields or columns by changing the **Decimals** option for **Fields with type** that matches **Numeric**. - -## View existing field overrides - -1. Navigate to the panel you want to edit, click the panel title, and then click **Edit**. -1. In the side pane, click **Overrides**. - -Grafana displays a list of field overrides, if any have been created for this panel. - -![View overrides](/static/img/docs/time-series-panel/panel-editor-8-0.png) - -## Add a field override - -You can override as many fields and add as many field options to each override as you want to. - -1. Navigate to the panel you want to edit, click the panel title, and then click **Edit**. -1. In the side pane, click **Overrides**. -1. Click **Add field override**. -1. Select which fields an override rule will be applied to: - - **Fields with name -** Select a field from the list of all available fields. Properties you add to a rule with this selector are only applied to this single field. - - **Fields with name matching regex -** Specify fields to override with a regular expression. Properties you add to a rule with this selector are applied to all fields where the field name match the regex. - - **Fields with type -** Select fields by type, such as string, numeric, and so on. Properties you add to a rule with this selector are applied to all fields that match the selected type. - - **Fields returned by query -** Select all fields returned by a specific query, such as A, B, or C. Properties you add to a rule with this selector are applied to all fields returned by the selected query. -1. Click **Add override property**. -1. Select the field option that you want to apply. -1. Enter options by adding values in the fields. To return options to default values, delete the white text in the fields. -1. Continue to add overrides to this field by clicking **Add override property**, or you can click **Add override** and select a different field to add overrides to. -1. When finished, click **Save** to save all panel edits to the dashboard. - -## Edit a field override - -1. Navigate to the panel you want to edit, click the panel title, and then click **Edit**. -1. In the side pane, click **Overrides**. -1. Locate the override that you want to change. You can: - - Edit settings on existing overrides or field selection parameters. - - Delete existing override properties by clicking the **X** next to the property. - - Add additional override properties by clicking **Add override property**. - -## Delete a field override - -1. Navigate to the panel you want to edit, click the panel title, and then click **Edit**. -1. In the side pane, click **Overrides**. -1. Click the trash can icon next to the override that you want to delete. - -## Example use case for field options and overrides - -The following examples show how you might use field options and overrides. - -## Field option example - -Let’s assume that our result set is a data frame that consists of two fields: time and temperature. - -| time | temperature | -| :-----------------: | :---------: | -| 2020-01-02 03:04:00 | 45.0 | -| 2020-01-02 03:05:00 | 47.0 | -| 2020-01-02 03:06:00 | 48.0 | - -Each field (column) of this structure can have field options applied that alter the way its values are displayed. This means that you can, for example, set the Unit to Temperature > Celsius, resulting in the following table: - -| time | temperature | -| :-----------------: | :---------: | -| 2020-01-02 03:04:00 | 45.0 °C | -| 2020-01-02 03:05:00 | 47.0 °C | -| 2020-01-02 03:06:00 | 48.0 °C | - -While we're at it, the decimal place doesn't add anything to this display. You can change the Decimals from `auto` to zero (`0`), resulting in the following table: - -| time | temperature | -| :-----------------: | :---------: | -| 2020-01-02 03:04:00 | 45 °C | -| 2020-01-02 03:05:00 | 47 °C | -| 2020-01-02 03:06:00 | 48 °C | - -## Field override example - -Let’s assume that our result set is a data frame that consists of four fields: time, high temp, low temp, and humidity. - -| time | high temp | low temp | humidity | -| ------------------- | --------- | -------- | -------- | -| 2020-01-02 03:04:00 | 45.0 | 30.0 | 67 | -| 2020-01-02 03:05:00 | 47.0 | 34.0 | 68 | -| 2020-01-02 03:06:00 | 48.0 | 31.0 | 68 | - -Let's apply the field options from the [field option example](#field-option-example) to apply the Celsius unit and get rid of the decimal place. This results in the following table: - -| time | high temp | low temp | humidity | -| ------------------- | --------- | -------- | -------- | -| 2020-01-02 03:04:00 | 45 °C | 30 °C | 67 °C | -| 2020-01-02 03:05:00 | 47 °C | 34 °C | 68 °C | -| 2020-01-02 03:06:00 | 48 °C | 31 °C | 68 °C | - -The temperature fields look good, but the humidity is nonsensical. We can fix this by applying a field option override to the humidity field and change the unit to Misc > percent (0-100). This results in a table that makes a lot more sense: - -| time | high temp | low temp | humidity | -| ------------------- | --------- | -------- | -------- | -| 2020-01-02 03:04:00 | 45 °C | 30 °C | 67% | -| 2020-01-02 03:05:00 | 47 °C | 34 °C | 68% | -| 2020-01-02 03:06:00 | 48 °C | 31 °C | 68% | diff --git a/docs/sources/panels/format-data/_index.md b/docs/sources/panels/format-data/_index.md new file mode 100644 index 00000000000..2d46c4501ca --- /dev/null +++ b/docs/sources/panels/format-data/_index.md @@ -0,0 +1,11 @@ ++++ +title = "Format data using value mapping" +aliases = ["/docs/grafana/latest/panels/value-mappings/", "/docs/sources/panels/format-data/"] +weight = 600 ++++ + +# Format data using value mapping + +In addition to field overrides, value mapping is a technique that you can use to change the visual treatment of data that appears in a visualization. + +{{< section >}} diff --git a/docs/sources/panels/format-data/about-value-mapping.md b/docs/sources/panels/format-data/about-value-mapping.md new file mode 100644 index 00000000000..99836f22323 --- /dev/null +++ b/docs/sources/panels/format-data/about-value-mapping.md @@ -0,0 +1,52 @@ ++++ +title = "About value mapping" +weight = 10 +aliases = ["/docs/sources/panels/format-data/about-value-mapping/"] ++++ + +# About value mapping + +Value mapping allows you to replace values or ranges in your visualizations with words or emojis. + +Values mapped via value mappings will skip the unit formatting. This means that a text value mapped to a numerical value will not be formatted using the configured unit. + +![Value mappings example](/static/img/docs/value-mappings/value-mappings-example-8-0.png) + +If value mappings are present in a panel, then Grafana displays a summary in the side pane of the panel editor. + +> **Note:** The new value mappings are not compatible with some visualizations, such as Graph (old), Text, and Heatmap. + +## Types of value mappings + +Grafana supports the following value mappings: + +- **Value** maps text values to a color or different display text. For example, if a value is `10`, I want Grafana to display **Perfection!** rather than the number. +- **Range** maps numerical ranges to a display text and color. For example, if a value is within a certain range, I want Grafana to display **Low** or **High** rather than the number. +- **Regex** maps regular expressions to replacement text and a color. For example, if a value is 'www.example.com', I want Grafana to display just **www**, truncating the domain. +- **Special** maps special values like `Null`, `NaN` (not a number), and boolean values like `true` and `false` to a display text and color. For example, if Grafana encounters a `null`, I want Grafana to display **N/A**. + +You can also use the dots on the left as a "handle" to drag and reorder value mappings in the list. + +## Time series example + +Here's an example showing a Time series visualization with value mappings. Value mapping colors are not applied to this visualization, but the display text is shown on the axis. + +![Value mappings time series example](/static/img/docs/value-mappings/value-mappings-summary-example-8-0.png) + +## Stat example + +Here's an example showing a Stat visualization with value mappings. You might want to hide the sparkline so it doesn't interfere with the values. Value mapping text colors are applied. + +![Value mappings stat example](/static/img/docs/value-mappings/value-mappings-stat-example-8-0.png) + +## Bar gauge example + +Here's an example showing a Bar gauge visualization with value mappings. The value mapping colors are applied to the text but not the gauges. + +![Value mappings bar gauge example](/static/img/docs/value-mappings/value-mappings-bar-gauge-example-8-0.png) + +## Table example + +Here's an example showing a Table visualization with value mappings. If you want value mapping colors displayed on the table, then set the cell display mode to **Color text** or **Color background**. + +![Value mappings table example](/static/img/docs/value-mappings/value-mappings-table-example-8-0.png) diff --git a/docs/sources/panels/format-data/edit-value-mapping.md b/docs/sources/panels/format-data/edit-value-mapping.md new file mode 100644 index 00000000000..9529539ffda --- /dev/null +++ b/docs/sources/panels/format-data/edit-value-mapping.md @@ -0,0 +1,20 @@ ++++ +title = "Edit a value mapping" +weight = 60 +aliases = ["/docs/sources/panels/format-data/edit-value-mapping/"] ++++ + +# Edit a value mapping + +You can change a value mapping at any time. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). +- Ensure you have an existing value mapping to edit. + +**To edit a value mapping**: + +1. Edit the panel that contains the value mapping you want to edit. +1. In the panel display options, in the **Value mappings** section, click **Edit value mappings**. +1. Make the changes and click **Update**. diff --git a/docs/sources/panels/format-data/map-a-range.md b/docs/sources/panels/format-data/map-a-range.md new file mode 100644 index 00000000000..6cdc3022028 --- /dev/null +++ b/docs/sources/panels/format-data/map-a-range.md @@ -0,0 +1,25 @@ ++++ +title = "Map a range" +weight = 30 +aliases = ["/docs/sources/panels/format-data/map-a-range/"] ++++ + +# Map a range + +Map a range of values when you want to format multiple, continuous values. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To map a range**: + +1. Edit the panel for which you want to map a range of values. +1. In panel display options, in the **Value mappings** section, click **Add value mappings**. +1. Click **Add a new mapping** and then select **Range**. +1. Enter the beginning and ending values in the range for Grafana to match. +1. (Optional) Enter display text. +1. (Optional) Set the color. +1. Click **Update** to save the value mapping. + +![Map a range](/static/img/docs/value-mappings/map-range-8-0.png) diff --git a/docs/sources/panels/format-data/map-a-regular-expression.md b/docs/sources/panels/format-data/map-a-regular-expression.md new file mode 100644 index 00000000000..f1d09827154 --- /dev/null +++ b/docs/sources/panels/format-data/map-a-regular-expression.md @@ -0,0 +1,23 @@ ++++ +title = "Map a regular expression" +weight = 40 +aliases = ["/docs/sources/panels/format-data/map-a-regular-expression/"] ++++ + +# Map a regular expression + +Map a regular expression when you want to format the text and color of a regular expression value. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To map a regular expression**: + +1. Edit the panel for which you want to map a regular expression. +1. In the panel display options, in the **Value mappings** section, click **Add value mappings**. +1. Click **Add a new mapping** and then select **Regex**. +1. Enter the regular expression pattern for Grafana to match. +1. (Optional) Enter display text. +1. (Optional) Set the color. +1. Click **Update** to save the value mapping. diff --git a/docs/sources/panels/format-data/map-a-special-value.md b/docs/sources/panels/format-data/map-a-special-value.md new file mode 100644 index 00000000000..b229e6bd5b2 --- /dev/null +++ b/docs/sources/panels/format-data/map-a-special-value.md @@ -0,0 +1,31 @@ ++++ +title = "Map a special value" +weight = 50 +aliases = ["/docs/sources/panels/format-data/map-a-special-value/"] ++++ + +# Map a special value + +Map a special value when you want to format uncommon, boolean, or empty values. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To map a special value**: + +1. Edit the panel for which you want to map a special value. +1. In panel display options, locate the **Value mappings** section and click **Add value mappings**. +1. Click **Add a new mapping** and then select **Special**. +1. Select the special value for Grafana to match. Options include: + - Null + - NaN (Not a Number) + - Null + NaN + - True + - False + - Empty +1. (Optional) Enter display text. +1. (Optional) Set the color. +1. Click **Update** to save the value mapping. + +![Map a value](/static/img/docs/value-mappings/map-special-value-8-0.png) diff --git a/docs/sources/panels/format-data/map-a-value.md b/docs/sources/panels/format-data/map-a-value.md new file mode 100644 index 00000000000..f50e7f4f2b4 --- /dev/null +++ b/docs/sources/panels/format-data/map-a-value.md @@ -0,0 +1,25 @@ ++++ +title = "Map a value" +weight = 20 +aliases = ["/docs/sources/panels/format-data/map-a-value/"] ++++ + +# Map a value + +Map a value when you want to format a single value. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To map a value**: + +1. Open a panel for which you want to map a value. +1. In panel display options, locate the **Value mappings** section and click **Add value mappings**. +1. Click **Add a new mapping** and then select **Value**. +1. Enter the value for Grafana to match. +1. (Optional) Enter display text. +1. (Optional) Set the color. +1. Click **Update** to save the value mapping. + +![Map a value](/static/img/docs/value-mappings/map-value-8-0.png) diff --git a/docs/sources/panels/inspect-panel.md b/docs/sources/panels/inspect-panel.md deleted file mode 100644 index e881711e224..00000000000 --- a/docs/sources/panels/inspect-panel.md +++ /dev/null @@ -1,101 +0,0 @@ -+++ -title = "Inspect a panel" -weight = 400 -+++ - -# Inspect a panel - -The panel inspector helps you understand and troubleshoot your panels. You can inspect the raw data for any Grafana panel, export that data to a comma-separated values (CSV) file, view query requests, and export panel and data JSON. - -## Panel inspector UI - -The panel inspector displays Inspect: at the top of the pane. Click the arrow in the upper right corner to expand or reduce the pane. - -The panel inspector consists of following tabs: - -- **Data tab -** Shows the raw data returned by the query with transformations applied. Field options such as overrides and value mappings are not applied by default. -- **Stats tab -** Shows how long your query takes and how much it returns. -- **JSON tab -** Allows you to view and copy the panel JSON, panel data JSON, and data frame structure JSON. This is useful if you are provisioning or administering Grafana. -- **Query tab -** Shows you the requests to the server sent when Grafana queries the data source. -- **Error tab -** Shows the error. Only visible when query returns error. - -> **Note:** Not all panel types include all four tabs. For example, dashboard list panels do not have raw data to inspect, so they do not display the Stats, Data, or Query tabs. - -## Panel inspector tasks - -Tasks you can perform in the panel inspector are described below. - -### Open the panel inspector - -You can inspect any panel that you can view. - -1. In Grafana, navigate to the dashboard that contains the panel you want to inspect. -1. Click the title of the panel you want to inspect and then click **Inspect**. - Or - Hover your cursor over the panel title and then press **i**. - -The panel inspector pane opens on the right side of the screen. - -### Inspect raw query results - -View raw query results in a table. This is the data returned by the query with transformations applied and before the panel applies field options or field option overrides. - -In the inspector click the **Data** tab or in the panel menu click **Inspect > Data**. - -If your panel contains multiple queries or queries multiple nodes, then you have additional options. - -- **Select result -** Choose which result set data you want to view. -- **Transform data** - - - **Join by time -** View raw data from all your queries at once, one result set per column. Click a column heading to reorder the data. - - View raw query results in a table with field options and options overrides applied: - - 1. Open the **Data** tab in panel inspector. - 1. Click on **Data display options** above the table. - 1. Click on **Apply field configuration** toggle button. - -### Download raw query results as CSV - -Grafana generates a CSV file in your default browser download location. You can open it in the viewer of your choice. - -1. Open the panel inspector. -1. Inspect the raw query results as described above. Adjust settings until you see the raw data that you want to export. -1. Click **Download CSV**. - -To download a CSV file specifically formatted for Excel, expand the **Data options** panel and enable the **Download for Excel** toggle before clicking **Download CSV**. - -### Download log results - -Grafana generates a text (.txt) file in your default browser download location. You can open the log in the viewer of your choice. - -1. Open the panel inspector. -1. Inspect the log query results as described above. -1. Click **Download logs**. - -### Inspect query performance - -The Stats tab displays statistics that tell you how long your query takes, how many queries you send, and the number of rows returned. This information can help you troubleshoot your queries, especially if any of the numbers are unexpectedly high or low. - -1. Open the panel inspector. -1. Navigate to the Stats tab. - -Statistics are displayed in read-only format. - -### View panel JSON model - -Explore and export panel, panel data, and data frame JSON models. - -1. Open the panel inspector and then click the **JSON** tab or in the panel menu click **Inspect > Panel JSON**. -1. In Select source, choose one of the following options: - - **Panel JSON -** Displays a JSON object representing the panel. - - **Panel data -** Displays a JSON object representing the data that was passed to the panel. - - **DataFrame structure -** Displays the raw result set with transformations, field configuration, and overrides configuration applied. -1. You can expand or collapse portions of the JSON to explore it, or you can click **Copy to clipboard** and paste the JSON in another application. - -### View raw request and response to data source - -1. Open the panel inspector and then click the **Query** tab or in the panel menu click **Inspect > Query**. -1. Click **Refresh**. - -Grafana sends a query to the server to collect information and then displays the result. You can now drill down on specific portions of the query, expand or collapse all of it, or copy the data to the clipboard to use in other applications. diff --git a/docs/sources/panels/legend-options.md b/docs/sources/panels/legend-options.md deleted file mode 100644 index 69a42224da6..00000000000 --- a/docs/sources/panels/legend-options.md +++ /dev/null @@ -1,50 +0,0 @@ -+++ -title = "Legend options" -aliases = ["/docs/grafana/latest/panels/visualizations/panel-legend/"] -weight = 950 -+++ - -# Legend options - -Use the legend to adjust how a visualization displays series. This legend functionality only applies to a few panels now, but it will eventually be common to all visualizations. - -This topic currently applies to the following visualizations: - -- [Bar chart panel]({{< relref "../visualizations/bar-chart.md">}}) -- [Histogram panel]({{< relref "../visualizations/histogram.md">}}) -- [Pie chart panel]({{< relref "../visualizations/pie-chart-panel.md">}}) -- [State timeline panel]({{< relref "../visualizations/state-timeline.md">}}) -- [Status history panel]({{< relref "../visualizations/status-history.md">}}) -- [Time series panel]({{< relref "../visualizations/time-series/_index.md" >}}) -- XY chart panel - -## Toggle series - -To toggle a series: -Click on the series label in the legend to isolate the series in the visualization. -All other series are hidden in the visualization. The data of the hidden series is still accessible. - -Use Cmd/Ctrl+click on the series label to hide the isolated series and remove the toggle. - -> **Note:** This option is persistent when you save the dashboard. - -![Toggle series visibility](/static/img/docs/legend/legend-series-toggle-7-5.png) - -This creates a system override that hides the other series. You can view this override in the Overrides tab. If you delete the override, then it removes the toggle. - -![Series toggle override example](/static/img/docs/legend/legend-series-override-7-5.png) - -## Change series color - -Click on the series icon (colored line beside the series label) in the legend to change selected series color. - -![Change legend series color](/static/img/docs/legend/legend-series-color-7-5.png) - -## Sort series - -Change legend mode to **Table** and choose [calculations]({{< relref "./calculations-list.md" >}}) to be displayed in the legend. Click the calculation name header in the legend table to sort the values in the table in ascending or descending order. -The sort order affects the positions of the bars in the Bar chart panel as well as the order of stacked series in the Time series and Bar chart panels. - -> **Note:** This feature is only supported in these panels: Bar chart, Histogram, Time series, XY Chart. - -![Sort legend series](/static/img/docs/legend/legend-series-sort-8-3.png) diff --git a/docs/sources/panels/library-panels/_index.md b/docs/sources/panels/library-panels/_index.md new file mode 100644 index 00000000000..4faa00668ef --- /dev/null +++ b/docs/sources/panels/library-panels/_index.md @@ -0,0 +1,11 @@ ++++ +title = "Create reusable Grafana panels" +aliases = ["/docs/grafana/latest/panels/panel-library/" "/docs/sources/panels/library-panels/"] +weight = 700 ++++ + +# Create reusable Grafana panels + +Library panels enable you to create reusable panels that you can add to any dashboard in your organization. + +{{< section >}} diff --git a/docs/sources/panels/library-panels/about-library-panels.md b/docs/sources/panels/library-panels/about-library-panels.md new file mode 100644 index 00000000000..f9d4c693164 --- /dev/null +++ b/docs/sources/panels/library-panels/about-library-panels.md @@ -0,0 +1,11 @@ ++++ +title = "About Grafana library panels" +weight = 10 +aliases = ["/docs/sources/panels/library-panels/about-library-panels/"] ++++ + +# About Grafana panel libraries + +A library panel is a reusable panel that you can use in any dashboard. When you make a change to a library panel, that change propegates to all instances of where the panel is used. Library panels streamline reuse of panels across multiple dashboards. + +You can save a library panel in a folder alongside saved dashboards. diff --git a/docs/sources/panels/library-panels/add-library-panel.md b/docs/sources/panels/library-panels/add-library-panel.md new file mode 100644 index 00000000000..2522a9d126f --- /dev/null +++ b/docs/sources/panels/library-panels/add-library-panel.md @@ -0,0 +1,27 @@ ++++ +title = "Add a Grafana library panel to a dashboard" +weight = 30 +aliases = ["/docs/sources/panels/library-panels/add-library-panel/"] ++++ + +# Add a Grafana library panel to a dashboard + +Add a Grafana library panel to a dashboard when you want to provide visualizations to other dashboard users. + +## Before you begin + +- [Create a library panel]({{< relref "../library-panels/create-library-panel.md" >}}). + +**To add a library panel to a dashboard**: + +1. Hover over the **+** option on the left menu, then select **Create** from the drop-down options. + + The **Add** panel dialog opens. + {{< figure src="/static/img/docs/library-panels/add-library-panel-8-0.png" class="docs-image--no-shadow" max-width= "900px" caption="Screenshot of the edit panel" >}} + +1. Click the **Add a panel from the panel library** option. + + You will see a list of your library panels. + +1. Filter the list or search to find the panel you want to add. +1. Click a panel to add it to the dashboard. diff --git a/docs/sources/panels/library-panels/create-library-panel.md b/docs/sources/panels/library-panels/create-library-panel.md new file mode 100644 index 00000000000..910f4744827 --- /dev/null +++ b/docs/sources/panels/library-panels/create-library-panel.md @@ -0,0 +1,29 @@ ++++ +title = "Create a Grafana library panel" +weight = 20 +aliases = ["/docs/sources/panels/library-panels/create-library-panel/"] ++++ + +# Create a Grafana library panel + +When you create a library panel, the panel on the source dashboard is converted to a library panel as well. You need to save the original dashboard once a panel is converted. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To create a library panel**: + +1. Open a panel in edit mode. +1. In the panel display options, click the down arrow option to bring changes to the visualization. + {{< figure src="/static/img/docs/library-panels/create-lib-panel-from-edit-8-0.png" class="docs-image--no-shadow" max-width= "800px" caption="Screenshot of the edit panel" >}} +1. Click the **Library panels** option, and then click **Create new library panel** to open the create dialog. + {{< figure src="/static/img/docs/library-panels/create-lib-panel-8-0.png" class="docs-image--no-shadow" max-width= "500px" caption="Screenshot of the create library panel dialog" >}} +1. In **Library panel name**, enter the name. +1. In **Save in folder**, select the folder to save the library panel. +1. Click **Create library panel** to save your changes. +1. Save the dashboard. + +Once created, you can modify the library panel using any dashboard on which it appears. After you save the changes, all instances of the library panel reflect these modifications. + +{{< figure src="/static/img/docs/library-panels/create-from-more-8-0.png" class="docs-image--no-shadow" max-width= "900px" caption="Screenshot of the edit panel" >}} diff --git a/docs/sources/panels/library-panels/delete-library-panel.md b/docs/sources/panels/library-panels/delete-library-panel.md new file mode 100644 index 00000000000..49b9c80a65c --- /dev/null +++ b/docs/sources/panels/library-panels/delete-library-panel.md @@ -0,0 +1,23 @@ ++++ +title = "Delete a Grafana library panel" +weight = 60 +aliases = ["/docs/sources/panels/library-panels/delete-library-panel/"] ++++ + +# Delete a Grafana library panel + +Delete a library panel when you no longer need it. + +## Before you begin + +- Verify that the panel does not appear on a dashboard. + +**To delete a library panel**: + +1. Hover over **Dashboard** on the left menu, and select Library panels from the drop-down options. +1. Select a library panel that is being used in different dashboards. + + You will see a list of all the dashboards. + +1. Select the panel you want to delete. +1. Click the delete icon next to the library panel name. diff --git a/docs/sources/panels/library-panels/manage-library-panel.md b/docs/sources/panels/library-panels/manage-library-panel.md new file mode 100644 index 00000000000..2505ed09451 --- /dev/null +++ b/docs/sources/panels/library-panels/manage-library-panel.md @@ -0,0 +1,24 @@ ++++ +title = "Manage Grafana library panels" +weight = 40 +aliases = ["/docs/sources/panels/library-panels/manage-library-panel/"] ++++ + +# Manage Grafana library panels + +You can adjust library panel configuration at any time. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To view and manage a library panel**: + +1. Hover over the **Dashboard** option on the left menu, then click **Library panels**. + + You can see a list of previously defined library panels. + {{< figure src="/static/img/docs/library-panels/library-panel-list-8-0.png" class="docs-image--no-shadow" max-width= "900px" caption="Screenshot of the edit panel" >}} + +1. Search for a specific library panel if you know its name. + + You can also filter the panels by folder or type. diff --git a/docs/sources/panels/library-panels/unlink-library-panel.md b/docs/sources/panels/library-panels/unlink-library-panel.md new file mode 100644 index 00000000000..5dfbfead199 --- /dev/null +++ b/docs/sources/panels/library-panels/unlink-library-panel.md @@ -0,0 +1,21 @@ ++++ +title = "Unlink a Grafana library panel" +weight = 50 +aliases = ["/docs/sources/panels/library-panels/unlink-library-panel/"] ++++ + +# Unlink a Grafana library panel + +Unlink a library panel when you want to make a change to the panel and not affect other instances of the library panel. + +## Before you begin + +- Identify the panel you want to unlink. + +**To unlink a library panel from a dashboard**: + +1. Hover over **Dashboard** on the left menu, and then click **Library panels**. +1. Select a library panel that is being used in different dashboards. +1. Select the panel you want to unlink. +1. Click the title of the panel and then click **Edit**. The panel opens in edit mode. +1. Click the **Unlink** option on the top right corner of the page. diff --git a/docs/sources/panels/override-field-values/_index.md b/docs/sources/panels/override-field-values/_index.md new file mode 100644 index 00000000000..b475ba52d4c --- /dev/null +++ b/docs/sources/panels/override-field-values/_index.md @@ -0,0 +1,11 @@ ++++ +title = "Override field values" +aliases = ["/docs/grafana/latest/panels/field-overrides/", "/docs/sources/panels/override-field-values/"] +weight = 400 ++++ + +# Override field values + +Apply field overrides when you want to format field values that appear in a visualization. + +{{< section >}} diff --git a/docs/sources/panels/override-field-values/about-field-overrides.md b/docs/sources/panels/override-field-values/about-field-overrides.md new file mode 100644 index 00000000000..0db0105ab0b --- /dev/null +++ b/docs/sources/panels/override-field-values/about-field-overrides.md @@ -0,0 +1,63 @@ ++++ +title = "About field overrides" +weight = 10 +aliases = ["/docs/sources/panels/override-field-values/about-field-overrides/"] ++++ + +# About field overrides + +Overrides allow you to change the settings for one or more fields. Field options for overrides are the same as the field options available in a particular visualization. The only difference is that you choose which fields to apply them to. + +For example, you could change the number of decimal places shown in all numeric fields or columns by changing the **Decimals** option for **Fields with type** that matches **Numeric**. + +## Example 1: Format temperature + +Let’s assume that our result set is a data frame that consists of two fields: time and temperature. + +| time | temperature | +| :-----------------: | :---------: | +| 2020-01-02 03:04:00 | 45.0 | +| 2020-01-02 03:05:00 | 47.0 | +| 2020-01-02 03:06:00 | 48.0 | + +Each field (column) of this structure can have field options applied that alter the way its values are displayed. This means that you can, for example, set the Unit to Temperature > Celsius, resulting in the following table: + +| time | temperature | +| :-----------------: | :---------: | +| 2020-01-02 03:04:00 | 45.0 °C | +| 2020-01-02 03:05:00 | 47.0 °C | +| 2020-01-02 03:06:00 | 48.0 °C | + +In addition, the decimal place is not required, so we can remove it. You can change the Decimals from `auto` to zero (`0`), resulting in the following table: + +| time | temperature | +| :-----------------: | :---------: | +| 2020-01-02 03:04:00 | 45 °C | +| 2020-01-02 03:05:00 | 47 °C | +| 2020-01-02 03:06:00 | 48 °C | + +## Example 2: Format temperature and humidity + +Let’s assume that our result set is a data frame that consists of four fields: time, high temp, low temp, and humidity. + +| time | high temp | low temp | humidity | +| ------------------- | --------- | -------- | -------- | +| 2020-01-02 03:04:00 | 45.0 | 30.0 | 67 | +| 2020-01-02 03:05:00 | 47.0 | 34.0 | 68 | +| 2020-01-02 03:06:00 | 48.0 | 31.0 | 68 | + +Let's add the Celsius unit and get rid of the decimal place. This results in the following table: + +| time | high temp | low temp | humidity | +| ------------------- | --------- | -------- | -------- | +| 2020-01-02 03:04:00 | 45 °C | 30 °C | 67 °C | +| 2020-01-02 03:05:00 | 47 °C | 34 °C | 68 °C | +| 2020-01-02 03:06:00 | 48 °C | 31 °C | 68 °C | + +The temperature fields look good, but the humidity must now be changed. We can fix this by applying a field option override to the humidity field and change the unit to Misc > percent (0-100). + +| time | high temp | low temp | humidity | +| ------------------- | --------- | -------- | -------- | +| 2020-01-02 03:04:00 | 45 °C | 30 °C | 67% | +| 2020-01-02 03:05:00 | 47 °C | 34 °C | 68% | +| 2020-01-02 03:06:00 | 48 °C | 31 °C | 68% | diff --git a/docs/sources/panels/override-field-values/add-a-field-override.md b/docs/sources/panels/override-field-values/add-a-field-override.md new file mode 100644 index 00000000000..94b692b18f8 --- /dev/null +++ b/docs/sources/panels/override-field-values/add-a-field-override.md @@ -0,0 +1,29 @@ ++++ +title = "Add a field override" +weight = 30 +aliases = ["/docs/sources/panels/override-field-values/add-a-field-override/"] ++++ + +# Add a field override + +You can override a field when you want to change the display of the value in the visualization. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To add a field override**: + +1. Edit the panel to which you want to add an override. +1. In the panel display options, in the **Overrides** section, click **Add field override**. + +1. Select which fields an override rule will be applied to: + - **Fields with name -** Select a field from the list of all available fields. Properties you add to a rule with this selector are only applied to this single field. + - **Fields with name matching regex -** Specify fields to override with a regular expression. Properties you add to a rule with this selector are applied to all fields where the field name match the regex. + - **Fields with type -** Select fields by type, such as string, numeric, and so on. Properties you add to a rule with this selector are applied to all fields that match the selected type. + - **Fields returned by query -** Select all fields returned by a specific query, such as A, B, or C. Properties you add to a rule with this selector are applied to all fields returned by the selected query. +1. Click **Add override property**. +1. Select the field option that you want to apply. +1. Enter options by adding values in the fields. To return options to default values, delete the white text in the fields. +1. Continue to add overrides to this field by clicking **Add override property**, or you can click **Add override** and select a different field to add overrides to. +1. When finished, click **Save** to save all panel edits to the dashboard. diff --git a/docs/sources/panels/override-field-values/delete-a-field-override.md b/docs/sources/panels/override-field-values/delete-a-field-override.md new file mode 100644 index 00000000000..2dfb853a79c --- /dev/null +++ b/docs/sources/panels/override-field-values/delete-a-field-override.md @@ -0,0 +1,22 @@ ++++ +title = "Delete a field override" +weight = 50 +aliases = ["/docs/sources/panels/override-field-values/delete-a-field-override/"] ++++ + +# Delete a field override + +Delete a field override when you no longer need it. + +When you delete an override, the appearance of value defaults to its original format. This change impacts dashboards and dashboard users that rely on an affected panel. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). +- [Add a field override]({{< relref "../override-field-values/add-a-field-override.md" >}}). + +**To delete a field override**: + +1. Edit the panel that contains the override you want to delete. +1. In panel display options, click the **Overrides** tab. +1. Click the override you want to delete and then click the associated trash icon. diff --git a/docs/sources/panels/override-field-values/edit-field-override.md b/docs/sources/panels/override-field-values/edit-field-override.md new file mode 100644 index 00000000000..9ff10bdab18 --- /dev/null +++ b/docs/sources/panels/override-field-values/edit-field-override.md @@ -0,0 +1,24 @@ ++++ +title = "Edit a field overrides" +weight = 40 +aliases = ["/docs/sources/panels/override-field-values/edit-field-override/"] ++++ + +# Edit a field override + +Edit a field override when you want to make changes to an override setting. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). +- [Add a field override]({{< relref "../override-field-values/add-a-field-override.md" >}}). + +**To edit a field override**: + +1. Edit the panel that contains the overrides you want to edit. +1. In the panel display options, click the **Overrides** tab. +1. Locate the override that you want to change. +1. Perform any of the following: + - Edit settings on existing overrides or field selection parameters. + - Delete existing override properties by clicking the **X** next to the property. + - Add an override properties by clicking **Add override property**. diff --git a/docs/sources/panels/override-field-values/view-field-override.md b/docs/sources/panels/override-field-values/view-field-override.md new file mode 100644 index 00000000000..a017b32a2ad --- /dev/null +++ b/docs/sources/panels/override-field-values/view-field-override.md @@ -0,0 +1,21 @@ ++++ +title = "View field overrides" +weight = 20 +aliases = ["/docs/sources/panels/override-field-values/view-field-override/"] ++++ + +# View field overrides + +You can view field overrides in the panel display options. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). +- [Add a field override]({{< relref "../override-field-values/add-a-field-override.md" >}}). + +**To view field overrides**: + +1. Open for edit the panel that contains the overrides you want to view. +1. In the panel display options, click the **Overrides** tab. + +> The override settings that appear on the **All** tab are the same as the settings that appear on the **Overrides** tab. diff --git a/docs/sources/panels/panel-editor.md b/docs/sources/panels/panel-editor.md deleted file mode 100644 index 845ddead5c1..00000000000 --- a/docs/sources/panels/panel-editor.md +++ /dev/null @@ -1,86 +0,0 @@ -+++ -title = "Panel editor" -weight = 200 -+++ - -# Panel editor - -This page describes the parts of the Grafana panel editor and links to where you can find more information. - -{{< figure src="/static/img/docs/panel-editor/panel-editor-8-0.png" class="docs-image--no-shadow" max-width="1500px" >}} - -If your user account is assigned the appropriate [organization role]({{< relref "../permissions/organization_roles.md" >}}) or [permissions]({{< relref "../permissions/_index.md" >}}), then you can edit or update a panel at any point after it is created. For more information about creating panels, refer to [Add a panel]({{< relref "./add-a-panel.md" >}}). - -## Open the panel editor - -There are several ways to access the panel editor, also called the **Edit Panel** screen, _edit mode_, or _panel edit mode_: - -- Click the **Add panel** icon at the top of the screen and then click **Add new panel**. The new panel opens in the panel editor. For detailed instructions on how to add a panel, refer to [Add a panel]({{< relref "add-a-panel.md" >}}) -- Click the title of an existing panel and then click **Edit**. The panel opens in edit mode. -- Click anywhere on an existing panel and then press **e** on your keyboard. The panel opens in edit mode. - -## Parts of the panel editor - -This section describes the parts of the panel editor screen and a bit about fields, options, or tasks associated with each part. Some sections in this page link to pages where sections or tasks are documented more fully. - -### Header - -The header section lists the name of the dashboard that the panel is in and some dashboard commands. You can also click the **Go back** arrow to return to the dashboard. - -{{< figure src="/static/img/docs/panel-editor/edit-panel-header-7-0.png" class="docs-image--no-shadow" max-width="1000px" >}} - -On the right side of the header are the following options: - -- **Dashboard settings (gear) icon -** Click to access the dashboard settings. -- **Discard -** Discards all changes you have made to the panel since you last saved the dashboard. -- **Save -** Saves the dashboard, including all changes you have made in the panel editor. -- **Apply -** Applies changes you made and then closes the panel editor, returning you to the dashboard. You will have to save the dashboard to persist the applied changes. - -### Visualization preview - -The visualization preview section contains viewing options, time range controls, the visualization preview, and (if applicable) the panel title, axes, and legend. - -{{< figure src="/static/img/docs/panel-editor/visualization-preview-8-0.png" class="docs-image--no-shadow" max-width="1200px" >}} - -- **Table view -** Convert any visualization to a table so that you can see the data. Useful for troubleshooting. -- **Fill -** The visualization preview will fill the available space in the preview part. If you change the width of the side pane or height of the bottom pane the visualization will adapt to fill whatever space is available. -- **Fit -** The visualization preview will fill the available space in but preserve the aspect ratio of the panel. -- **Exact -** The visualization preview will have the exact size as the size on the dashboard. If not enough space is available, the visualization will scale down preserving the aspect ratio. -- **Time range controls -** For more information, refer to [Time range controls]({{< relref "../dashboards/time-range-controls.md" >}}). - -### Data section (bottom pane) - -The section contains tabs where you enter queries, transform your data, and create alert rules (if applicable). - -{{< figure src="/static/img/docs/panel-editor/data-section-8-0.png" class="docs-image--no-shadow" max-width="1200px" >}} - -- **Query tab -** Select your data source and enter queries here. For more information, refer to [Queries]({{< relref "queries.md" >}}). -- **Transform tab -** Apply data transformations. For more information, refer to [Transformations]({{< relref "transformations/_index.md" >}}). -- **Alert tab -** Write alert rules. For more information, refer to [Create alerts]({{< relref "../alerting/old-alerting/create-alerts.md" >}}). - -### Panel display options (side pane) - -The section contains tabs where you configure almost every aspect of your data Visualization. Not all options are available for each visualization. - -The data model used in Grafana, namely the [data frame]({{< relref "../developers/plugins/data-frames.md" >}}), is a columnar-oriented table structure that unifies both time series and table query results. Each column within this structure is called a _field_. A field can represent a single time series or table column. - -Field options allow you to change how the data is displayed in your visualizations. Options and overrides that you apply do not change the data, however they change how Grafana displays the data. - -#### Field options - -When you change an option, it is applied to all fields, meaning all series or columns. For example, if you change the unit to percentage, then all fields with numeric values are displayed in percentages. - -## Field overrides - -_Field overrides_ can be added or viewed in the Overrides tab in the panel editor side menu. You can apply options to specific fields (series or columns) rather than all fields. Learn how to apply an override in [Field overrides]({{< relref "./field-overrides.md" >}}). - -Options are documented in the following topics: - -- [Add a panel]({{< relref "./add-a-panel.md" >}}) describes how to add a panel to a dashboard. -- [Panel options]({{< relref "./panel-options.md" >}}) -- [Visualization options]({{< relref "../visualizations/_index.md" >}}) vary widely. They are described in the individual visualization topic. -- [Standard options]({{< relref "./standard-options.md" >}}) -- [Thresholds]({{< relref "./thresholds.md" >}}) -- [Value mappings]({{< relref "./value-mappings.md" >}}) -- [Panel links]({{< relref "../linking/panel-links.md" >}}) and [Data links]({{< relref "../linking/data-links.md" >}}) help you connect your visualization to other resources. -- [Overrides]({{< relref "./field-overrides.md" >}}) diff --git a/docs/sources/panels/panel-library.md b/docs/sources/panels/panel-library.md deleted file mode 100644 index b3bdd019236..00000000000 --- a/docs/sources/panels/panel-library.md +++ /dev/null @@ -1,77 +0,0 @@ -+++ -title = "Library panels" -weight = 410 -+++ - -# Library panels - -Library panels allow users to create reusable panels where any changes made to one instance of the library panel is reflected on every dashboard affecting all other instances where the panel is used. These panels can be saved in folders alongside Dashboards and streamline reuse of panels across multiple dashboards. - -## Create a library panel - -Before you can use library panels, you have to create them. - -> **Note:** When you create library panels, the panel on the source dashboard is converted to a library panel as well. You will need to save the original dashboard once a panel is converted. - -To create a library panel: - -1. Create a Grafana panel as you normally would, following instructions in [Add a panel]({{< relref "./add-a-panel.md" >}}). You an also use an existing panel. -1. Click the title of the panel and then click **Edit**. The panel opens in edit mode. -1. In the panel display options (side pane), click the down arrow option to bring changes to the visualization. - {{< figure src="/static/img/docs/library-panels/create-lib-panel-from-edit-8-0.png" class="docs-image--no-shadow" max-width= "800px" caption="Screenshot of the edit panel" >}} -1. Click the **Library panels** option, and then click **Create new library panel** to open the create dialog. - {{< figure src="/static/img/docs/library-panels/create-lib-panel-8-0.png" class="docs-image--no-shadow" max-width= "500px" caption="Screenshot of the create library panel dialog" >}} -1. In **Library panel name**, enter the name. -1. In **Save in folder**, select the folder to save the library panel. By default, the General folder is selected. -1. Click **Create library panel** to save your changes. -1. Save the dashboard. - -Optionally, you can click title of the panel and then click **More > Create Library panel**. Next, follow the instructions Step 5 in the procedure above. - -{{< figure src="/static/img/docs/library-panels/create-from-more-8-0.png" class="docs-image--no-shadow" max-width= "900px" caption="Screenshot of the edit panel" >}} - -Also, library panels can also be created using the “Share” option for any panel. - -Once created, you can modify the library panel using any dashboard on which it appears. Once the library panel changes are saved, all instances of the library panel will reflect these modifications. - -## Add a library panel - -To add a library panel to a dashboard: - -1. Hover over the **+** option on the left menu, then select **Create** from the drop-down options. The Add panel dialog opens. - {{< figure src="/static/img/docs/library-panels/add-library-panel-8-0.png" class="docs-image--no-shadow" max-width= "900px" caption="Screenshot of the edit panel" >}} -1. Click the **Add a panel from the panel library** option. You will see a list of your library panels. -1. Filter the list or search to find the panel you want to add. -1. Click your desired panel and add it to the dashboard. - -## Manage library panels - -To view and manage existing library panels: - -1. Hover over the **Dashboard** option on the left menu, then click **Library panels**. You can see a list of previously defined library panels. - {{< figure src="/static/img/docs/library-panels/library-panel-list-8-0.png" class="docs-image--no-shadow" max-width= "900px" caption="Screenshot of the edit panel" >}} -1. Search for a specific library panel if you know its name. You can also filter the panels by folder or type. - -## Unlink a library panel - -In case you have a library panel on your dashboard that want to modify it without affecting all other instances of the library panel, you can unlink the library panel. - -To unlink a library panel from a dashboard: - -1. Hover over **Dashboard** on the left menu, and then click **Library panels**. -1. Select a library panel that is being used in different dashboards. You will see a list of all the dashboards where the library panel is used. - {{< figure src="/static/img/docs/library-panels/unlink-8-0.png" class="docs-image--no-shadow" max-width= "900px" caption="Screenshot of the edit panel" >}} -1. Select the panel you want to unlink and update. -1. Click the title of the panel and then click **Edit**. The panel opens in edit mode. -1. Click the **Unlink** option on the top right corner of the UI. - -## Delete a library panel - -Before you delete a library panel, verify that it is no longer in use on any dashboard. - -To delete a library panel: - -1. Hover over **Dashboard** on the left menu, and select Library panels from the drop-down options. -1. Select a library panel that is being used in different dashboards. You will see a list of all the dashboards. -1. Select the panel you want to delete. -1. Click the delete icon next to the library panel name. diff --git a/docs/sources/panels/panel-options.md b/docs/sources/panels/panel-options.md deleted file mode 100644 index 175c803b29b..00000000000 --- a/docs/sources/panels/panel-options.md +++ /dev/null @@ -1,38 +0,0 @@ -+++ -title = "Panel options" -weight = 900 -+++ - -# Panel options - -Panel options are common to all panels. They are basic options to add information and clarity to your panels. Fields are described below. - -While not required, we recommend that you add a helpful title and description to all panels. - -![](/static/img/docs/panels/panel-options-8-0.png) - -## Title - -Text entered in this field is displayed at the top of your panel in the panel editor and in the dashboard. You can use [variables you have defined]({{< relref "../variables/_index.md" >}}) in either field, but not [global variables]({{< relref "../variables/variable-types/global-variables.md" >}}). - -## Description - -Text entered in this field is displayed in a tooltip in the upper left corner of the panel. Write a description of the panel and the data you are displaying. Pretend you are explaining it to a new user six months from now, when it is no longer fresh in your mind. Future editors (possibly yourself) will thank you. - -You can use [variables you have defined]({{< relref "../variables/_index.md" >}}) in either field, but not [global variables]({{< relref "../variables/variable-types/global-variables.md" >}}). - -## Transparent background - -Toggle the transparent background option on your panel display. - -## Panel links - -{{< docs/shared "panels/panel-links-intro.md" >}} - -For more information, refer to [Panel links]({{< relref "../linking/panel-links.md" >}}). - -## Repeat options - -{{< docs/shared "panels/repeat-panels-intro.md" >}} - -For more information, refer to [Repeat panels or rows]({{< relref "./repeat-panels-or-rows.md" >}}). diff --git a/docs/sources/panels/queries.md b/docs/sources/panels/queries.md deleted file mode 100644 index 141d1388211..00000000000 --- a/docs/sources/panels/queries.md +++ /dev/null @@ -1,156 +0,0 @@ -+++ -title = "Queries" -weight = 300 -+++ - -# Queries - -_Queries_ are how Grafana panels communicate with data sources to get data for the visualization. A query is a question written in the query language used by the data source. Grafana asks, "Hey data source, would you send me this data, organized this way?" If the query is properly formed, then the data source responds. How often the query is sent to the data source and how many data points are collected can be adjusted in the panel data source options. - -Grafana supports up to 26 queries per panel. - -## Query editors - -Query editors are forms that help you write queries. Depending on your data source, the query editor might provide auto-completion, metric names, or variable suggestion. - -Because of the difference between query languages, data sources may have query editors that look different. Here are two examples of query editors: - -**InfluxDB query editor** - -{{< figure src="/static/img/docs/queries/influxdb-query-editor-7-2.png" class="docs-image--no-shadow" max-width="1000px" >}} - -**Prometheus (PromQL) query editor** - -{{< figure src="/static/img/docs/queries/prometheus-query-editor-7-4.png" class="docs-image--no-shadow" max-width="1000px" >}} - -## Query syntax - -Data sources have different query languages and syntaxes to ask for the data. Here are two query examples: - -**PostgreSQL** - -``` -SELECT hostname FROM host WHERE region IN($region) -``` - -**PromQL** - -``` -query_result(max_over_time([${__range_s}s]) != ) -``` - -For more information about writing a query for your data source, refer to the specific [Grafana data source]({{< relref "../datasources/_index.md" >}}) documentation. - -## Query tab UI - -The Query tab consists of the following elements: - -- Data source selector -- Query options -- Query inspector button -- Query editor list -- Expressions - -{{< figure src="/static/img/docs/queries/query-editor-7-2.png" class="docs-image--no-shadow" max-width="1000px" >}} - -### Data source selector - -The data source selector is a drop-down list. Click it to select a data source you have added. When you create a panel, Grafana automatically selects your default data source. For more information about adding data sources, refer to [Add a data source]({{< relref "../datasources/add-a-data-source.md" >}}). - -{{< figure src="/static/img/docs/queries/data-source-selector-7-0.png" class="docs-image--no-shadow" max-width="250px" >}} - -In addition to the data sources that you have configured in your Grafana, there are three special data sources available: - -- **Grafana -** A built-in data source that generates random walk data. Useful for testing visualizations and running experiments. -- **Mixed -** Select this to query multiple data sources in the same panel. When this data source is selected, Grafana allows you to select a data source for every new query that you add. - - The first query will use the data source that was selected before you selected **Mixed**. - - You cannot change an existing query to use the Mixed Data Source. -- **Dashboard -** Select this to use a result set from another panel in the same dashboard. - -### Query options - -Click **Query options** next to the data source selector to see settings for your selected data source. Changes you make here affect only queries made in this panel. - -{{< figure src="/static/img/docs/queries/data-source-options-7-0.png" class="docs-image--no-shadow" max-width="1000px" >}} - -Grafana sets defaults that are shown in dark gray text. Changes are displayed in white text. To return a field to the default setting, delete the white text from the field. - -Panel data source query options: - -- **Max data points -** If the data source supports it, sets the maximum numbers of data points for each series returned. If the query returns more data points than the max data points setting, then the data source consolidates them (reduces the number of points returned by aggregating them together by average or max or other function). - - There are two main reasons for limiting the number of points, performance and smoothing the line. The default value is the width (or number of pixels) of the graph as there is no point in having more data points than the graph panel can display. - - With streaming data, the max data points value is used for the rolling buffer. (Streaming is a continuous flow of data and buffering is a way of dividing the stream into chunks). Loki streams data in the live tailing mode. - -- **Min interval -** Sets a minimum limit for the automatically calculated interval, typically the minimum scrape interval. If a data point is saved every 15 seconds, then there's no point in having an interval lower than that. Another use case is to set it to a higher minimum than the scrape interval to get more coarse-grained, well-functioning queries. - -- **Interval -** The interval is a time span that you can use when aggregating or grouping data points by time. - - Grafana automatically calculates an appropriate interval and it can be used as a variable in templated queries. The variable is either in seconds: `$__interval` or in milliseconds: `$__interval_ms`. It is typically used in aggregation functions like sum or average. For example, a Prometheus query using the interval variable: `rate(http_requests_total[$__interval])`. - - This automatic interval is calculated based on the width of the graph. If the user zooms out a lot then the interval becomes greater, resulting in a more coarse grained aggregation whereas if the user zooms in then the interval decreases resulting in a more fine grained aggregation. - - For more information, refer to [Global variables]({{< relref "../variables/variable-types/global-variables.md" >}}). - -- **Relative time -** You can override the relative time range for individual panels, causing them to be different than what is selected in the dashboard time picker in the top right corner of the dashboard. This allows you to show metrics from different time periods or days on the same dashboard. - - > **Note:** Panel time overrides have no effect when the time range for the dashboard is absolute. - -- **Time shift -** The time shift function is another way to override the time range for individual panels. It only works with relative time ranges and allows you to adjust the time range. - - For example, you could shift the time range for the panel to be two hours earlier than the dashboard time picker. For more information, refer to [Time range controls]({{< relref "../dashboards/time-range-controls.md" >}}). - -- **Cache timeout -** (This field is only visible if available in your data source.) If your time series store has a query cache, then this option can override the default cache timeout. Specified as a numeric value in seconds. - -### Examples: - -- **Relative time:** - -| Example | Relative time field | -| ---------------- | ------------------- | -| Last 5 minutes | `now-5m` | -| The day so far | `now/d` | -| Last 5 days | `now-5d/d` | -| This week so far | `now/w` | -| Last 2 years | `now-2y/y` | - -- **Time shift:** - -| Example | Time shift field | -| -------------------- | ---------------- | -| Last entire week | `1w/w` | -| Two entire weeks ago | `2w/w` | -| Last entire month | `1M/M` | -| This entire year | `1d/y` | -| Last entire year | `1y/y` | - -### Query inspector button - -You can click **Query inspector** to open the Query tab of the panel inspector where you can see the query request sent by the panel and the response. - -Click **Refresh** to see the full text of the request sent by this panel to the server. - -> **Note:** You need to add at least one query before the Query inspector can return results. - -For more information about the panel inspector, refer to [Inspect a panel]({{< relref "inspect-panel.md" >}}). - -### Query editor list - -In the UI, queries are organized in collapsible query rows. Each query row contains a query editor and is identified with a letter (A, B, C, and so on). - -You can: - -| Icon | Description | -| :-----------------------------------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| {{< figure src="/static/img/docs/queries/query-editor-help-7-4.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Toggle query editor help. If supported by the data source, click this icon to display information on how to use the query editor or provide quick access to common queries. | -| {{< figure src="/static/img/docs/queries/duplicate-query-icon-7-0.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Copy a query. Duplicating queries is useful when working with multiple complex queries that are similar and you want to either experiment with different variants or do minor alterations. | -| {{< figure src="/static/img/docs/queries/hide-query-icon-7-0.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Hide a query. Grafana does not send hidden queries to the data source. | -| {{< figure src="/static/img/docs/queries/remove-query-icon-7-0.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Remove a query. Removing a query permanently deletes it, but sometimes you can recover deleted queries by reverting to previously saved versions of the panel. | -| {{< figure src="/static/img/docs/queries/query-drag-icon-7-2.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Reorder queries. Change the order of queries by clicking and holding the drag icon, then drag queries where desired. The order of results reflects the order of the queries, so you can often adjust your visual results based on query order. | - -### Expressions - -If your data source supports them, then Grafana displays the **Expression** button and shows any existing expressions in the query editor list. - -For more information about expressions, refer to [Expressions]({{< relref "expressions.md" >}}). diff --git a/docs/sources/panels/query-a-data-source/_index.md b/docs/sources/panels/query-a-data-source/_index.md new file mode 100644 index 00000000000..73c9c5b46c0 --- /dev/null +++ b/docs/sources/panels/query-a-data-source/_index.md @@ -0,0 +1,11 @@ ++++ +title = "Query a data source" +aliases = ["/docs/grafana/latest/panels/queries/", "/docs/grafana/latest/panels/expressions/", "/docs/grafana/latest/panels/inspect-panel/", "/docs/grafana/latest/panels/share-query-results/", "/docs/sources/panels/query-a-data-source/_index/"] +weight = 200 ++++ + +# Query a data source + +Data source queries return data that appear in visualizations. The query language in which you write depends on the data source you query. Grafana supports connections with many types of data sources. + +{{< section >}} diff --git a/docs/sources/panels/query-a-data-source/about-queries.md b/docs/sources/panels/query-a-data-source/about-queries.md new file mode 100644 index 00000000000..5f7b18c3d0b --- /dev/null +++ b/docs/sources/panels/query-a-data-source/about-queries.md @@ -0,0 +1,55 @@ ++++ +title = "About queries" +weight = 10 +aliases = ["/docs/sources/panels/query-a-data-source/about-queries/"] ++++ + +# About queries + +_Queries_ are how Grafana panels communicate with data sources to get data for the visualization. A query is a question written in the query language used by the data source. How often the query is sent to the data source and how many data points are collected can be adjusted in the panel data source options. + +Use you a query editor to write a query. Each data source has its own query editor that we have customized to include the features and capabilities of the data source. Grafana supports up to 26 queries per panel. + +> Important! You must be familiar with the query language of the data source. For more information about data sources, refer to [Data sources](../../../datasources/). + +## Query editors + +Depending on your data source, the query editor might provide auto-completion, metric names, or variable suggestions. + +Because of the difference between query languages, data sources have query editors that look different. Here are two examples of query editors. + +**InfluxDB query editor** + +{{< figure src="/static/img/docs/queries/influxdb-query-editor-7-2.png" class="docs-image--no-shadow" max-width="1000px" >}} + +**Prometheus (PromQL) query editor** + +{{< figure src="/static/img/docs/queries/prometheus-query-editor-7-4.png" class="docs-image--no-shadow" max-width="1000px" >}} + +## Query syntax + +Data sources use different query languages to return data. Here are two query examples: + +**PostgreSQL** + +``` +SELECT hostname FROM host WHERE region IN($region) +``` + +**PromQL** + +``` +query_result(max_over_time([${__range_s}s]) != ) +``` + +## Data sources used in queries + +In addition to the data sources that you have configured in Grafana, there are three special data sources available: + +- **Grafana -** A built-in data source that generates random walk data, which can be useful for testing visualizations and running experiments. +- **Mixed -** Select this option to query multiple data sources in the same panel. When you select this data source, Grafana enables you to select a data source for every new query that you add. + - The first query uses the data source that was selected before you selected **Mixed**. + - You cannot change an existing query to use the Mixed Data Source. +- **Dashboard -** Select this option to use a result set from another panel in the same dashboard. + +You can combine data from multiple data sources onto a single dashboard, but each panel is tied to a specific data source that belongs to a particular Organization. diff --git a/docs/sources/panels/query-a-data-source/add-a-query.md b/docs/sources/panels/query-a-data-source/add-a-query.md new file mode 100644 index 00000000000..bf67f641fa5 --- /dev/null +++ b/docs/sources/panels/query-a-data-source/add-a-query.md @@ -0,0 +1,30 @@ ++++ +title = "Add a query" +weight = 30 +aliases = ["/docs/sources/panels/query-a-data-source/add-a-query/"] ++++ + +# Add a query + +A query returns data that Grafana visualizes in dashboards. When you create a panel, Grafana automatically selects the default data source. + +## Before you begin + +- [Add a data source](../../../datasources/add-a-data-source). +- Ensure that you know the query language of the data source. +- [Add a panel]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To add a query**: + +1. Edit the panel to which you are adding a query. +1. Click the **Query** tab. +1. Click the **Data source** drop-down menu and select a data source. +1. Click **Query options** to configure the maximum number of data points returned by the query and how frequently you want the query to request data from the data source. + + For more information about query options, refer to [Reference: Query options]({{< relref "../reference-query-options.md" >}}). + +1. Write the query. + +1. Click **Apply**. + +The system queries the data source and presents the data in the visualization. diff --git a/docs/sources/panels/query-a-data-source/download-raw-query-results.md b/docs/sources/panels/query-a-data-source/download-raw-query-results.md new file mode 100644 index 00000000000..c7b34c1ccdb --- /dev/null +++ b/docs/sources/panels/query-a-data-source/download-raw-query-results.md @@ -0,0 +1,30 @@ ++++ +title = "Download raw query results" +weight = 70 +aliases = ["/docs/sources/panels/query-a-data-source/download-raw-query-results/"] ++++ + +# Download raw query results + +Grafana generates a CSV file that contains your data, including any transformations to that data. You can choose to view the data before or after the panel applies field options or field option overrides. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). +- [Add a query]({{< relref "../query-a-data-source/add-a-query.md" >}}). + +**To download raw query results**: + +1. Edit the panel that contains the query data you want to download. +1. In the query editor, click **Query Inspector**. +1. Click **Data**. + + If your panel contains multiple queries or queries multiple nodes, then you have additional options. + + - **Select result**: Choose which result set data you want to view. + - **Transform data** + - **Join by time**: View raw data from all your queries at once, one result set per column. Click a column heading to reorder the data. + +1. To see data before the system applies field overrides, click the **Formatted data** toggle. +1. To download a CSV file specifically formatted for Excel, click the **Download for Excel** toggle . +1. Click **Download CSV**. diff --git a/docs/sources/panels/query-a-data-source/inspect-query-performance.md b/docs/sources/panels/query-a-data-source/inspect-query-performance.md new file mode 100644 index 00000000000..96276a78520 --- /dev/null +++ b/docs/sources/panels/query-a-data-source/inspect-query-performance.md @@ -0,0 +1,22 @@ ++++ +title = "Inspect query performance" +weight = 80 +aliases = ["/docs/sources/panels/query-a-data-source/inspect-query-performance/"] ++++ + +# Inspect query performance + +The **Stats** tab displays statistics that tell you how long your query takes, how many queries you send, and the number of rows returned. This information can help you troubleshoot your queries, especially if any of the numbers are unexpectedly high or low. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). +- [Add a query]({{< relref "../query-a-data-source/add-a-query.md" >}}). + +**To inspect query performance**: + +1. Edit the panel that contains the query with performance you want to inspect. +1. In the query editor, click **Query Inspector**. +1. Click **Stats**. + +Statistics are displayed in read-only format. diff --git a/docs/sources/panels/query-a-data-source/inspect-request-and-response-data.md b/docs/sources/panels/query-a-data-source/inspect-request-and-response-data.md new file mode 100644 index 00000000000..53895c8b685 --- /dev/null +++ b/docs/sources/panels/query-a-data-source/inspect-request-and-response-data.md @@ -0,0 +1,25 @@ ++++ +title = "Inspect query request and response data" +weight = 90 +aliases = ["/docs/sources/panels/query-a-data-source/inspect-request-and-response-data/"] ++++ + +# Inspect query request and response data + +Inspect query request and response data when you want to troubleshoot a query that returns unexpected results, or fails to return expected results. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). +- [Add a query]({{< relref "../query-a-data-source/add-a-query.md" >}}). + +**To inspect query request and response data**: + +1. Edit the panel that contains the query you want to export. +1. In the query editor, click **Query Inspector**. +1. Click **Refresh**. + + The panel populates with response data. + +1. Make adjustments, as necessary and re-run the query. +1. To download the query request and response data, click the **Copy to clipboard** icon and paste the results into another application. diff --git a/docs/sources/panels/query-a-data-source/manage-queries.md b/docs/sources/panels/query-a-data-source/manage-queries.md new file mode 100644 index 00000000000..c91e8dc2157 --- /dev/null +++ b/docs/sources/panels/query-a-data-source/manage-queries.md @@ -0,0 +1,19 @@ ++++ +title = "Manage queries" +weight = 50 +aliases = ["/docs/sources/panels/query-a-data-source/manage-queries/"] ++++ + +# Manage queries + +Queries are organized in collapsible query rows. Each query row contains a query editor and is identified with a letter (A, B, C, and so on). + +You can: + +| Icon | Description | +| :-----------------------------------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| {{< figure src="/static/img/docs/queries/query-editor-help-7-4.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Toggle query editor help. If supported by the data source, click this icon to display information on how to use the query editor or provide quick access to common queries. | +| {{< figure src="/static/img/docs/queries/duplicate-query-icon-7-0.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Copy a query. Duplicating queries is useful when working with multiple complex queries that are similar and you want to either experiment with different variants or do minor alterations. | +| {{< figure src="/static/img/docs/queries/hide-query-icon-7-0.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Hide a query. Grafana does not send hidden queries to the data source. | +| {{< figure src="/static/img/docs/queries/remove-query-icon-7-0.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Remove a query. Removing a query permanently deletes it, but sometimes you can recover deleted queries by reverting to previously saved versions of the panel. | +| {{< figure src="/static/img/docs/queries/query-drag-icon-7-2.png" class="docs-image--no-shadow" max-width="30px" max-height="30px" >}} | Reorder queries. Change the order of queries by clicking and holding the drag icon, then drag queries where desired. The order of results reflects the order of the queries, so you can often adjust your visual results based on query order. | diff --git a/docs/sources/panels/query-a-data-source/navigate-query-tab.md b/docs/sources/panels/query-a-data-source/navigate-query-tab.md new file mode 100644 index 00000000000..5eeab60ee4b --- /dev/null +++ b/docs/sources/panels/query-a-data-source/navigate-query-tab.md @@ -0,0 +1,17 @@ ++++ +title = "Navigate the query tab" +weight = 20 +aliases = ["/docs/sources/panels/query-a-data-source/navigate-query-tab/"] ++++ + +# Navigate the query tab + +The Query tab consists of the following elements: + +- Data source selector: Use the data source selector to select the source of the data you want to query. For more information about data sources, refer to [Data sources](../../../datasources/). +- Query options: Enables you to set maximum data retrieved parameters and query execution time intervals. +- Query inspector button: Open the query inspector panel where you can view and optimize your query. +- Query editor list: Lists the queries you have written. +- Expressions: Use the expression builder to create alert expressions. For more information about expressions, refer to [Use expressions to manipulate data](../use-expressions-to-manipulate-data/). + +{{< figure src="/static/img/docs/queries/query-editor-7-2.png" class="docs-image--no-shadow" max-width="1000px" >}} diff --git a/docs/sources/panels/share-query-results.md b/docs/sources/panels/query-a-data-source/share-query.md similarity index 62% rename from docs/sources/panels/share-query-results.md rename to docs/sources/panels/query-a-data-source/share-query.md index a61f5475a01..3d50c490bad 100644 --- a/docs/sources/panels/share-query-results.md +++ b/docs/sources/panels/query-a-data-source/share-query.md @@ -1,9 +1,10 @@ +++ -title = "Share query results" -weight = 310 +title = "Share query results with another panel" +weight = 60 +aliases = ["/docs/sources/panels/query-a-data-source/share-query/"] +++ -# Share query results between panels +# Share query results with another panel Grafana let you use the query result from one panel for any other panel in the dashboard. Sharing query results across panels reduces the number of queries made to your data source, which can improve the performance of your dashboard. @@ -11,12 +12,15 @@ The Dashboard data source lets you select a panel in your dashboard that contain This strategy can drastically reduce the number of queries being made when you for example have several panels visualizing the same data. -To share data source queries with another panel: +**To share data source queries with another panel**: -1. [Create a dashboard]({{< relref "../getting-started/getting-started.md#create-a-dashboard" >}}). -1. [Add a panel]({{< relref "add-a-panel.md" >}}) to the dashboard. +1. [Create a dashboard]({{< relref "../../getting-started/getting-started.md/#step-3-create-a-dashboard" >}}). +1. [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). 1. Change the title to "Source panel". You'll use this panel as a source for the other panels. - Define the [query]({{< relref "queries.md" >}}) or queries that will be shared. If you don't have a data source available at the moment, then you can use the **Grafana** data source, which returns a random time series that you can use for testing. +1. Define the [query]({{< relref "../query-a-data-source/add-a-query.md" >}}) or queries that you want share. + + If you don't have a data source available, use the **Grafana** data source, which returns a random time series that you can use for testing. + 1. Add a second panel and select the **Dashboard** data source in the query editor. 1. In the **Use results from panel list**, select the first panel you created. diff --git a/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/_index.md b/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/_index.md new file mode 100644 index 00000000000..1c24070b9b5 --- /dev/null +++ b/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/_index.md @@ -0,0 +1,11 @@ ++++ +title = "Use expressions to manipulate data" +weight = 40 +aliases = ["/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/"] ++++ + +# Use expressions to manipulate data + +Server-side expressions enable you to manipulate data returned from queries with math and other operations. Expressions create new data and do not manipulate the data returned by data sources. + +{{< section >}} diff --git a/docs/sources/panels/expressions.md b/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions.md similarity index 85% rename from docs/sources/panels/expressions.md rename to docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions.md index 7883c45602b..5fdade2cdd2 100644 --- a/docs/sources/panels/expressions.md +++ b/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions.md @@ -1,9 +1,10 @@ +++ -title = "Expressions" -weight = 800 +title = "About expressions" +weight = 10 +aliases = ["/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions/"] +++ -# Server-side expressions +# About expressions > **Note:** This documentation is for a beta feature. @@ -11,7 +12,7 @@ Server-side expressions allow you to manipulate data returned from queries with ## Using expressions -Expressions are primarily used by the new [Grafana 8 alerts]({{< relref "../alerting/unified-alerting/" >}}). The processing is done server-side, so expressions can operate without a browser session. However, expressions can also be used with backend data sources and visualization. +Expressions are primarily used by the new [Grafana 8 alerts]({{< relref "../../../alerting/unified-alerting/_index.md" >}}). The processing is done server-side, so expressions can operate without a browser session. However, expressions can also be used with backend data sources and visualization. > **Note:** Expressions do not work with legacy dashboard alerts. @@ -21,20 +22,20 @@ Expressions are meant to augment data sources by enabling queries from different > **Note:** When possible, you should do data processing inside the data source. Copying data from storage to the Grafana server for processing is inefficient, so expressions are targeted at lightweight data processing. -Expressions work with data source queries that return time series or number data. They also operate on [multiple-dimensional data]({{< relref "../basics/timeseries-dimensions.md" >}}). For example, a query that returns multiple series, where each series is identified by labels or tags. +Expressions work with data source queries that return time series or number data. They also operate on [multiple-dimensional data]({{< relref "../../../basics/timeseries-dimensions.md" >}}). For example, a query that returns multiple series, where each series is identified by labels or tags. An individual expression takes one or more queries or other expressions as input and adds data to the result. Each individual expression or query is represented by a variable that is a named identifier known as its RefID (e.g., the default letter `A` or `B`). To reference the output of an individual expression or a data source query in another expression, this identifier is used as a variable. -## Types of expression +## Types of expressions Expressions work with two types of data. - A collection of time series. -- A collection of numbers, where each item in the collection is a single number. +- A collection of numbers, where each number is an item. -Each collection is returned from a single data source query or expression and represented by the RefID. Each collection is a set, where each item in the set is uniquely identified by it dimensions which are stored as [labels]({{< relref "../basics/timeseries-dimensions.md#labels" >}}) or key-value pairs. +Each collection is returned from a single data source query or expression and represented by the RefID. Each collection is a set, where each item in the set is uniquely identified by its dimensions which are stored as [labels]({{< relref "../../../basics/timeseries-dimensions.md#labels" >}}) or key-value pairs. ## Data source queries @@ -49,7 +50,7 @@ Currently, the only non-time series format (number) is supported when using data | MIA | A | 1 | | NYC | B | 2 | -will produce a number that works with expressions. The string columns become labels and the number column the corresponding value. For example `{"Loc": "MIA", "Host": "A"}` with a value of 1. +The example above will produce a number that works with expressions. The string columns become labels and the number column the corresponding value. For example `{"Loc": "MIA", "Host": "A"}` with a value of 1. ## Operations @@ -75,7 +76,7 @@ With binary operations, such as `$A + $B` or `$A || $B`, the operator is applied - If one variable is a number, and the other variable is a time series, then the operation between the value of each point in the time series and the number is performed. - If both `$A` and `$B` are time series data, then the operation between each value in the two series is performed for each time stamp that exists in both `$A` and `$B`. The Resample operation can be used to line up time stamps. (**Note:** in the future, we plan to add options to the Math operation for different behaviors). -So in summary: +Summary: - Number OP number = number - Number OP series = series @@ -127,6 +128,18 @@ Log returns the natural logarithm of of its argument which can be a number or a The inf, infn, nan, and null functions all return a single value of the name. They primarily exist for testing. Example: `null()`. +##### round + +Round returns a rounded integer value. For example, `round(3.123)` or `round($A)`. (This function should probably take an argument so it can add precision to the rounded value). + +##### ceil + +Ceil rounds the number up to the nearest integer value. For example, `ceil(3.123)` returns 4. + +##### floor + +Floor rounds the number down to the nearest integer value. For example, `floor(3.123)` returns 3. + ### Reduce Reduce takes one or more time series returned from a query or an expression and turns each series into a single number. The labels of the time series are kept as labels on each outputted reduced number. @@ -156,6 +169,10 @@ Min and Max return the smallest or largest value in the series respectively. If Sum returns the total of all values in the series. If series is of zero length, the sum will be 0. If there are any NaN or Null values in the series, NaN is returned. +#### Last + +Last returns the last number in the series. If the series has no values then returns NaN. + ### Resample Resample changes the time stamps in each time series to have a consistent time interval. The main use case is so you can resample time series that do not share the same timestamps so math can be performed between them. This can be done by resample each of the two series, and then in a Math operation referencing the resampled variables. diff --git a/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression.md b/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression.md new file mode 100644 index 00000000000..fde40e9b3d2 --- /dev/null +++ b/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression.md @@ -0,0 +1,26 @@ ++++ +title = "Write an expression" +weight = 20 +aliases = ["/docs/sources/panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression/"] ++++ + +# Write an expression + +If your data source supports them, then Grafana displays the **Expression** button and shows any existing expressions in the query editor list. + +For more information about expressions, refer to [About expressions]({{< relref "./about-expressions.md" >}}). + +## Before you begin + +- [Add a panel]({{< relref "../../working-with-panels/add-panel.md" >}}). + +**To write an expression**: + +1. Open the panel. +1. Below the query, click **Expression**. +1. In the **Operation** field, select the type of expression you want to write. + + For more information about expression operations, refer to [About expressions]({{< relref "./about-expressions.md" >}}). + +1. Write the expression. +1. Click **Apply**. diff --git a/docs/sources/panels/calculations-list.md b/docs/sources/panels/reference-calculation-types.md similarity index 80% rename from docs/sources/panels/calculations-list.md rename to docs/sources/panels/reference-calculation-types.md index 95dd82389b3..f341a739c25 100644 --- a/docs/sources/panels/calculations-list.md +++ b/docs/sources/panels/reference-calculation-types.md @@ -1,13 +1,12 @@ +++ -title = "Calculations list" -weight = 460 +title = "Reference: Calculation types" +aliases = ["/docs/grafana/latest/panels/calculations-list/", "/docs/sources/panels/reference-calculation-types/"] +weight = 1100 +++ -# List of calculations +# Reference: Calculations -This topic lists and defines calculations used in Grafana. - -Among other places, these calculations are used in the Transform tab and the bar gauge, gauge, and stat visualizations. +Refer to the following list of calculations you can perform in Grafana. You can find these calculations in the **Transform** tab and in the bar gauge, gauge, and stat visualizations. | Calculation | Description | | :----------------- | :-------------------------------------------------------- | diff --git a/docs/sources/panels/reference-query-options.md b/docs/sources/panels/reference-query-options.md new file mode 100644 index 00000000000..e7dcc60c98a --- /dev/null +++ b/docs/sources/panels/reference-query-options.md @@ -0,0 +1,61 @@ ++++ +title = "Reference: Query options" +aliases = ["/docs/sources/panels/reference-query-options/"] +weight = 900 ++++ + +# Reference: Query options + +Click **Query options** next to the data source selector to see settings for your selected data source. Changes you make here affect only queries made in this panel. + +{{< figure src="/static/img/docs/queries/data-source-options-7-0.png" class="docs-image--no-shadow" max-width="1000px" >}} + +Grafana sets defaults that are shown in dark gray text. Changes are displayed in white text. To return a field to the default setting, delete the white text from the field. + +Panel data source query options: + +- **Max data points -** If the data source supports it, sets the maximum numbers of data points for each series returned. If the query returns more data points than the max data points setting, then the data source consolidates them (reduces the number of points returned by aggregating them together by average or max or other function). + + There are two main reasons for limiting the number of points, performance and smoothing the line. The default value is the width (or number of pixels) of the graph as there is no point in having more data points than the graph panel can display. + + With streaming data, the max data points value is used for the rolling buffer. (Streaming is a continuous flow of data and buffering is a way of dividing the stream into chunks). Loki streams data in the live tailing mode. + +- **Min interval -** Sets a minimum limit for the automatically calculated interval, typically the minimum scrape interval. If a data point is saved every 15 seconds, then there's no point in having an interval lower than that. Another use case is to set it to a higher minimum than the scrape interval to get more coarse-grained, well-functioning queries. + +- **Interval -** The interval is a time span that you can use when aggregating or grouping data points by time. + + Grafana automatically calculates an appropriate interval and it can be used as a variable in templated queries. The variable is either in seconds: `$__interval` or in milliseconds: `$__interval_ms`. It is typically used in aggregation functions like sum or average. For example, a Prometheus query using the interval variable: `rate(http_requests_total[$__interval])`. + + This automatic interval is calculated based on the width of the graph. If the user zooms out a lot then the interval becomes greater, resulting in a more coarse grained aggregation whereas if the user zooms in then the interval decreases resulting in a more fine grained aggregation. + + For more information, refer to [Global variables]({{< relref "../variables/variable-types/global-variables.md" >}}). + +- **Relative time -** You can override the relative time range for individual panels, causing them to be different than what is selected in the dashboard time picker in the top right corner of the dashboard. This allows you to show metrics from different time periods or days on the same dashboard. + +- **Time shift -** The time shift function is another way to override the time range for individual panels. It only works with relative time ranges and allows you to adjust the time range. + + For example, you could shift the time range for the panel to be two hours earlier than the dashboard time picker. For more information, refer to [Time range controls]({{< relref "../dashboards/time-range-controls.md" >}}). + +- **Cache timeout -** (This field is only visible if available in your data source.) If your time series store has a query cache, then this option can override the default cache timeout. Specified as a numeric value in seconds. + +### Examples: + +- **Relative time:** + +| Example | Relative time field | +| ---------------- | ------------------- | +| Last 5 minutes | `now-5m` | +| The day so far | `now/d` | +| Last 5 days | `now-5d/d` | +| This week so far | `now/w` | +| Last 2 years | `now-2y/y` | + +- **Time shift:** + +| Example | Time shift field | +| -------------------- | ---------------- | +| Last entire week | `1w/w` | +| Two entire weeks ago | `2w/w` | +| Last entire month | `1M/M` | +| This entire year | `1d/y` | +| Last entire year | `1y/y` | diff --git a/docs/sources/panels/standard-options.md b/docs/sources/panels/reference-standard-field-definitions.md similarity index 63% rename from docs/sources/panels/standard-options.md rename to docs/sources/panels/reference-standard-field-definitions.md index 1743079f91a..d8815dd2749 100644 --- a/docs/sources/panels/standard-options.md +++ b/docs/sources/panels/reference-standard-field-definitions.md @@ -1,11 +1,10 @@ +++ -title = "Standard field options" -keywords = ["grafana", "table options", "documentation", "format tables"] -aliases = ["/docs/grafana/latest/panels/field-options/standard-field-options/"] -weight = 430 +title = "Reference: Standard field definitions" +aliases = ["/docs/sources/panels/reference-standard-field-definitions/"] +weight = 1200 +++ -# Standard field options +# Reference: Standard field definitions This section explains all available standard options. They are listed in alphabetical order. @@ -82,35 +81,3 @@ You can also paste a native emoji in the unit picker and pick it as a custom uni ### String units Grafana can sometime be too aggressive in parsing strings and displaying them as numbers. To make Grafana show the original string create a field override and add a unit property with the `string` unit. - -## Color scheme - -{{< figure src="/static/img/docs/v73/color_scheme_dropdown.png" max-width="350px" caption="Color scheme" class="pull-right" >}} - -The color scheme option defines how Grafana colors series or fields. There are multiple modes here that work very differently and their utility depends largely on what visualization you currently have selected. - -Some visualizations have different color options. - -### Color by value - -In addition to deriving color from thresholds there are also continuous (gradient) color schemes. These are useful for visualizations that color individual values. For example, stat panels and the table panel. - -Continuous color modes use the percentage of a value relative to min and max to interpolate a color. - -
- -### Palettes - -Select a palette from the **Color scheme** list. - -| Color mode | Description | -| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Single color** | Specify a single color, useful in an override rule | -| **From thresholds** | Informs Grafana to take the color from the matching threshold | -| **Classic palette** | Grafana will assign color by looking up a color in a palette by series index. Useful for Graphs and pie charts and other categorical data visualizations | -| **Green-Yellow-Red (by value)** | Continuous color scheme | -| **Blue-Yellow-Red (by value)** | Continuous color scheme | -| **Blues (by value)** | Continuous color scheme (panel background to blue) | -| **Reds (by value)** | Continuous color scheme (panel background color to blue) | -| **Greens (by value)** | Continuous color scheme (panel background color to blue) | -| **Purple (by value)** | Continuous color scheme (panel background color to blue) | diff --git a/docs/sources/panels/transformations/types-options.md b/docs/sources/panels/reference-transformation-functions.md similarity index 81% rename from docs/sources/panels/transformations/types-options.md rename to docs/sources/panels/reference-transformation-functions.md index a898d213524..818c7c449e0 100644 --- a/docs/sources/panels/transformations/types-options.md +++ b/docs/sources/panels/reference-transformation-functions.md @@ -1,71 +1,33 @@ +++ -title = "Types and options" -weight = 2 +title = "Reference: Transformation functions" +aliases = ["/docs/grafana/latest/panels/transformations/types-options/", "docs/sources/panels/reference-transformation-functions/"] +weight = 1000 +++ -# Transformation types and options +# Reference: Transformation functions -Grafana comes with the following transformations: +You can perform the following transformations on your data. -- [Add field from calculation]({{< relref "./types-options.md#add-field-from-calculation" >}}) -- [Concatenate fields]({{< relref "./types-options.md#concatenate-fields" >}}) -- [Config from query results]({{< relref "./config-from-query.md" >}}) -- [Convert field type]({{< relref "./types-options.md#convert-field-type" >}}) -- [Filter data by name]({{< relref "./types-options.md#filter-data-by-name" >}}) -- [Filter data by query]({{< relref "./types-options.md#filter-data-by-query" >}}) -- [Filter data by value]({{< relref "./types-options.md#filter-data-by-value" >}}) -- [Group by]({{< relref "./types-options.md#group-by" >}}) -- [Labels to fields]({{< relref "./types-options.md#labels-to-fields" >}}) -- [Merge]({{< relref "./types-options.md#merge" >}}) -- [Organize fields]({{< relref "./types-options.md#organize-fields" >}}) -- [Outer join]({{< relref "./types-options.md#join-by-field-outer-join" >}}) -- [Reduce]({{< relref "./types-options.md#reduce" >}}) -- [Rename by regex]({{< relref "./types-options.md#rename-by-regex" >}}) -- [Rows to fields]({{< relref "./rows-to-fields" >}}) -- [Series to rows]({{< relref "./types-options.md#series-to-rows" >}}) -- [Sort by]({{< relref "./types-options.md#sort-by" >}}) -- [Prepare-time-series]({{< relref "./types-options.md#prepare-time-series" >}}) +## Add field from calculation -Keep reading for detailed descriptions of each type of transformation and the options available for each, as well as suggestions on how to use them. +Use this transformation to add a new field calculated from two other fields. Each transformation allows you to add one new field. -## Reduce +- **Mode -** Select a mode: + - **Reduce row -** Apply selected calculation on each row of selected fields independently. + - **Binary option -** Apply basic math operation(sum, multiply, etc) on values in a single row from two selected fields. +- **Field name -** Select the names of fields you want to use in the calculation for the new field. +- **Calculation -** If you select **Reduce row** mode, then the **Calculation** field appears. Click in the field to see a list of calculation choices you can use to create the new field. For information about available calculations, refer to [Reference: Calculations]({{< relref "./reference-calculation-types.md" >}}). +- **Operation -** If you select **Binary option** mode, then the **Operation** fields appear. These fields allow you to do basic math operations on values in a single row from two selected fields. You can also use numerical values for binary operations. +- **Alias -** (Optional) Enter the name of your new field. If you leave this blank, then the field will be named to match the calculation. +- **Replace all fields -** (Optional) Select this option if you want to hide all other fields and display only your calculated field in the visualization. -The _Reduce_ transformation applies a calculation to each field in the frame and return a single value. Time fields are removed when applying -this transformation. +In the example below, I added two fields together and named them Sum. -Consider the input: +{{< figure src="/static/img/docs/transformations/add-field-from-calc-stat-example-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} -Query A: +## Concatenate fields -| Time | Temp | Uptime | -| ------------------- | ---- | ------- | -| 2020-07-07 11:34:20 | 12.3 | 256122 | -| 2020-07-07 11:24:20 | 15.4 | 1230233 | - -Query B: - -| Time | AQI | Errors | -| ------------------- | --- | ------ | -| 2020-07-07 11:34:20 | 6.5 | 15 | -| 2020-07-07 11:24:20 | 3.2 | 5 | - -The reduce transformer has two modes: - -- **Series to rows -** Creates a row for each field and a column for each calculation. -- **Reduce fields -** Keeps the existing frame structure, but collapses each field into a single value. - -For example, if you used the **First** and **Last** calculation with a **Series to rows** transformation, then -the result would be: - -| Field | First | Last | -| ------ | ------ | ------- | -| Temp | 12.3 | 15.4 | -| Uptime | 256122 | 1230233 | -| AQI | 6.5 | 3.2 | -| Errors | 15 | 5 | - -The **Reduce fields** with the **Last** calculation, -results in two frames, each with one row: +This transformation combines all fields from all frames into one result. Consider: Query A: @@ -79,32 +41,47 @@ Query B: | --- | ------ | | 3.2 | 5 | -## Merge +After you concatenate the fields, the data frame would be: -Use this transformation to combine the result from multiple queries into one single result. This is helpful when using the table panel visualization. Values that can be merged are combined into the same row. Values are mergeable if the shared fields contain the same data. For information, refer to [Table panel]({{< relref "../../visualizations/table/_index.md" >}}). +| Temp | Uptime | AQI | Errors | +| ---- | ------- | --- | ------ | +| 15.4 | 1230233 | 3.2 | 5 | -In the example below, we have two queries returning table data. It is visualized as two separate tables before applying the transformation. +## Config from query results -Query A: +This transformation allow you to select one query and from it extract standard options like **Min**, **Max**, **Unit** and **Thresholds** and apply it to other query results. This enables dynamic query driven visualization configuration. -| Time | Job | Uptime | -| ------------------- | ------- | --------- | -| 2020-07-07 11:34:20 | node | 25260122 | -| 2020-07-07 11:24:20 | postgre | 123001233 | +If you want to extract a unique config for every row in the config query result then try the rows to fields transformation. -Query B: +## Convert field type -| Time | Job | Errors | -| ------------------- | ------- | ------ | -| 2020-07-07 11:34:20 | node | 15 | -| 2020-07-07 11:24:20 | postgre | 5 | +This transformation changes the field type of the specified field. -Here is the result after applying the Merge transformation. +- **Field -** Select from available fields +- **as -** Select the FieldType to convert to + - **Numeric -** attempts to make the values numbers + - **String -** will make the values strings + - **Time -** attempts to parse the values as time + - Will show an option to specify a DateFormat as input by a string like yyyy-mm-dd or DD MM YYYY hh:mm:ss + - **Boolean -** will make the values booleans -| Time | Job | Errors | Uptime | -| ------------------- | ------- | ------ | --------- | -| 2020-07-07 11:34:20 | node | 15 | 25260122 | -| 2020-07-07 11:24:20 | postgre | 5 | 123001233 | +For example the following query could be modified by selecting the time field, as Time, and Date Format as YYYY. + +| Time | Mark | Value | +| ---------- | ----- | ----- | +| 2017-07-01 | above | 25 | +| 2018-08-02 | below | 22 | +| 2019-09-02 | below | 29 | +| 2020-10-04 | above | 22 | + +The result: + +| Time | Mark | Value | +| ------------------- | ----- | ----- | +| 2017-01-01 00:00:00 | above | 25 | +| 2018-01-01 00:00:00 | below | 22 | +| 2019-01-01 00:00:00 | below | 29 | +| 2020-01-01 00:00:00 | above | 22 | ## Filter data by name @@ -143,272 +120,6 @@ In the example below, the panel has three queries (A, B, C). I removed the B que > **Note:** This transformation is not available for Graphite because this data source does not support correlating returned data with queries. -## Organize fields - -Use this transformation to rename, reorder, or hide fields returned by the query. - -> **Note:** This transformation only works in panels with a single query. If your panel has multiple queries, then you must either apply an Outer join transformation or remove the extra queries. - -Grafana displays a list of fields returned by the query. You can: - -- Change field order by hovering your cursor over a field. The cursor turns into a hand and then you can drag the field to its new place. -- Hide or show a field by clicking the eye icon next to the field name. -- Rename fields by typing a new name in the **Rename ** box. - -In the example below, I hid the value field and renamed Max and Min. - -{{< figure src="/static/img/docs/transformations/organize-fields-stat-example-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} - -## Join by field (outer join) - -Use this transformation to join multiple time series from a result set by field. - -This transformation is especially useful if you want to combine queries so that you can calculate results from the fields. - -In the example below, I have a template query displaying time series data from multiple servers in a table visualization. I can only view the results of one query at a time. - -{{< figure src="/static/img/docs/transformations/join-fields-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} - -I applied a transformation to join the query results using the time field. Now I can run calculations, combine, and organize the results in this new table. - -{{< figure src="/static/img/docs/transformations/join-fields-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} - -## Add field from calculation - -Use this transformation to add a new field calculated from two other fields. Each transformation allows you to add one new field. - -- **Mode -** Select a mode: - - **Reduce row -** Apply selected calculation on each row of selected fields independently. - - **Binary option -** Apply basic math operation(sum, multiply, etc) on values in a single row from two selected fields. -- **Field name -** Select the names of fields you want to use in the calculation for the new field. -- **Calculation -** If you select **Reduce row** mode, then the **Calculation** field appears. Click in the field to see a list of calculation choices you can use to create the new field. For information about available calculations, refer to the [Calculation list]({{< relref "../calculations-list.md" >}}). -- **Operation -** If you select **Binary option** mode, then the **Operation** fields appear. These fields allow you to do basic math operations on values in a single row from two selected fields. You can also use numerical values for binary operations. -- **Alias -** (Optional) Enter the name of your new field. If you leave this blank, then the field will be named to match the calculation. -- **Replace all fields -** (Optional) Select this option if you want to hide all other fields and display only your calculated field in the visualization. - -In the example below, I added two fields together and named them Sum. - -{{< figure src="/static/img/docs/transformations/add-field-from-calc-stat-example-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} - -## Labels to fields - -This transformation changes time series results that include labels or tags into to a table structure where each label keys and values -are included in the table result. The labels can be displayed either as columns or as row values. - -Given a query result of two time series: - -- Series 1: labels Server=Server A, Datacenter=EU -- Series 2: labels Server=Server B, Datacenter=EU - -In "Columns" mode, the result looks like this: - -| Time | Server | Datacenter | Value | -| ------------------- | -------- | ---------- | ----- | -| 2020-07-07 11:34:20 | Server A | EU | 1 | -| 2020-07-07 11:34:20 | Server B | EU | 2 | - -In "Rows" mode, the result has a table for each series and show each label value like this: - -| label | value | -| ---------- | -------- | -| Server | Server A | -| Datacenter | EU | - -| label | value | -| ---------- | -------- | -| Server | Server B | -| Datacenter | EU | - -### Value field name - -If you selected Server as the **Value field name**, then you would get one field for every value of the Server label. - -| Time | Datacenter | Server A | Server B | -| ------------------- | ---------- | -------- | -------- | -| 2020-07-07 11:34:20 | EU | 1 | 2 | - -### Merging behavior - -The labels to fields transformer is internally two separate transformations. The first acts on single series and extracts labels to fields. The second is the [merge](#merge) transformation that joins all the results into a single table. The merge transformation tries to join on all matching fields. This merge step is required and cannot be turned off. - -To illustrate this, here is an example where you have two queries that return time series with no overlapping labels. - -- Series 1: labels Server=ServerA -- Series 2: labels Datacenter=EU - -This will first result in these two tables: - -| Time | Server | Value | -| ------------------- | ------- | ----- | -| 2020-07-07 11:34:20 | ServerA | 10 | - -| Time | Datacenter | Value | -| ------------------- | ---------- | ----- | -| 2020-07-07 11:34:20 | EU | 20 | - -After merge: - -| Time | Server | Value | Datacenter | -| ------------------- | ------- | ----- | ---------- | -| 2020-07-07 11:34:20 | ServerA | 10 | | -| 2020-07-07 11:34:20 | | 20 | EU | - -## Sort by - -This transformation will sort each frame by the configured field, When `reverse` is checked, the values will return in -the opposite order. - -## Group by - -This transformation groups the data by a specified field (column) value and processes calculations on each group. Click to see a list of calculation choices. For information about available calculations, refer to the [List of calculations]({{< relref "../calculations-list.md" >}}). - -Here's an example of original data. - -| Time | Server ID | CPU Temperature | Server Status | -| ------------------- | --------- | --------------- | ------------- | -| 2020-07-07 11:34:20 | server 1 | 80 | Shutdown | -| 2020-07-07 11:34:20 | server 3 | 62 | OK | -| 2020-07-07 10:32:20 | server 2 | 90 | Overload | -| 2020-07-07 10:31:22 | server 3 | 55 | OK | -| 2020-07-07 09:30:57 | server 3 | 62 | Rebooting | -| 2020-07-07 09:30:05 | server 2 | 88 | OK | -| 2020-07-07 09:28:06 | server 1 | 80 | OK | -| 2020-07-07 09:25:05 | server 2 | 88 | OK | -| 2020-07-07 09:23:07 | server 1 | 86 | OK | - -This transformation goes in two steps. First you specify one or multiple fields to group the data by. This will group all the same values of those fields together, as if you sorted them. For instance if we group by the Server ID field, then it would group the data this way: - -| Time | Server ID | CPU Temperature | Server Status | -| ------------------- | -------------- | --------------- | ------------- | -| 2020-07-07 11:34:20 | **server 1** | 80 | Shutdown | -| 2020-07-07 09:28:06 | **server 1** | 80 | OK | -| 2020-07-07 09:23:07 | **server 1** | 86 | OK | -| 2020-07-07 10:32:20 | server 2 | 90 | Overload | -| 2020-07-07 09:30:05 | server 2 | 88 | OK | -| 2020-07-07 09:25:05 | server 2 | 88 | OK | -| 2020-07-07 11:34:20 | **_server 3_** | 62 | OK | -| 2020-07-07 10:31:22 | **_server 3_** | 55 | OK | -| 2020-07-07 09:30:57 | **_server 3_** | 62 | Rebooting | - -All rows with the same value of Server ID are grouped together. - -After choosing which field you want to group your data by, you can add various calculations on the other fields, and apply the calculation to each group of rows. For instance, we could want to calculate the average CPU temperature for each of those servers. So we can add the _mean_ calculation applied on the CPU Temperature field to get the following: - -| Server ID | CPU Temperature (mean) | -| --------- | ---------------------- | -| server 1 | 82 | -| server 2 | 88.6 | -| server 3 | 59.6 | - -And we can add more than one calculation. For instance: - -- For field Time, we can calculate the _Last_ value, to know when the last data point was received for each server -- For field Server Status, we can calculate the _Last_ value to know what is the last state value for each server -- For field Temperature, we can also calculate the _Last_ value to know what is the latest monitored temperature for each server - -We would then get : - -| Server ID | CPU Temperature (mean) | CPU Temperature (last) | Time (last) | Server Status (last) | -| --------- | ---------------------- | ---------------------- | ------------------- | -------------------- | -| server 1 | 82 | 80 | 2020-07-07 11:34:20 | Shutdown | -| server 2 | 88.6 | 90 | 2020-07-07 10:32:20 | Overload | -| server 3 | 59.6 | 62 | 2020-07-07 11:34:20 | OK | - -This transformation allows you to extract some key information out of your time series and display them in a convenient way. - -## Concatenate fields - -This transformation combines all fields from all frames into one result. Consider: - -Query A: - -| Temp | Uptime | -| ---- | ------- | -| 15.4 | 1230233 | - -Query B: - -| AQI | Errors | -| --- | ------ | -| 3.2 | 5 | - -After you concatenate the fields, the data frame would be: - -| Temp | Uptime | AQI | Errors | -| ---- | ------- | --- | ------ | -| 15.4 | 1230233 | 3.2 | 5 | - -## Convert field type - -This transformation changes the field type of the specified field. - -- **Field -** Select from available fields -- **as -** Select the FieldType to convert to - - **Numeric -** attempts to make the values numbers - - **String -** will make the values strings - - **Time -** attempts to parse the values as time - - Will show an option to specify a DateFormat for the input field like yyyy-mm-dd or DD MM YYYY hh:mm:ss - - **Boolean -** will make the values booleans - - **JSON -** attempts to parse the values as JSON, potentially resulting in complex objects or arrays - -For example the following query could be modified by: - -- selecting the time field as Time, and Input format as YYYY -- selecting the JSON field as JSON - -| Time | Mark | Value | JSON | -| ---------- | ----- | ----- | -------- | -| 2017-07-01 | above | 25 | "[0, 1]" | -| 2018-08-02 | below | 22 | "[2, 3]" | -| 2019-09-02 | below | 29 | "[4, 5]" | -| 2020-10-04 | above | 22 | "[6, 7]" | - -The result: - -| Time | Mark | Value | JSON | -| ------------------- | ----- | ----- | ------ | -| 2017-01-01 00:00:00 | above | 25 | [0, 1] | -| 2018-01-01 00:00:00 | below | 22 | [2, 3] | -| 2019-01-01 00:00:00 | below | 29 | [4, 5] | -| 2020-01-01 00:00:00 | above | 22 | [6, 7] | - -## Series to rows - -> **Note:** This transformation is available in Grafana 7.1+. - -Use this transformation to combine the result from multiple time series data queries into one single result. This is helpful when using the table panel visualization. - -The result from this transformation will contain three columns: Time, Metric, and Value. The Metric column is added so you easily can see from which query the metric originates from. Customize this value by defining Label on the source query. - -In the example below, we have two queries returning time series data. It is visualized as two separate tables before applying the transformation. - -Query A: - -| Time | Temperature | -| ------------------- | ----------- | -| 2020-07-07 11:34:20 | 25 | -| 2020-07-07 10:31:22 | 22 | -| 2020-07-07 09:30:05 | 19 | - -Query B: - -| Time | Humidity | -| ------------------- | -------- | -| 2020-07-07 11:34:20 | 24 | -| 2020-07-07 10:32:20 | 29 | -| 2020-07-07 09:30:57 | 33 | - -Here is the result after applying the Series to rows transformation. - -| Time | Metric | Value | -| ------------------- | ----------- | ----- | -| 2020-07-07 11:34:20 | Temperature | 25 | -| 2020-07-07 11:34:20 | Humidity | 22 | -| 2020-07-07 10:32:20 | Humidity | 29 | -| 2020-07-07 10:31:22 | Temperature | 22 | -| 2020-07-07 09:30:57 | Humidity | 33 | -| 2020-07-07 09:30:05 | Temperature | 19 | - ## Filter data by value This transformation allows you to filter your data directly in Grafana and remove some data points from your query result. You have the option to include or exclude data that match one or more conditions you define. The conditions are applied on a selected field. @@ -470,6 +181,233 @@ In the example above we chose **Match all** because we wanted to include the row Conditions that are invalid or incompletely configured are ignored. +## Group by + +This transformation groups the data by a specified field (column) value and processes calculations on each group. Click to see a list of calculation choices. For information about available calculations, refer to the [List of calculations]({{< relref "./reference-calculation-types.md" >}}). + +Here's an example of original data. + +| Time | Server ID | CPU Temperature | Server Status | +| ------------------- | --------- | --------------- | ------------- | +| 2020-07-07 11:34:20 | server 1 | 80 | Shutdown | +| 2020-07-07 11:34:20 | server 3 | 62 | OK | +| 2020-07-07 10:32:20 | server 2 | 90 | Overload | +| 2020-07-07 10:31:22 | server 3 | 55 | OK | +| 2020-07-07 09:30:57 | server 3 | 62 | Rebooting | +| 2020-07-07 09:30:05 | server 2 | 88 | OK | +| 2020-07-07 09:28:06 | server 1 | 80 | OK | +| 2020-07-07 09:25:05 | server 2 | 88 | OK | +| 2020-07-07 09:23:07 | server 1 | 86 | OK | + +This transformation goes in two steps. First you specify one or multiple fields to group the data by. This will group all the same values of those fields together, as if you sorted them. For instance if we group by the Server ID field, then it would group the data this way: + +| Time | Server ID | CPU Temperature | Server Status | +| ------------------- | -------------- | --------------- | ------------- | +| 2020-07-07 11:34:20 | **server 1** | 80 | Shutdown | +| 2020-07-07 09:28:06 | **server 1** | 80 | OK | +| 2020-07-07 09:23:07 | **server 1** | 86 | OK | +| 2020-07-07 10:32:20 | server 2 | 90 | Overload | +| 2020-07-07 09:30:05 | server 2 | 88 | OK | +| 2020-07-07 09:25:05 | server 2 | 88 | OK | +| 2020-07-07 11:34:20 | **_server 3_** | 62 | OK | +| 2020-07-07 10:31:22 | **_server 3_** | 55 | OK | +| 2020-07-07 09:30:57 | **_server 3_** | 62 | Rebooting | + +All rows with the same value of Server ID are grouped together. + +After choosing which field you want to group your data by, you can add various calculations on the other fields, and apply the calculation to each group of rows. For instance, we could want to calculate the average CPU temperature for each of those servers. So we can add the _mean_ calculation applied on the CPU Temperature field to get the following: + +| Server ID | CPU Temperature (mean) | +| --------- | ---------------------- | +| server 1 | 82 | +| server 2 | 88.6 | +| server 3 | 59.6 | + +And we can add more than one calculation. For instance: + +- For field Time, we can calculate the _Last_ value, to know when the last data point was received for each server +- For field Server Status, we can calculate the _Last_ value to know what is the last state value for each server +- For field Temperature, we can also calculate the _Last_ value to know what is the latest monitored temperature for each server + +We would then get : + +| Server ID | CPU Temperature (mean) | CPU Temperature (last) | Time (last) | Server Status (last) | +| --------- | ---------------------- | ---------------------- | ------------------- | -------------------- | +| server 1 | 82 | 80 | 2020-07-07 11:34:20 | Shutdown | +| server 2 | 88.6 | 90 | 2020-07-07 10:32:20 | Overload | +| server 3 | 59.6 | 62 | 2020-07-07 11:34:20 | OK | + +This transformation allows you to extract some key information out of your time series and display them in a convenient way. + +## Join by field (outer join) + +Use this transformation to join multiple time series from a result set by field. + +This transformation is especially useful if you want to combine queries so that you can calculate results from the fields. + +In the example below, I have a template query displaying time series data from multiple servers in a table visualization. I can only view the results of one query at a time. + +{{< figure src="/static/img/docs/transformations/join-fields-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} + +I applied a transformation to join the query results using the time field. Now I can run calculations, combine, and organize the results in this new table. + +{{< figure src="/static/img/docs/transformations/join-fields-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} + +## Labels to fields + +This transformation changes time series results that include labels or tags into a table where each label keys and values are included in the table result. The labels can be displayed either as columns or as row values. + +Given a query result of two time series: + +- Series 1: labels Server=Server A, Datacenter=EU +- Series 2: labels Server=Server B, Datacenter=EU + +In "Columns" mode, the result looks like this: + +| Time | Server | Datacenter | Value | +| ------------------- | -------- | ---------- | ----- | +| 2020-07-07 11:34:20 | Server A | EU | 1 | +| 2020-07-07 11:34:20 | Server B | EU | 2 | + +In "Rows" mode, the result has a table for each series and show each label value like this: + +| label | value | +| ---------- | -------- | +| Server | Server A | +| Datacenter | EU | + +| label | value | +| ---------- | -------- | +| Server | Server B | +| Datacenter | EU | + +## Merge + +Use this transformation to combine the result from multiple queries into one single result. This is helpful when using the table panel visualization. Values that can be merged are combined into the same row. Values are mergeable if the shared fields contain the same data. For information, refer to [Table panel]({{< relref "../visualizations/table/_index.md" >}}). + +In the example below, we have two queries returning table data. It is visualized as two separate tables before applying the transformation. + +Query A: + +| Time | Job | Uptime | +| ------------------- | ------- | --------- | +| 2020-07-07 11:34:20 | node | 25260122 | +| 2020-07-07 11:24:20 | postgre | 123001233 | + +Query B: + +| Time | Job | Errors | +| ------------------- | ------- | ------ | +| 2020-07-07 11:34:20 | node | 15 | +| 2020-07-07 11:24:20 | postgre | 5 | + +Here is the result after applying the Merge transformation. + +| Time | Job | Errors | Uptime | +| ------------------- | ------- | ------ | --------- | +| 2020-07-07 11:34:20 | node | 15 | 25260122 | +| 2020-07-07 11:24:20 | postgre | 5 | 123001233 | + +## Organize fields + +Use this transformation to rename, reorder, or hide fields returned by the query. + +> **Note:** This transformation only works in panels with a single query. If your panel has multiple queries, then you must either apply an Outer join transformation or remove the extra queries. + +Grafana displays a list of fields returned by the query. You can: + +- Change field order by hovering your cursor over a field. The cursor turns into a hand and then you can drag the field to its new place. +- Hide or show a field by clicking the eye icon next to the field name. +- Rename fields by typing a new name in the **Rename ** box. + +In the example below, I hid the value field and renamed Max and Min. + +{{< figure src="/static/img/docs/transformations/organize-fields-stat-example-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} + +### Value field name + +If you selected Server as the **Value field name**, then you would get one field for every value of the Server label. + +| Time | Datacenter | Server A | Server B | +| ------------------- | ---------- | -------- | -------- | +| 2020-07-07 11:34:20 | EU | 1 | 2 | + +### Merging behavior + +The labels to fields transformer is internally two separate transformations. The first acts on single series and extracts labels to fields. The second is the [merge](#merge) transformation that joins all the results into a single table. The merge transformation tries to join on all matching fields. This merge step is required and cannot be turned off. + +To illustrate this, here is an example where you have two queries that return time series with no overlapping labels. + +- Series 1: labels Server=ServerA +- Series 2: labels Datacenter=EU + +This will first result in these two tables: + +| Time | Server | Value | +| ------------------- | ------- | ----- | +| 2020-07-07 11:34:20 | ServerA | 10 | + +| Time | Datacenter | Value | +| ------------------- | ---------- | ----- | +| 2020-07-07 11:34:20 | EU | 20 | + +After merge: + +| Time | Server | Value | Datacenter | +| ------------------- | ------- | ----- | ---------- | +| 2020-07-07 11:34:20 | ServerA | 10 | | +| 2020-07-07 11:34:20 | | 20 | EU | + +## Reduce + +The _Reduce_ transformation applies a calculation to each field in the frame and return a single value. Time fields are removed when applying this transformation. + +Consider the input: + +Query A: + +| Time | Temp | Uptime | +| ------------------- | ---- | ------- | +| 2020-07-07 11:34:20 | 12.3 | 256122 | +| 2020-07-07 11:24:20 | 15.4 | 1230233 | + +Query B: + +| Time | AQI | Errors | +| ------------------- | --- | ------ | +| 2020-07-07 11:34:20 | 6.5 | 15 | +| 2020-07-07 11:24:20 | 3.2 | 5 | + +The reduce transformer has two modes: + +- **Series to rows -** Creates a row for each field and a column for each calculation. +- **Reduce fields -** Keeps the existing frame structure, but collapses each field into a single value. + +For example, if you used the **First** and **Last** calculation with a **Series to rows** transformation, then +the result would be: + +| Field | First | Last | +| ------ | ------ | ------- | +| Temp | 12.3 | 15.4 | +| Uptime | 256122 | 1230233 | +| AQI | 6.5 | 3.2 | +| Errors | 15 | 5 | + +The **Reduce fields** with the **Last** calculation, +results in two frames, each with one row: + +Query A: + +| Temp | Uptime | +| ---- | ------- | +| 15.4 | 1230233 | + +Query B: + +| AQI | Errors | +| --- | ------ | +| 3.2 | 5 | + ## Rename by regex Use this transformation to rename parts of the query results using a regular expression and replacement pattern. @@ -484,14 +422,116 @@ With the transformation applied, you can see we are left with just the remainder {{< figure src="/static/img/docs/transformations/rename-by-regex-after-7-3.png" class="docs-image--no-shadow" max-width= "1100px" >}} +## Rows to fields + +The rows to fields transformation converts rows into separate fields. This can be useful as fields can be styled and configured individually. It can also use additional fields as sources for dynamic field configuration or map them to field labels. The additional labels can then be used to define better display names for the resulting fields. + +This transformation includes a field table which lists all fields in the data returned by the config query. This table gives you control over what field should be mapped to each config property (the \*Use as\*\* option). You can also choose which value to select if there are multiple rows in the returned data. + +This transformation requires: + +- One field to use as the source of field names. + + By default, the transform uses the first string field as the source. You can override this default setting by selecting **Field name** in the **Use as** column for the field you want to use instead. + +- One field to use as the source of values. + + By default, the transform uses the first number field as the source. But you can override this default setting by selecting **Field value** in the **Use as** column for the field you want to use instead. + +Useful when visualizing data in: + +- Gauge +- Stat +- Pie chart + +### Map extra fields to labels + +If a field does not map to config property Grafana will automatically use it as source for a label on the output field- + +Example: + +| Name | DataCenter | Value | +| ------- | ---------- | ----- | +| ServerA | US | 100 | +| ServerB | EU | 200 | + +Output: + +| ServerA (labels: DataCenter: US) | ServerB (labels: DataCenter: EU) | +| -------------------------------- | -------------------------------- | +| 10 | 20 | + +The extra labels can now be used in the field display name provide more complete field names. + +If you want to extract config from one query and appply it to another you should use the config from query results transformation. + +### Example + +Input: + +| Name | Value | Max | +| ------- | ----- | --- | +| ServerA | 10 | 100 | +| ServerB | 20 | 200 | +| ServerC | 30 | 300 | + +Output: + +| ServerA (config: max=100) | ServerB (config: max=200) | ServerC (config: max=300) | +| ------------------------- | ------------------------- | ------------------------- | +| 10 | 20 | 30 | + +As you can see each row in the source data becomes a separate field. Each field now also has a max config option set. Options like **Min**, **Max**, **Unit** and **Thresholds** are all part of field configuration and if set like this will be used by the visualization instead of any options manually configured in the panel editor options pane. + ## Prepare time series > **Note:** This transformation is available in Grafana 7.5.10+ and Grafana 8.0.6+. -Prepare time series transformation is useful when a data source returns time series data in a format that isn't supported by the panel you want to use. [Read more about the different data frame formats here]({{< relref "../../developers/plugins/data-frames.md" >}}). +Prepare time series transformation is useful when a data source returns time series data in a format that isn't supported by the panel you want to use. For more information about data frame formats, refer to [Data frames]({{< relref "../developers/plugins/data-frames.md" >}}). This transformation helps you resolve this issue by converting the time series data from either the wide format to the long format or the other way around. Select the `Multi-frame time series` option to transform the time series data frame from the wide to the long format. Select the `Wide time series` option to transform the time series data frame from the long to the wide format. + +## Series to rows + +> **Note:** This transformation is available in Grafana 7.1+. + +Use this transformation to combine the result from multiple time series data queries into one single result. This is helpful when using the table panel visualization. + +The result from this transformation will contain three columns: Time, Metric, and Value. The Metric column is added so you easily can see from which query the metric originates from. Customize this value by defining Label on the source query. + +In the example below, we have two queries returning time series data. It is visualized as two separate tables before applying the transformation. + +Query A: + +| Time | Temperature | +| ------------------- | ----------- | +| 2020-07-07 11:34:20 | 25 | +| 2020-07-07 10:31:22 | 22 | +| 2020-07-07 09:30:05 | 19 | + +Query B: + +| Time | Humidity | +| ------------------- | -------- | +| 2020-07-07 11:34:20 | 24 | +| 2020-07-07 10:32:20 | 29 | +| 2020-07-07 09:30:57 | 33 | + +Here is the result after applying the Series to rows transformation. + +| Time | Metric | Value | +| ------------------- | ----------- | ----- | +| 2020-07-07 11:34:20 | Temperature | 25 | +| 2020-07-07 11:34:20 | Humidity | 22 | +| 2020-07-07 10:32:20 | Humidity | 29 | +| 2020-07-07 10:31:22 | Temperature | 22 | +| 2020-07-07 09:30:57 | Humidity | 33 | +| 2020-07-07 09:30:05 | Temperature | 19 | + +## Sort by + +This transformation will sort each frame by the configured field, When `reverse` is checked, the values will return in the opposite order. diff --git a/docs/sources/panels/repeat-panels-or-rows.md b/docs/sources/panels/repeat-panels-or-rows.md deleted file mode 100644 index 5385a0c4051..00000000000 --- a/docs/sources/panels/repeat-panels-or-rows.md +++ /dev/null @@ -1,51 +0,0 @@ -+++ -title = "Repeat panels or rows" -keywords = ["grafana", "templating", "documentation", "guide", "template", "variable", "repeat"] -aliases = ["/docs/grafana/latest/variables/repeat-panels-or-rows/"] -weight = 800 -+++ - -# Repeat panels or rows - -{{< docs/shared "panels/repeat-panels-intro.md" >}} - -Template variables can be very useful to dynamically change your queries across a whole dashboard. If you want -Grafana to dynamically create new panels or rows based on what values you have selected, you can use the _Repeat_ feature. - -## Grafana Play examples - -You can see examples in the following dashboards: - -- [Prometheus repeat](https://play.grafana.org/d/000000036/prometheus-repeat) -- [Repeated Rows Dashboard](https://play.grafana.org/d/000000153/repeat-rows) - -## Repeating panels - -If you have a variable with `Multi-value` or `Include all value` options enabled you can choose one panel and have Grafana repeat that panel -for every selected value. You find the _Repeat_ feature under the _General tab_ in panel edit mode. - -The `direction` controls how the panels will be arranged. - -By choosing `horizontal` the panels will be arranged side-by-side. Grafana will automatically adjust the width -of each repeated panel so that the whole row is filled. Currently, you cannot mix other panels on a row with a repeated -panel. - -Set `Max per row` to tell grafana how many panels per row you want at most. It defaults to _4_ if you don't set anything. - -By choosing `vertical` the panels will be arranged from top to bottom in a column. The width of the repeated panels will be the same as of the first panel (the original template) being repeated. - -Only make changes to the first panel (the original template). To have the changes take effect on all panels you need to trigger a dynamic dashboard re-build. -You can do this by either changing the variable value (that is the basis for the repeat) or reload the dashboard. - -> **Note:** Repeating panels require variables to have one or more items selected; you cannot repeat a panel zero times to hide it. - -## Repeating rows - -As seen above with the panels you can also repeat rows if you have variables set with `Multi-value` or -`Include all value` selection option. - -To enable this feature you need to first add a new _Row_ using the _Add Panel_ menu. Then by hovering the row title and -clicking on the cog button, you will access the `Row Options` configuration panel. You can then select the variable -you want to repeat the row for. - -It may be a good idea to use a variable in the row title as well. diff --git a/docs/sources/panels/specify-thresholds/_index.md b/docs/sources/panels/specify-thresholds/_index.md new file mode 100644 index 00000000000..7c879fc2116 --- /dev/null +++ b/docs/sources/panels/specify-thresholds/_index.md @@ -0,0 +1,11 @@ ++++ +title = "Modify visualization text and background colors" +aliases = ["/docs/grafana/next/panels/thresholds/", "/docs/sources/panels/specify-thresholds/"] +weight = 300 ++++ + +# Modify visualization text and background colors + +Use thresholds to set the color of a visualization text and background. + +{{< section >}} diff --git a/docs/sources/panels/specify-thresholds/about-thresholds.md b/docs/sources/panels/specify-thresholds/about-thresholds.md new file mode 100644 index 00000000000..c99ae25c479 --- /dev/null +++ b/docs/sources/panels/specify-thresholds/about-thresholds.md @@ -0,0 +1,26 @@ ++++ +title = "About thresholds" +weight = 10 +aliases = ["/docs/sources/panels/specify-thresholds/about-thresholds/"] ++++ + +# About thresholds + +Thresholds set the color of either the value text or the background based on conditions that you define. + +There are two types of thresholds: + +- **Absolute** thresholds are defined based on a number. For example, 80 on a scale of 1 to 150. +- **Percentage** thresholds are defined relative to minimum or maximum. For example, 80 percent. + +You can apply thresholds to most, but not all, visualizations. + +## Default thresholds + +On visualizations that support it, Grafana sets default threshold values of: + +- 80 = red +- Base = green +- Mode = Absolute + +The **Base** value represents minus infinity. It is generally the “good” color. diff --git a/docs/sources/panels/specify-thresholds/add-a-threshold.md b/docs/sources/panels/specify-thresholds/add-a-threshold.md new file mode 100644 index 00000000000..27ce198f93a --- /dev/null +++ b/docs/sources/panels/specify-thresholds/add-a-threshold.md @@ -0,0 +1,29 @@ ++++ +title = "Add a threshold" +weight = 20 +aliases = ["/docs/sources/panels/specify-thresholds/add-a-threshold/"] ++++ + +# Add a threshold + +You can add as many thresholds to a panel as you want. Grafana automatically sorts thresholds from highest value to lowest. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To add a threshold**: + +1. Edit the panel to which you want to add a threshold. +1. On the panel display options, locate the **Thresholds** section. +1. Click **+ Add threshold**. + + Grafana adds a threshold value and color. + +1. Accept the recommendations or edit the threshold. + - **Edit color:** To select a color, click the color dot. + - **Edit number:** To change the threshold value, click in the field and enter a number. +1. Select a **Threshold mode**. + Threshold mode applies to all thresholds on this panel. +1. In the **Show thresholds** drop-down, select a threshold display option. +1. Click **Save**. diff --git a/docs/sources/panels/specify-thresholds/add-threshold-to-graph.md b/docs/sources/panels/specify-thresholds/add-threshold-to-graph.md new file mode 100644 index 00000000000..edc6f2044a0 --- /dev/null +++ b/docs/sources/panels/specify-thresholds/add-threshold-to-graph.md @@ -0,0 +1,32 @@ ++++ +title = "Add a threshold to a legacy graph panel" +weight = 40 +aliases = ["/docs/sources/panels/specify-thresholds/add-threshold-to-graph/"] ++++ + +# Add a threshold to a legacy graph panel + +In the Graph panel visualization, thresholds enable you to add lines or sections to a graph to make it easier to recognize when the graph crosses a threshold. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To add a threshold to a graph panel**: + +1. Navigate to the graph panel to which you want to add a threshold. +1. On the **Panel** tab, click **Thresholds**. +1. Click **Add threshold**. +1. Complete the following fields: + - **T1 -** Both values are required to display a threshold. + - **lt** or **gt** - Select **lt** for less than or **gt** for greater than to indicate what the threshold applies to. + - **Value -** Enter a threshold value. Grafana draws a threshold line along the Y-axis at that value. + - **Color -** Choose a condition that corresponds to a color, or define your own color. + - **custom -** You define the fill color and line color. + - **critical -** Fill and line color are red. + - **warning -** Fill and line color are yellow. + - **ok -** Fill and line color are green. + - **Fill -** Controls whether the threshold fill is displayed. + - **Line -** Controls whether the threshold line is displayed. + - **Y-Axis -** Choose **left** or **right**. +1. Click **Save** to save the changes in the dashboard. diff --git a/docs/sources/panels/specify-thresholds/delete-a-threshold.md b/docs/sources/panels/specify-thresholds/delete-a-threshold.md new file mode 100644 index 00000000000..c673e211fc6 --- /dev/null +++ b/docs/sources/panels/specify-thresholds/delete-a-threshold.md @@ -0,0 +1,20 @@ ++++ +title = "Delete a threshold" +weight = 30 +aliases = ["/docs/sources/panels/specify-thresholds/delete-a-threshold/"] ++++ + +# Delete a threshold + +Delete a threshold when it is no longer relevant to your business operations. When you delete a threshold, the system removes the threshold from all visualizations that include the threshold. + +## Before you begin + +- [Add a threshold]({{< relref "./add-a-threshold.md" >}}). + +**To delete a threshold**: + +1. Navigate to the panel to which you want to add a threshold. +1. Click the **Field** tab. (Or **Panel** tab for a graph panel.) +1. Click the trash can icon next to the threshold you want to remove. +1. Click **Save** to save the changes in the dashboard. diff --git a/docs/sources/panels/thresholds.md b/docs/sources/panels/thresholds.md deleted file mode 100644 index 09b17ebd2ed..00000000000 --- a/docs/sources/panels/thresholds.md +++ /dev/null @@ -1,67 +0,0 @@ -+++ -title = "Thresholds" -weight = 440 -+++ - -# Thresholds - -Thresholds set the color of either the value text or the background depending on conditions that you define. - -You can define thresholds one of two ways: - -- **Absolute** thresholds are defined based on a number. For example, 80 on a scale of 1 to 150. -- **Percentage** thresholds are defined relative to minimum or maximum. For example, 80 percent. - -You can apply thresholds to most, but not all, visualizations. - -## Default thresholds - -On visualizations that support it, Grafana sets default threshold values of: - -- 80 = red -- Base = green -- Mode = Absolute - -The **Base** value represents minus infinity. It is generally the “good” color. - -## Add a threshold - -You can add as many thresholds to a panel as you want. Grafana automatically sorts thresholds from highest value to lowest. - -1. Navigate to the panel you want to add a threshold to. -1. Open the category **Thresholds** in the right side options pane. -1. Click **Add threshold**. -1. Grafana adds a threshold with suggested numerical and color values. -1. Accept the recommendations or edit the new threshold. - - **Edit color:** Click the color dot you wish to change and then select a new color. - - **Edit number:** Click the number you wish to change and then enter a new number. - - **Thresholds mode -** Click the mode to change it for all thresholds on this panel. -1. Click **Save** to save the changes in the dashboard. - -## Add a threshold to a Graph (old) panel - -In the Graph panel visualization, thresholds allow you to add arbitrary lines or sections to the graph to make it easier to see when the graph crosses a particular threshold. - -1. Navigate to the graph panel you want to add a threshold to. -1. On the Panel tab, click **Thresholds**. -1. Click **Add threshold**. -1. Fill in as many fields as you want. Only the **T1** fields are required. - - **T1 -** Both values are required to display a threshold. - - **lt** or **gt** - Select **lt** for less than or **gt** for greater than to indicate what the threshold applies to. - - **Value -** Enter a threshold value. Grafana draws a threshold line along the Y-axis at that value. - - **Color -** Choose a condition that corresponds to a color, or define your own color. - - **custom -** You define the fill color and line color. - - **critical -** Fill and line color are red. - - **warning -** Fill and line color are yellow. - - **ok -** Fill and line color are green. - - **Fill -** Controls whether the threshold fill is displayed. - - **Line -** Controls whether the threshold line is displayed. - - **Y-Axis -** Choose **left** or **right**. -1. Click **Save** to save the changes in the dashboard. - -## Delete a threshold - -1. Navigate to the panel you want to add a threshold to. -1. Click the **Field** tab. (Or **Panel** tab for a graph panel.) -1. Click the trash can icon next to the threshold you want to remove. -1. Click **Save** to save the changes in the dashboard. diff --git a/docs/sources/panels/transform-data/_index.md b/docs/sources/panels/transform-data/_index.md new file mode 100644 index 00000000000..2d1458d0fd9 --- /dev/null +++ b/docs/sources/panels/transform-data/_index.md @@ -0,0 +1,20 @@ ++++ +title = "Transform data" +aliases = ["/docs/grafana/latest/panels/transformations/", "/docs/grafana/latest/panels/transformations/config-from-query/", "/docs/grafana/latest/panels/transformations/rows-to-fields/", "/docs/grafana/latest/panels/transformations/apply-transformations/", "/docs/sources/panels/transform-data/"] +weight = 500 ++++ + +# Transform data + +Transformations are a powerful way to manipulate data returned by a query before the system applies a visualization. + +With transformations, you can: + +- Rename fields +- Join time series data +- Perform mathematical operations across queries +- Use the output of one transformation as the input to another transformation + +This section includes the following topics: + +{{< section >}} diff --git a/docs/sources/panels/transform-data/about-transformation.md b/docs/sources/panels/transform-data/about-transformation.md new file mode 100644 index 00000000000..d357e48a19d --- /dev/null +++ b/docs/sources/panels/transform-data/about-transformation.md @@ -0,0 +1,31 @@ ++++ +title = "About transformations" +weight = 10 +aliases = ["/docs/sources/panels/transform-data/about-transformation/"] ++++ + +# About transformations + +Transformations process the result set of a query before it’s passed on for visualization. Using transformations, you can: + +- Rename fields +- Join time series data +- Perform mathematical operations across queries +- Use the output of one transformation as the input to another transformation + +For users that rely on multiple views of the same dataset, transformations offer an efficient method of creating and maintaining numerous dashboards. + +You can also use the output of one transformation as the input to another transformation, which results in a performance gain. + +> Sometimes the system cannot graph transformed data. When that happens, click the `Table view` toggle above the visualization to switch to a table view of the data. This can help you understand the final result of your transformations. + +## Transformation types + +Grafana provides a number of ways that you can transform data. For a complete list of transformations, refer to +[Reference: Transformation functions]({{< relref "../reference-transformation-functions.md" >}}). + +## Order of transformations + +When there are multiple transformations, Grafana applies them in the order they are listed. Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. + +The order in which Grafana applies transformations directly impacts the results. For example, if you use a Reduce transformation to condense all the results of one column into a single value, then you can only apply transformations to that single value. diff --git a/docs/sources/panels/transform-data/apply-transformation-to-data.md b/docs/sources/panels/transform-data/apply-transformation-to-data.md new file mode 100644 index 00000000000..17588bb18e3 --- /dev/null +++ b/docs/sources/panels/transform-data/apply-transformation-to-data.md @@ -0,0 +1,36 @@ ++++ +title = "Apply a transformation to data" +weight = 20 +aliases = ["/docs/sources/panels/transform-data/apply-transformation-to-data/"] ++++ + +# Apply a transformation function to data + +The following steps guide you in applying a transformation to data. This documentation does not include steps for each type of transformation. For a complete list of transformations, refer to [Reference: Transformation functions]({{< relref "../reference-transformation-functions.md" >}}). + +## Before you begin + +- [Add a query]({{< relref "../query-a-data-source/add-a-query.md" >}}). + +**To apply a transformation function to data**: + +1. Navigate to the panel where you want to add one or more transformations. +1. Click the panel title and then click **Edit**. +1. Click the **Transform** tab. +1. Click a transformation. + + A transformation row appear where you configure the transformation options. For more information about how to configure a transformation, refer to [Reference: Transformation functions]({{< relref "../reference-transformation-functions.md" >}}). + + For information about available calculations, refer to [Reference: Calculations]({{< relref "../reference-calculation-types.md" >}}). + +1. To apply another transformation, click **Add transformation**. + + This transformation acts on the result set returned by the previous transformation. + + {{< figure src="/static/img/docs/transformations/transformations-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} + +## Options + +- **Config query**: Select the query that returns the data you want to use as configuration. +- **Apply to**: Select what fields or series to apply the configuration to. +- **Apply to options**: Usually a field type or field name regex depending on what option you selected in **Apply to**. diff --git a/docs/sources/panels/transform-data/debug-transformation.md b/docs/sources/panels/transform-data/debug-transformation.md new file mode 100644 index 00000000000..95764f97603 --- /dev/null +++ b/docs/sources/panels/transform-data/debug-transformation.md @@ -0,0 +1,13 @@ ++++ +title = "Debug a transformation" +weight = 30 +aliases = ["/docs/sources/panels/transform-data/debug-transformation/"] ++++ + +# Debug a transformation + +To see the input and the output result sets of the transformation, click the bug icon on the right side of the transformation row. + +The input and output results sets can help you debug a transformation. + +{{< figure src="/static/img/docs/transformations/debug-transformations-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} diff --git a/docs/sources/panels/transform-data/delete-transformation.md b/docs/sources/panels/transform-data/delete-transformation.md new file mode 100644 index 00000000000..76f762c780e --- /dev/null +++ b/docs/sources/panels/transform-data/delete-transformation.md @@ -0,0 +1,19 @@ ++++ +title = "Delete a transformation" +weight = 40 +aliases = ["/docs/sources/panels/transform-data/delete-transformation/"] ++++ + +# Delete a transformation + +We recommend that you remove transformations that you don't need. When you delete a transformation, you remove the data from the visualization. + +## Before you begin + +- Identify all dashboards that rely on the transformation and inform impacted dashboard users. + +**To delete a transformation**: + +1. Open a panel for editing. +1. Click the **Transform** tab. +1. Click the trash icon next to the transformation you want to delete. diff --git a/docs/sources/panels/transformations/_index.md b/docs/sources/panels/transformations/_index.md deleted file mode 100644 index 73bea335b8a..00000000000 --- a/docs/sources/panels/transformations/_index.md +++ /dev/null @@ -1,45 +0,0 @@ -+++ -title = "Transformations" -weight = 350 -+++ - -# Transformations - -Transformations process the result set of a query before it’s passed on for visualization. They allow you to rename fields, join separate time series together, do math across queries, and more. For users, with numerous dashboards or with a large volume of queries, the ability to reuse the query result from one panel in another panel can be a huge performance gain. - -The transformations feature is accessible from the **Transform** tab of the Grafana panel editor. - -Transformations sometimes result in data that cannot be graphed. When that happens, click the `Table view` toggle above the visualization to switch to a table view of the data. This can help you understand -the final result of your transformations. - -## Order of transformations - -In case there are multiple transformations, Grafana applies them in the exact sequence in which they are listed. Each transformation creates a new result set that is passed onto the next transformation in the pipeline for processing. - -The order in which transformations are applied can make a huge difference in how your results look. For example, if you use a Reduce transformation to condense all the results of one column into a single value, then you can only apply transformations to that single value. - -## Prerequisites - -Before you can configure and apply transformations: - -- You must have entered a query that return data. For more information on queries, refer to [Queries]({{< relref "../queries.md" >}}). - -## List of transformations - -- [Add field from calculation]({{< relref "./types-options.md#add-field-from-calculation" >}}) -- [Concatenate fields]({{< relref "./types-options.md#concatenate-fields" >}}) -- [Config from query results]({{< relref "./config-from-query.md" >}}) -- [Convert field type]({{< relref "./types-options.md#convert-field-type" >}}) -- [Filter data by name]({{< relref "./types-options.md#filter-data-by-name" >}}) -- [Filter data by query]({{< relref "./types-options.md#filter-data-by-query" >}}) -- [Filter data by value]({{< relref "./types-options.md#filter-data-by-value" >}}) -- [Group by]({{< relref "./types-options.md#group-by" >}}) -- [Labels to fields]({{< relref "./types-options.md#labels-to-fields" >}}) -- [Merge]({{< relref "./types-options.md#merge" >}}) -- [Organize fields]({{< relref "./types-options.md#organize-fields" >}}) -- [Outer join]({{< relref "./types-options.md#join-by-field-outer-join" >}}) -- [Reduce]({{< relref "./types-options.md#reduce" >}}) -- [Rename by regex]({{< relref "./types-options.md#rename-by-regex" >}}) -- [Rows to fields]({{< relref "./rows-to-fields" >}}) -- [Series to rows]({{< relref "./types-options.md#series-to-rows" >}}) -- [Sort by]({{< relref "./types-options.md#sort-by" >}}) diff --git a/docs/sources/panels/transformations/apply-transformations.md b/docs/sources/panels/transformations/apply-transformations.md deleted file mode 100644 index 50b21bdd877..00000000000 --- a/docs/sources/panels/transformations/apply-transformations.md +++ /dev/null @@ -1,33 +0,0 @@ -+++ -title = "Apply a transformation" -weight = 1 -+++ - -# Apply transformations - -You can apply transformations from the Transform tab of the Grafana panel editor, which is located next to the Queries tab. See also, [Debug transformations](#debug-transformations). - -To apply a transformation: - -1. Navigate to the panel where you want to add one or more transformations. -1. Click the panel title and then click **Edit**. -1. Click the **Transform** tab. -1. Click a transformation to select it. - - A transformation row displays. You can configure the transformation options here. For more information, refer to [Transformation types and options]({{< relref "types-options.md" >}}). - -1. Click **Add transformation** to apply another transformation. - - This next transformation acts on the result set returned by the previous transformation. - - {{< figure src="/static/img/docs/transformations/transformations-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} - -## Delete a transformation - -To remove a transformation that is no longer needed, click the trash can icon. - -## Debug transformations - -To see the input and the output result sets of the transformation, click the bug icon on the right side of the transformation row -Grafana displays the transformation debug view below the transformation row. -{{< figure src="/static/img/docs/transformations/debug-transformations-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} diff --git a/docs/sources/panels/transformations/config-from-query.md b/docs/sources/panels/transformations/config-from-query.md deleted file mode 100644 index 6f639d45aef..00000000000 --- a/docs/sources/panels/transformations/config-from-query.md +++ /dev/null @@ -1,69 +0,0 @@ -+++ -title = "Config from query results" -weight = 300 -+++ - -# Config from query results transform - -> **Note:** This is a new beta transformation introduced in v8.1. - -This transformation allow you select one query and from it extract [standard options]({{< relref "../standard-options.md" >}}) like **Min**, **Max**, **Unit** and **Thresholds** and apply it to other query results. This enables dynamic query driven visualization configuration. - -If you want to extract a unique config for every row in the config query result then try the [Rows to fields]({{< relref "./rows-to-fields" >}}) transformation instead. - -### Options - -- **Config query**: Select the query that returns the data you want to use as configuration. -- **Apply to**: Select what fields or series to apply the configuration to. -- **Apply to options**: Usually a field type or field name regex depending on what option you selected in **Apply to**. - -### Field mapping table - -This transformation includes a field table which lists all fields in the data returned by the config query. This table gives you control over what field should be mapped to each config property (the \*Use as\*\* option). You can also choose which value to select if there are multiple rows in the returned data. - -## Example - -Input[0] (From query: A, name: ServerA) - -| Time | Value | -| ------------- | ----- | -| 1626178119127 | 10 | -| 1626178119129 | 30 | - -Input[1] (From query: B) - -| Time | Value | -| ------------- | ----- | -| 1626178119127 | 100 | -| 1626178119129 | 100 | - -Output (Same as Input[0] but now with config on the Value field) - -| Time | Value (config: Max=100) | -| ------------- | ----------------------- | -| 1626178119127 | 10 | -| 1626178119129 | 30 | - -As you can see each row in the source data becomes a separate field. Each field now also has a max config option set. Options like min, max, unit and thresholds are all part of field configuration and if set like this will be used by the visualization instead of any options manually configured in the panel editor options pane. - -## Value mappings - -You can also transform a query result into value mappings. This is is a bit different as here every row in the config query result will be used to define a single value mapping row. See example below. - -Config query result: - -| Value | Text | Color | -| ----- | ------ | ----- | -| L | Low | blue | -| M | Medium | green | -| H | High | red | - -In the field mapping specify: - -| Field | Use as | Select | -| ----- | ----------------------- | ---------- | -| Value | Value mappings / Value | All values | -| Text | Value mappings / Text | All values | -| Color | Value mappings / Ciolor | All values | - -Grafana will build the value mappings from you query result and apply it the the real data query results. You should see values being mapped and colored according to the config query results. diff --git a/docs/sources/panels/transformations/rows-to-fields.md b/docs/sources/panels/transformations/rows-to-fields.md deleted file mode 100644 index b9536ef9687..00000000000 --- a/docs/sources/panels/transformations/rows-to-fields.md +++ /dev/null @@ -1,72 +0,0 @@ -+++ -title = "Rows to fields" -weight = 300 -+++ - -# Rows to fields transform - -> **Note:** This is a new beta transformation introduced in v8.1. - -This transforms rows into separate fields. This can be useful as fields can be styled and configured individually, something rows cannot. It can also use additional fields as sources for dynamic field configuration or map them to field labels. The additional labels can then be used to define better display names for the resulting fields. - -Useful when visualizing data in: - -- Gauge -- Stat -- Pie chart - -If you want to extract config from one query and appply it to another you should use the [Config from query results]({{< relref "./config-from-query.md" >}}) transformation instead. - -## Example - -Input: - -| Name | Value | Max | -| ------- | ----- | --- | -| ServerA | 10 | 100 | -| ServerB | 20 | 200 | -| ServerC | 30 | 300 | - -Output: - -| ServerA (config: max=100) | ServerB (config: max=200) | ServerC (config: max=300) | -| ------------------------- | ------------------------- | ------------------------- | -| 10 | 20 | 30 | - -As you can see each row in the source data becomes a separate field. Each field now also has a max config option set. Options like **Min**, **Max**, **Unit** and **Thresholds** are all part of field configuration and if set like this will be used by the visualization instead of any options manually configured in the panel editor options pane. - -## Configuration - -To do this transformation Grafana needs to know how to use each field in the input data. The UI options for this transform shows you all fields and **Use as** option to tell Grafana how to use the field. - -### The name field (required) - -This transformation needs one field to use as the source of field names. By default the transform will use the first string field for this. But you can override this default behavior by selecting **Field name** in the **Use as** column for the field you want to use instead. - -### The value field (required) - -This transformation needs one field to use as the source of values. By default the transform will use the first number field for this. But you can override this default behavior by selecting **Field value** in the **Use as** column for the field you want to use instead. - -### Map extra fields to config - -You can map extra fields to configuration like min, max, unit and threshold. If the field name maps directly -to one of these config fields Grafana will handle this mapping automatically. - -### Map extra fields to labels - -If a field does not map to config property Grafana will automatically use it as source for a label on the output field- - -Example: - -| Name | DataCenter | Value | -| ------- | ---------- | ----- | -| ServerA | US | 100 | -| ServerB | EU | 200 | - -Output: - -| ServerA (labels: DataCenter: US) | ServerB (labels: DataCenter: EU) | -| -------------------------------- | -------------------------------- | -| 10 | 20 | - -The extra labels can now be used in the field display name provide more complete field names. diff --git a/docs/sources/panels/value-mappings.md b/docs/sources/panels/value-mappings.md deleted file mode 100644 index 2aa3a5b3a1c..00000000000 --- a/docs/sources/panels/value-mappings.md +++ /dev/null @@ -1,155 +0,0 @@ -+++ -title = "Value mappings" -weight = 450 -+++ - -# Value mappings - -Value mapping concept allow you to replace values or ranges in your visualizations with words or emojis. - -Values mapped via value mappings will skip the unit formatting. This means that a text value mapped to a numerical value will not be formatted using the configured unit. - -![Value mappings example](/static/img/docs/value-mappings/value-mappings-example-8-0.png) - -If value mappings are present in a panel, then Grafana displays a summary in the side pane of the panel editor. - -> **Note:** The new value mappings are not compatible with some visualizations, such as Graph (old), Text, and Heatmap. - -## Value mapping examples - -Value mappings are displayed differently in different visualizations. - -### Time series example - -Here's an example showing a Time series visualization with value mappings. Value mapping colors are not applied to this visualization, but the display text is shown on the axis. - -![Value mappings time series example](/static/img/docs/value-mappings/value-mappings-summary-example-8-0.png) - -### Stat example - -Here's an example showing a Stat visualization with value mappings. You might want to hide the sparkline so it doesn't interfere with the values. Value mapping text colors are applied. - -![Value mappings stat example](/static/img/docs/value-mappings/value-mappings-stat-example-8-0.png) - -### Bar gauge example - -Here's an example showing a Bar gauge visualization with value mappings. The value mapping colors are applied to the text but not the gauges. - -![Value mappings bar gauge example](/static/img/docs/value-mappings/value-mappings-bar-gauge-example-8-0.png) - -### Table example - -Here's an example showing a Table visualization with value mappings. If you want value mapping colors displayed on the table, then set the cell display mode to **Color text** or **Color background**. - -![Value mappings table example](/static/img/docs/value-mappings/value-mappings-table-example-8-0.png) - -## Value mappings fields - -This section describes options in the Value mappings user interface. - -> **Note:** Display text and Color are both optional. If you want to only assign colors to the text values, then you can leave the display text empty and the original value is displayed. - -### Condition - -This column lists the type of condition a value mapping is triggered by and the values. - -You can map values to three different conditions: - -- **Value** maps text values to a color or different display text. For example, if a value is `10`, I want Grafana to display **Perfection!** rather than the number. -- **Range** maps numerical ranges to a display text and color. For example, if a value is within a certain range, I want Grafana to display **Low** or **High** rather than the number. -- **Regex** maps regular expressions to replacement text and a color. For example, if a value is 'www.example.com', I want Grafana to display just **www**, truncating the domain. -- **Special** maps special values like `Null`, `NaN` (not a number), and boolean values like `true` and `false` to a display text and color. For example, if Grafana encounters a `null`, I want Grafana to display **N/A**. - -You can also use the dots on the left as a "handle" to drag and reorder value mappings in the list. - -### Display text - -The _display text_ is what Grafana displays instead of a number when the listed condition is met. - -You can enter any Ascii character or emoji in this field. - -### Color - -You can select a color to for Grafana to display the value mapping text in. - -- **Set color -** Click **Set color** to see a range of recommended colors. Click **Custom** to choose your own color. -- **Text color -** The primary text color for the current theme, i.e. white in dark theme and black in light theme. -- **Transparent -** Makes the color transparent so that the value mapping color shows whatever color is behind it, such as a panel background color. - -![Set color](/static/img/docs/value-mappings/set-color-8-0.png) - -### Copy icon - -Click the copy icon in the value mapping row that you want to copy. - -### Trash icon - -Click the trash can icon to delete a value mapping. Once deleted, you cannot recover it. - -## Map a value - -Create a mapping for a single value. - -![Map a value](/static/img/docs/value-mappings/map-value-8-0.png) - -1. [Open the panel editor]({{< relref "./panel-editor.md#open-the-panel-editor" >}}). -1. In the Value mappings section of the side pane, click **Add value mappings**. -1. Click **Add a new mapping** and then select **Value**. -1. Enter the exact value for Grafana to match. -1. (Optional) Enter display text. -1. (Optional) Set the color. -1. Click **Update** to save the value mapping. - -## Map a range - -Create a mapping for a range of values. - -![Map a range](/static/img/docs/value-mappings/map-range-8-0.png) - -1. [Open the panel editor]({{< relref "./panel-editor.md#open-the-panel-editor" >}}). -1. In the Value mappings section of the side pane, click **Add value mappings**. -1. Click **Add a new mapping** and then select **Range**. -1. Enter the beginning and ending values in the range for Grafana to match. -1. (Optional) Enter display text. -1. (Optional) Set the color. -1. Click **Update** to save the value mapping. - -## Map a regular expression - -Create a mapping based on a regular expression. - -![Map a regular expression](/static/img/docs/value-mappings/map-regex-8-0.png) - -1. [Open the panel editor]({{< relref "./panel-editor.md#open-the-panel-editor" >}}). -1. In the Value mappings section of the side pane, click **Add value mappings**. -1. Click **Add a new mapping** and then select **Regex**. -1. Enter the regular expression pattern for Grafana to match. -1. (Optional) Enter display text, which can include items such as $1 for replacements. -1. (Optional) Set the color. -1. Click **Update** to save the value mapping. - -## Map a special value - -Create a mapping for a special value. - -![Map a value](/static/img/docs/value-mappings/map-special-value-8-0.png) - -1. [Open the panel editor]({{< relref "./panel-editor.md#open-the-panel-editor" >}}). -1. In the Value mappings section of the side pane, click **Add value mappings**. -1. Click **Add a new mapping** and then select **Special**. -1. Select the special value for Grafana to match. Options include: - - Null - - NaN (Not a Number) - - Null + NaN - - True - - False - - Empty -1. (Optional) Enter display text. -1. (Optional) Set the color. -1. Click **Update** to save the value mapping. - -## Edit a value mapping - -1. [Open the panel editor]({{< relref "./panel-editor.md#open-the-panel-editor" >}}). -1. In the Value mappings section of the side pane, click **Edit value mappings**. -1. Make any necessary changes in the fields and then click **Update**. diff --git a/docs/sources/panels/working-with-panels/_index.md b/docs/sources/panels/working-with-panels/_index.md new file mode 100644 index 00000000000..7d1e8b19d48 --- /dev/null +++ b/docs/sources/panels/working-with-panels/_index.md @@ -0,0 +1,13 @@ ++++ +title = "Working with Grafana panels" +aliases = ["/docs/grafana/latest/panels/panel-editor/", "/docs/grafana/latest/panels/add-a-panel/", "/docs/grafana/latest/panels/standard-options/", "/docs/grafana/latest/panels/panel-options/", "/docs/grafana/latest/panels/legend-options/", "/docs/sources/panels/working-with-panels/"] +weight = 100 ++++ + +# Working with Grafana panels + +A Grafana panel is the user interface you use to define a data source query, and transform and format data that appears in visualizations. + +A panel editor includes a query builder and a series of options that you can use to transform data and add information to your panels. + +{{< section >}} diff --git a/docs/sources/panels/working-with-panels/add-link-to-panel.md b/docs/sources/panels/working-with-panels/add-link-to-panel.md new file mode 100644 index 00000000000..5439505fb68 --- /dev/null +++ b/docs/sources/panels/working-with-panels/add-link-to-panel.md @@ -0,0 +1,11 @@ ++++ +title = "Add a link to a panel" +aliases = ["/docs/sources/panels/working-with-panels/add-link-to-panel/"] +weight = 60 ++++ + +# Add a link to a panel + +{{< docs/shared "panels/panel-links-intro.md" >}} + +For more information, refer to [Panel links]({{< relref "../../linking/panel-links.md" >}}). diff --git a/docs/sources/panels/working-with-panels/add-panel.md b/docs/sources/panels/working-with-panels/add-panel.md new file mode 100644 index 00000000000..7416c5804bf --- /dev/null +++ b/docs/sources/panels/working-with-panels/add-panel.md @@ -0,0 +1,55 @@ ++++ +title = "Add a panel to a dashboard" +aliases = ["/docs/sources/panels/working-with-panels/add-panel/"] +weight = 20 ++++ + +# Add a panel to a dashboard + +Panels allow you to show your data in visual form. Each panel needs at least one query to display a visualization. + +## Before you begin + +- Ensure that you have the proper [organization role]({{< relref "../../permissions/organization_roles.md" >}}) or [permissions]({{< relref "../../permissions/_index.md" >}}). +- Identify the dashboard to which you want to add the panel. +- Understand the query language of the target data source. +- Ensure that data source for which you are writing a query has been added. For more information about adding a data source, refer to [Add a data source]({{< relref "../../datasources/add-a-data-source.md" >}}) if you need instructions. + +**To add a panel to a dashboard**: + +1. Navigate to the dashboard to which you want to add a panel. +1. Click the **Add panel** icon. + + ![](/static/img/docs/panels/add-panel-icon-7-0.png) + +1. Click **Add an empty panel**. + + Grafana creates an empty time-series panel and selects the default data source. + +1. In the first line of the **Query** tab, click the drop-down list and select a data source. + +1. Write or construct a query in the query language of your data source. + + For more information about data sources, refer to [Data sources]({{< relref "../../datasources/_index.md" >}}) for specific guidelines. + +1. In the Visualization list, select a visualization type. + + Grafana displays a preview of your query results with the visualization applied. + + ![](/static/img/docs/panel-editor/select-visualization-8-0.png) + + For more information about individual visualizations, refer to [Visualizations options]({{< relref "../../visualizations/_index.md" >}}). + +1. Refer to the following documentation for ways you adjust panel settings. + + While not required, most visualizations need some adjustment before they properly display the information that you need. + + - [Format data using value mapping]({{< relref "../format-data/about-value-mapping.md" >}}) + - [Visualization-specific options]({{< relref "../../visualizations/_index.md" >}}) + - [Override field values]({{< relref "../override-field-values/about-field-overrides.md" >}}) + - [Specify thresholds to set the color of visualization text and background]({{< relref "../specify-thresholds/about-thresholds.md" >}}) + - [Apply color to series and fields]({{< relref "./apply-color-to-series.md" >}}) + +1. Add a note to describe the visualization (or describe your changes) and then click **Save** in the upper-right corner of the page. + + Notes can be helpful if you need to revert the dashboard to a previous version. diff --git a/docs/sources/panels/working-with-panels/add-title-and-description.md b/docs/sources/panels/working-with-panels/add-title-and-description.md new file mode 100644 index 00000000000..fb2b428ae17 --- /dev/null +++ b/docs/sources/panels/working-with-panels/add-title-and-description.md @@ -0,0 +1,33 @@ ++++ +title = "Add a title and description to a panel" +aliases = ["/docs/sources/panels/working-with-panels/add-title-and-description/"] +weight = 30 ++++ + +# Add a title and description to a panel + +Add a title and description to a panel to share with users any important information about the visualization. For example, use the description to document the purpose of the visualization. + +## Before you begin: + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To add a title and description to a panel**: + +1. Open a panel. + +1. In the panel display options pane, locate the **Panel options** section. + +1. Enter a **Title**. + + Text entered in this field is displayed at the top of your panel in the panel editor and in the dashboard. + + You can use [variables you have defined]({{< relref "../../variables/_index.md" >}}) in either field, but not [global variables]({{< relref "../../variables/variable-types/global-variables.md" >}}). + +1. Write a description of the panel and the data you are displaying. + + Text entered in this field is displayed in a tooltip in the upper left corner of the panel. + + You can use [variables you have defined]({{< relref "../../variables/_index.md" >}}) in either field, but not [global variables]({{< relref "../../variables/variable-types/global-variables.md" >}}). + + ![](/static/img/docs/panels/panel-options-8-0.png) diff --git a/docs/sources/panels/working-with-panels/apply-color-to-series.md b/docs/sources/panels/working-with-panels/apply-color-to-series.md new file mode 100644 index 00000000000..89aa8052e17 --- /dev/null +++ b/docs/sources/panels/working-with-panels/apply-color-to-series.md @@ -0,0 +1,38 @@ ++++ +title = "Apply color to series and fields" +aliases = ["/docs/sources/panels/working-with-panels/apply-color-to-series/"] +weight = 50 ++++ + +# Apply color to series and fields + +In addition to specifying color based on thresholds, you can configure the color of series and field data. The color options and their effect on the visualization depends on the visualization you are working with. Some visualizations have different color options. + +You can specify a single color, or select a continuous (gradient) color schemes, based on a value. +Continuous color interpolates a color using the percentage of a value relative to min and max. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To apply color to series and fields**: + +1. In panel display options, scroll to the **Standard options** or **override** section. + +1. Click the **Standard options Color scheme** drop-down, and select one of the following palettes: + +
+ +| Color mode | Description | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | --- | +| **Single color** | Specify a single color, useful in an override rule | +| **From thresholds** | Informs Grafana to take the color from the matching threshold | +| **Classic palette** | Grafana will assign color by looking up a color in a palette by series index. Useful for Graphs and pie charts and other categorical data visualizations | +| **Green-Yellow-Red (by value)** | Continuous color scheme | +| **Blue-Yellow-Red (by value)** | Continuous color scheme | +| **Blues (by value)** | Continuous color scheme (panel background to blue) | +| **Reds (by value)** | Continuous color scheme (panel background color to blue) | +| **Greens (by value)** | Continuous color scheme (panel background color to blue) | +| **Purple (by value)** | Continuous color scheme (panel background color to blue) | . | + +{{< figure src="/static/img/docs/v73/color_scheme_dropdown.png" max-width="350px" caption="Color scheme" class="pull-right" >}} diff --git a/docs/sources/panels/working-with-panels/configure-legend.md b/docs/sources/panels/working-with-panels/configure-legend.md new file mode 100644 index 00000000000..6c97260c7ee --- /dev/null +++ b/docs/sources/panels/working-with-panels/configure-legend.md @@ -0,0 +1,92 @@ ++++ +title = "Configure a legend" +aliases = ["/docs/sources/panels/working-with-panels/configure-legend/"] +weight = 70 ++++ + +# Configure a legend + +A panel includes a legend that you can use to interpret data displayed in a visualization. Each legend option adds context and clarity to the data illustrated in a visualization. + +## Isolate series data in a visualization + +Visualizations can often be visually complex, and include many data series. You can simplify the view by removing series data from the visualization, which isolates the data you want to see. Grafana automatically creates a new override in the **Override** tab. + +When you apply your changes, the visualization changes appear to all users of the panel. + +### Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To isolate series data in a visualization**: + +1. Open the panel. + +1. In the legend, click the label of the series you want to isolate. + + The system removes from view all other series data. + +1. To incrementally add series data to an isolated series, press the **Ctrl** or **Command** key and click the label of the series you want to add. + +1. To revert back to the default view that includes all data, click any series label twice. + +1. To save your changes so that they appear to all viewers of the panel, click **Apply**. + +This topic currently applies to the following visualizations: + +- [Bar chart]({{< relref "../../visualizations/bar-chart.md" >}}) +- [Histogram]({{< relref "../../visualizations/histogram.md" >}}) +- [Pie chart]({{< relref "../../visualizations/pie-chart-panel.md" >}}) +- [State timeline]({{< relref "../../visualizations/state-timeline.md" >}}) +- [Status history]({{< relref "../../visualizations/status-history.md" >}}) +- [Time series]({{< relref "../../visualizations/time-series/_index.md" >}}) + +## Add values to a legend + +As way to add more context to a visualization, you can add series data values to a legend. You can add as many values as you'd like; after you apply your changes, you can horizontally scroll the legend to see all values. + +### Before you begin + +- Add a panel + +**To add values to a legend**: + +1. Open a panel. + +1. In the panel display options pane, locate the **Legend** section. + +1. In the **Legend values** field, select the values you want to appear in the legend. + +1. Click **Apply** to save your changes are navigate back to the dashboard. + +![Toggle series visibility](/static/img/docs/legend/legend-series-toggle-7-5.png) + +## Change a series color + +By default, Grafana specifies the color of your series data, which you can change. + +### Before you begin + +- Add a panel + +**To change a series color, perform the following steps**: + +1. Open the panel. + +1. In the legend, click the color bar associated with the series. + +1. Select a pre-set color or a custom color from the color palette. + +1. Click **Apply** to save your changes are navigate back to the dashboard. + +![Change legend series color](/static/img/docs/legend/legend-series-color-7-5.png) + +## Sort series + +Change legend mode to **Table** and choose [calculations]({{< relref "../reference-calculation-types.md" >}}) to be displayed in the legend. Click the calculation name header in the legend table to sort the values in the table in ascending or descending order. + +The sort order affects the positions of the bars in the Bar chart panel as well as the order of stacked series in the Time series and Bar chart panels. + +> **Note:** This feature is only supported in these panels: Bar chart, Histogram, Time series, XY Chart. + +![Sort legend series](/static/img/docs/legend/legend-series-sort-8-3.png). diff --git a/docs/sources/panels/working-with-panels/download-query-results.md b/docs/sources/panels/working-with-panels/download-query-results.md new file mode 100644 index 00000000000..5fccf22d5ba --- /dev/null +++ b/docs/sources/panels/working-with-panels/download-query-results.md @@ -0,0 +1,21 @@ ++++ +title = "Download query results" +aliases = ["/docs/sources/panels/working-with-panels/download-query-results/"] +weight = 110 ++++ + +# Download query results + +Grafana generates a text (.txt) file that contains the results of a query, which you can download. + +## Before you begin + +- [Add a query]({{< relref "../query-a-data-source/add-a-query.md" >}}). + +**To download query results**: + +1. Open the panel inspector. + +1. Inspect the query results. + +1. Click **Download**. diff --git a/docs/sources/panels/working-with-panels/format-standard-fields.md b/docs/sources/panels/working-with-panels/format-standard-fields.md new file mode 100644 index 00000000000..1a70f7115fc --- /dev/null +++ b/docs/sources/panels/working-with-panels/format-standard-fields.md @@ -0,0 +1,31 @@ ++++ +title = "Format a standard field" +aliases = ["/docs/sources/panels/working-with-panels/format-standard-fields/"] +weight = 40 ++++ + +# Format a standard field + +The data model used in Grafana, namely the [data frame]({{< relref "../../developers/plugins/data-frames.md" >}}), is a columnar-oriented table structure that unifies both time series and table query results. Each column within this structure is called a _field_. A field can represent a single time series or table column. + +Field options allow you to change how the data is displayed in your visualizations. Options and overrides that you apply do not change the data, they change how Grafana displays the data. When you change an option, it is applied to all fields, meaning all series or columns. For example, if you change the unit to percentage, then all fields with numeric values are displayed in percentages. + +For a complete list of field formatting options, refer to [Reference: Standard field definitions]({{< relref "../reference-standard-field-definitions.md" >}}). + +> You can apply standard options to most built-in Grafana panels. Some older panels and community panels that have not updated to the new panel and data model will be missing either all or some of these field options. + +## Before you begin + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To format a standard field**: + +1. Open a dashboard, click the panel title, and click **Edit**. + +1. In the panel display options pane, locate the **Standard options** section. + +1. Select the standard options you want to apply. + + For more information about standard options, refer to [Reference: Standard field definitions]({{< relref "../reference-standard-field-definitions/index.md" >}}). + +1. To preview your change, click outside of the field option box you are editing or press **Enter**. diff --git a/docs/sources/panels/working-with-panels/navigate-inspector-panel.md b/docs/sources/panels/working-with-panels/navigate-inspector-panel.md new file mode 100644 index 00000000000..5e02e3ff663 --- /dev/null +++ b/docs/sources/panels/working-with-panels/navigate-inspector-panel.md @@ -0,0 +1,25 @@ ++++ +title = "Navigate the Grafana Inspector panel" +aliases = ["/docs/sources/panels/working-with-panels/navigate-inspector-panel/"] +weight = 90 ++++ + +# Navigate the Grafana Inspector panel + +The panel inspector helps you understand and troubleshoot your panels. You can inspect the raw data for any Grafana panel, export that data to a comma-separated values (CSV) file, view query requests, and export panel and data JSON. + +> **Note:** Not all panel types include all tabs. For example, dashboard list panels do not have raw data to inspect, so they do not display the Stats, Data, or Query tabs. + +The panel inspector consists of the following options: + +1. The panel inspector displays Inspect: at the top of the pane. Click the arrow in the upper right corner to expand or reduce the pane. + +2. **Data tab -** Shows the raw data returned by the query with transformations applied. Field options such as overrides and value mappings are not applied by default. + +3. **Stats tab -** Shows how long your query takes and how much it returns. + +4. **JSON tab -** Allows you to view and copy the panel JSON, panel data JSON, and data frame structure JSON. This is useful if you are provisioning or administering Grafana. + +5. **Query tab -** Shows you the requests to the server sent when Grafana queries the data source. + +6. **Error tab -** Shows the error. Only visible when query returns error. diff --git a/docs/sources/panels/working-with-panels/navigate-panel-editor.md b/docs/sources/panels/working-with-panels/navigate-panel-editor.md new file mode 100644 index 00000000000..b39d212d71d --- /dev/null +++ b/docs/sources/panels/working-with-panels/navigate-panel-editor.md @@ -0,0 +1,40 @@ ++++ +title = "Navigate the Grafana panel editor" +weight = 10 +aliases = ["/docs/sources/panels/working-with-panels/navigate-panel-editor/"] ++++ + +# Navigate the Grafana panel editor + +This page describes the parts of the Grafana panel editor. + +1. Panel header: The header section lists the dashboard in which the panel appears and the following controls: + + - **Dashboard settings (gear) icon -** Click to access the dashboard settings. + - **Discard -** Discards changes you have made to the panel since you last saved the dashboard. + - **Save -** Saves changes you made to the panel. + - **Apply -** Applies changes you made and closes the panel editor, returning you to the dashboard. You will have to save the dashboard to persist the applied changes. + +2. Visualization preview: The visualization preview section contains the following options: + + - **Table view -** Convert any visualization to a table so that you can see the data. Table views are useful for troubleshooting. + - **Fill -** The visualization preview fills the available space. If you change the width of the side pane or height of the bottom pane the visualization changes to fill the available space. + - **Actual -** The visualization preview will have the exact size as the size on the dashboard. If not enough space is available, the visualization will scale down preserving the aspect ratio. + - **Time range controls -** For more information, refer to [Time range controls]({{< relref "../../dashboards/time-range-controls.md" >}}). + +3. Data section: The data section contains tabs where you enter queries, transform your data, and create alert rules (if applicable). + + - **Query tab -** Select your data source and enter queries here. For more information, refer to [Add a query]({{< relref "../query-a-data-source/add-a-query.md" >}}). + + - **Transform tab -** Apply data transformations. For more information, refer to [Transform data]({{< relref "../transform-data/_index.md" >}}). + - **Alert tab -** Write alert rules. For more information, refer to [Overview of Grafana 8 alerting]({{< relref "../../alerting/unified-alerting/_index.md" >}}). + +4. Panel display options: The display options section contains tabs where you configure almost every aspect of your data visualization, including: + + - [Apply color to series and fields]({{< relref "./apply-color-to-series.md" >}}) + - [Format a standard field]({{< relref "./format-standard-fields.md" >}}) + - [Add a title and description to a panel]({{< relref "./add-title-and-description.md" >}}) + +> Not all options are available for each visualization. + +{{< figure src="/static/img/docs/panel-editor/panel-editor-8-0.png" class="docs-image--no-shadow" max-width="1500px" >}} diff --git a/docs/sources/panels/working-with-panels/organize-dashboard.md b/docs/sources/panels/working-with-panels/organize-dashboard.md new file mode 100644 index 00000000000..d710c8fcad2 --- /dev/null +++ b/docs/sources/panels/working-with-panels/organize-dashboard.md @@ -0,0 +1,31 @@ ++++ +title = "Organize a dashboard" +aliases = ["/docs/sources/panels/working-with-panels/organize-dashboard/"] +weight = 80 ++++ + +# Organize a dashboard + +You can place any panel in any location you want and controls its size. The changes you make impact other users of the dashboard. + +## Before you begin + +- Add two or more panels +- Ensure that you sign in with Editor permissions + +**To organize a dashboard**: + +1. Hover your cursor over the panel, and click-and-drag the panel to its new location. + +1. To resize a penal, click the zoom in (+) and zoom out (-) icons. + +![](/static/img/docs/animated_gifs/drag_drop.gif) + +## Tips and shortcuts + +- Click the graph title and in the dropdown menu quickly duplicate the panel. +- Click the colored icon in the legend to change a series color or the y-axis. +- Click series name in the legend to hide series. +- Ctrl/Shift/Meta + click legend name to hide other series. +- Hover your cursor over a panel and press `e` to open the panel editor. +- Hover your cursor over a panel and press `v` to open the panel in full screen view.. diff --git a/docs/sources/panels/working-with-panels/view-json-model.md b/docs/sources/panels/working-with-panels/view-json-model.md new file mode 100644 index 00000000000..c0587a50107 --- /dev/null +++ b/docs/sources/panels/working-with-panels/view-json-model.md @@ -0,0 +1,25 @@ ++++ +title = "View a panel JSON model" +aliases = ["/docs/sources/panels/working-with-panels/view-json-model/"] +weight = 100 ++++ + +# View a panel JSON model + +Explore and export panel, panel data, and data frame JSON models. + +## Before you begin: + +- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). + +**To view a panel JSON model**: + +1. Open the panel inspector and then click the **JSON** tab or in the panel menu click **Inspect > Panel JSON**. + +1. In Select source, choose one of the following options: + + - **Panel JSON -** Displays a JSON object representing the panel. + - **Panel data -** Displays a JSON object representing the data that was passed to the panel. + - **DataFrame structure -** Displays the raw result set with transformations, field configuration, and overrides configuration applied. + +1. You can expand or collapse portions of the JSON to explore it, or you can click **Copy to clipboard** and paste the JSON in another application. diff --git a/docs/sources/release-notes/_index.md b/docs/sources/release-notes/_index.md index 143c051f4fd..2ae655180d9 100644 --- a/docs/sources/release-notes/_index.md +++ b/docs/sources/release-notes/_index.md @@ -8,6 +8,7 @@ weight = 10000 Here you can find detailed release notes that list everything that is included in every release as well as notices about deprecations, breaking changes as well as changes that relate to plugin development. +- [Release notes for 8.3.4]({{< relref "release-notes-8-3-4" >}}) - [Release notes for 8.3.3]({{< relref "release-notes-8-3-3" >}}) - [Release notes for 8.3.2]({{< relref "release-notes-8-3-2" >}}) - [Release notes for 8.3.1]({{< relref "release-notes-8-3-1" >}}) @@ -47,6 +48,7 @@ about deprecations, breaking changes as well as changes that relate to plugin de - [Release notes for 8.0.0-beta3]({{< relref "release-notes-8-0-0-beta3" >}}) - [Release notes for 8.0.0-beta2]({{< relref "release-notes-8-0-0-beta2" >}}) - [Release notes for 8.0.0-beta1]({{< relref "release-notes-8-0-0-beta1" >}}) +- [Release notes for 7.5.13]({{< relref "release-notes-7-5-13" >}}) - [Release notes for 7.5.12]({{< relref "release-notes-7-5-12" >}}) - [Release notes for 7.5.11]({{< relref "release-notes-7-5-11" >}}) - [Release notes for 7.5.10]({{< relref "release-notes-7-5-10" >}}) diff --git a/docs/sources/release-notes/release-notes-7-5-13.md b/docs/sources/release-notes/release-notes-7-5-13.md new file mode 100644 index 00000000000..c9542c5eab5 --- /dev/null +++ b/docs/sources/release-notes/release-notes-7-5-13.md @@ -0,0 +1,13 @@ ++++ +title = "Release notes for Grafana 7.5.13" +[_build] +list = false ++++ + + + +# Release notes for Grafana 7.5.13 + +### Bug fixes + +- **[v7.5.x] Alerting:** Fix NoDataFound for alert rules using AND operator (#41305). [#44066](https://github.com/grafana/grafana/pull/44066), [@armandgrillet](https://github.com/armandgrillet) diff --git a/docs/sources/release-notes/release-notes-8-3-4.md b/docs/sources/release-notes/release-notes-8-3-4.md new file mode 100644 index 00000000000..f4bb71d04d4 --- /dev/null +++ b/docs/sources/release-notes/release-notes-8-3-4.md @@ -0,0 +1,42 @@ ++++ +title = "Release notes for Grafana 8.3.4" +[_build] +list = false ++++ + + + +# Release notes for Grafana 8.3.4 + +### Features and enhancements + +- **Alerting:** Allow configuration of non-ready alertmanagers. [#43063](https://github.com/grafana/grafana/pull/43063), [@alexweav](https://github.com/alexweav) +- **Alerting:** Allow customization of Google chat message. [#43568](https://github.com/grafana/grafana/pull/43568), [@alexweav](https://github.com/alexweav) +- **Alerting:** Allow customization of Google chat message (#43568). [#43723](https://github.com/grafana/grafana/pull/43723), [@alexweav](https://github.com/alexweav) +- **AppPlugins:** Support app plugins with only default nav. [#43016](https://github.com/grafana/grafana/pull/43016), [@torkelo](https://github.com/torkelo) +- **InfluxDB:** InfluxQL: query editor: skip fields in metadata queries. [#42543](https://github.com/grafana/grafana/pull/42543), [@gabor](https://github.com/gabor) +- **Postgres/MySQL/MSSQL:** Cancel in-flight SQL query if user cancels query in grafana. [#43890](https://github.com/grafana/grafana/pull/43890), [@mdvictor](https://github.com/mdvictor) +- **Prometheus:** Forward oauth tokens after prometheus datasource migration. [#43686](https://github.com/grafana/grafana/pull/43686), [@MasslessParticle](https://github.com/MasslessParticle) + +### Bug fixes + +- **Azure Monitor:** Bug fix for variable interpolations in metrics dropdowns. [#43251](https://github.com/grafana/grafana/pull/43251), [@sarahzinger](https://github.com/sarahzinger) +- **Azure Monitor:** Improved error messages for variable queries. [#43213](https://github.com/grafana/grafana/pull/43213), [@sunker](https://github.com/sunker) +- **CloudMonitoring:** Fixes broken variable queries that use group bys. [#43914](https://github.com/grafana/grafana/pull/43914), [@sunker](https://github.com/sunker) +- **Configuration:** You can now see your expired API keys if you have no active ones. [#42452](https://github.com/grafana/grafana/pull/42452), [@ashharrison90](https://github.com/ashharrison90) +- **Elasticsearch:** Fix handling multiple datalinks for a single field. [#44029](https://github.com/grafana/grafana/pull/44029), [@Elfo404](https://github.com/Elfo404) +- **Export:** Fix error being thrown when exporting dashboards using query variables that reference the default datasource. [#44034](https://github.com/grafana/grafana/pull/44034), [@ashharrison90](https://github.com/ashharrison90) +- **ImportDashboard:** Fixes issue with importing dashboard and name ending up in uid. [#43451](https://github.com/grafana/grafana/pull/43451), [@torkelo](https://github.com/torkelo) +- **Login:** Page no longer overflows on mobile. [#43739](https://github.com/grafana/grafana/pull/43739), [@ashharrison90](https://github.com/ashharrison90) +- **Plugins:** Set backend metadata property for core plugins. [#43349](https://github.com/grafana/grafana/pull/43349), [@marefr](https://github.com/marefr) +- **Prometheus:** Fill missing steps with null values. [#43622](https://github.com/grafana/grafana/pull/43622), [@ivanahuckova](https://github.com/ivanahuckova) +- **Prometheus:** Fix interpolation of $\_\_rate_interval variable. [#44035](https://github.com/grafana/grafana/pull/44035), [@ivanahuckova](https://github.com/ivanahuckova) +- **Prometheus:** Interpolate variables with curly brackets syntax. [#42927](https://github.com/grafana/grafana/pull/42927), [@ivanahuckova](https://github.com/ivanahuckova) +- **Prometheus:** Respect the http-method data source setting. [#42753](https://github.com/grafana/grafana/pull/42753), [@gabor](https://github.com/gabor) +- **Table:** Fixes issue with field config applied to wrong fields when hiding columns. [#43376](https://github.com/grafana/grafana/pull/43376), [@torkelo](https://github.com/torkelo) +- **Toolkit:** Fix bug with rootUrls not being properly parsed when signing a private plugin. [#43014](https://github.com/grafana/grafana/pull/43014), [@dessen-xu](https://github.com/dessen-xu) +- **Variables:** Fix so data source variables are added to adhoc configuration. [#43881](https://github.com/grafana/grafana/pull/43881), [@hugohaggmark](https://github.com/hugohaggmark) + +### Plugin development fixes & changes + +- **Toolkit:** Revert build config so tslib is bundled with plugins to prevent plugins from crashing. [#43556](https://github.com/grafana/grafana/pull/43556), [@mckn](https://github.com/mckn) diff --git a/docs/sources/sharing/share-dashboard.md b/docs/sources/sharing/share-dashboard.md index 6a19ea5af16..771b55a3116 100644 --- a/docs/sources/sharing/share-dashboard.md +++ b/docs/sources/sharing/share-dashboard.md @@ -30,14 +30,14 @@ To share a direct link: A dashboard snapshot shares an interactive dashboard publicly. Grafana strips sensitive data like queries (metric, template and annotation) and panel links, leaving only the visible metric data and series names embedded into your dashboard. Dashboard snapshots can be accessed by anyone with the link. -You can publish snapshots to your local instance or to [snapshot.raintank.io](http://snapshot.raintank.io). The latter is a free service +You can publish snapshots to your local instance or to [snapshots.raintank.io](http://snapshots.raintank.io). The latter is a free service provided by Grafana Labs that allows you to publish dashboard snapshots to an external Grafana instance. The same rules still apply: anyone with the link can view it. You can set an expiration time if you want the snapshot removed after a certain time period. ![Dashboard share snapshot](/static/img/docs/sharing/share-dashboard-snapshot-7-3.png) To publish a snapshot: -1. Click on **Local Snapshot** or **Publish to snapshot.raintank.io**. This generates the link of the snapshot. +1. Click on **Local Snapshot** or **Publish to snapshots.raintank.io**. This generates the link of the snapshot. 1. Copy the snapshot link, and share it either within your organization or publicly on the web. In case you created a snapshot by mistake, click **delete snapshot** to remove the snapshot from your Grafana instance. diff --git a/docs/sources/sharing/share-panel.md b/docs/sources/sharing/share-panel.md index 05b3cc10cde..be16c994971 100644 --- a/docs/sources/sharing/share-panel.md +++ b/docs/sources/sharing/share-panel.md @@ -46,14 +46,14 @@ https://play.grafana.org/d/000000012/grafana-play-home?orgId=1&from=156871968017 A panel snapshot shares an interactive panel publicly. Grafana strips sensitive data leaving only the visible metric data and series names embedded into your dashboard. Panel snapshots can be accessed by anyone with the link. -You can publish snapshots to your local instance or to [snapshot.raintank.io](http://snapshot.raintank.io). The latter is a free service provided by [Raintank](http://raintank.io), that allows you to publish dashboard snapshots to an external Grafana instance. You can optionally set an expiration time if you want the snapshot to be removed after a certain time period. +You can publish snapshots to your local instance or to [snapshots.raintank.io](http://snapshots.raintank.io). The latter is a free service provided by [Grafana Labs](https://grafana.com), that allows you to publish dashboard snapshots to an external Grafana instance. You can optionally set an expiration time if you want the snapshot to be removed after a certain time period. ![Panel share snapshot](/static/img/docs/sharing/share-panel-snapshot-8-0.png) To publish a snapshot: 1. In the Share Panel dialog, click **Snapshot** to open the tab. -1. Click on **Local Snapshot** or **Publish to snapshot.raintank.io**. This generates the link of the snapshot. +1. Click on **Local Snapshot** or **Publish to snapshots.raintank.io**. This generates the link of the snapshot. 1. Copy the snapshot link, and share it either within your organization or publicly on the web. If you created a snapshot by mistake, click **delete snapshot** to remove the snapshot from your Grafana instance. @@ -70,7 +70,7 @@ Here is an example of the HTML code: ```html + ## Library panel diff --git a/docs/sources/troubleshooting/_index.md b/docs/sources/troubleshooting/_index.md index ef8c9f72fc4..f2e1838d9a8 100644 --- a/docs/sources/troubleshooting/_index.md +++ b/docs/sources/troubleshooting/_index.md @@ -21,7 +21,7 @@ For more information, refer to [Enable debug logging in Grafana CLI]({{< relref Order of transformations matters. If the final data output from multiple transformations looks wrong, try changing the transformation order. Each transformation transforms data returned by the previous transformation, not the original raw data. -For more information, refer to [Debug transformations]({{< relref "../panels/transformations/apply-transformations.md" >}}). +For more information, refer to [Debug a transformation]({{< relref "../panels/transform-data/debug-transformation.md" >}}). ## Text missing with server-side image rendering (RPM-based Linux) diff --git a/docs/sources/troubleshooting/troubleshoot-queries.md b/docs/sources/troubleshooting/troubleshoot-queries.md index 2283bdb7ef9..4126e65535d 100644 --- a/docs/sources/troubleshooting/troubleshoot-queries.md +++ b/docs/sources/troubleshooting/troubleshoot-queries.md @@ -19,7 +19,7 @@ The most common problems are related to the query and response from your data so like a bug or visualization issue in Grafana, it is almost always a problem with the data source query or the data source response. Start by inspecting your panel query and response. -For more information, refer to [Inspect a panel]({{< relref "../panels/inspect-panel.md" >}}). +For more information, refer to [Inspect request and response data]({{< relref "../panels/query-a-data-source/inspect-request-and-response-data.md" >}}). ## My query is slow diff --git a/docs/sources/visualizations/_index.md b/docs/sources/visualizations/_index.md index d9b8bc0d8d0..a5119e5b543 100644 --- a/docs/sources/visualizations/_index.md +++ b/docs/sources/visualizations/_index.md @@ -8,7 +8,7 @@ aliases = ["/docs/grafana/latest/panels/visualizations/"] Grafana offers a variety of visualizations to support different use cases. This section of the documentation highlights the built-in panels, their options and typical usage. -> **Note:** If you are unsure which visualization to pick, Grafana can provide visualization suggestions based on the panel query. When you select a visualization, Grafana will show a preview with that visualization applied. For more information, see the [add a panel]({{< relref "../panels/add-a-panel.md" >}}) documentation. +> **Note:** If you are unsure which visualization to pick, Grafana can provide visualization suggestions based on the panel query. When you select a visualization, Grafana will show a preview with that visualization applied. For more information, see the [add a panel]({{< relref "../panels/working-with-panels/add-panel.md" >}}) documentation. - Graphs & charts - [Time series]({{< relref "./time-series/_index.md" >}}) is the default and main Graph visualization. diff --git a/docs/sources/visualizations/bar-chart.md b/docs/sources/visualizations/bar-chart.md index c918ab65b31..04f60103314 100644 --- a/docs/sources/visualizations/bar-chart.md +++ b/docs/sources/visualizations/bar-chart.md @@ -76,7 +76,7 @@ Controls the fill opacity bars. ### Gradient mode -Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard [color scheme]({{< relref "../panels/standard-options.md#color-scheme" >}}) field option. +Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard color scheme field option. Gradient appearance is influenced by the **Fill opacity** setting. @@ -98,7 +98,7 @@ Gradient color is generated based on the hue of the line color. ### Legend calculations -Choose which of the [standard calculations]({{< relref "../panels/calculations-list.md">}}) to show in the legend. You can have more than one. +Choose which of the [standard calculations]({{< relref "../panels/reference-calculation-types.md">}}) to show in the legend. You can have more than one. ## Text size @@ -148,4 +148,4 @@ Set a **Soft min** or **soft max** option for better control of Y-axis limits. B **Soft min** and **soft max** settings can prevent blips from turning into mountains when the data is mostly flat, and hard min or max derived from standard min and max field options can prevent intermittent spikes from flattening useful detail by clipping the spikes past a defined point. -You can set standard min/max options to define hard limits of the Y-axis. For more information, refer to [Standard field options]({{< relref "../panels/standard-options.md#max" >}}). +You can set standard min/max options to define hard limits of the Y-axis. For more information, refer to [Standard field options]({{< relref "../panels/reference-standard-field-definitions.md#max" >}}). diff --git a/docs/sources/visualizations/bar-gauge-panel.md b/docs/sources/visualizations/bar-gauge-panel.md index d8101db0340..d3394d9f51f 100644 --- a/docs/sources/visualizations/bar-gauge-panel.md +++ b/docs/sources/visualizations/bar-gauge-panel.md @@ -26,7 +26,7 @@ Choose how Grafana displays your data. Show a calculated value based on all rows. -- **Calculation -** Select a reducer function that Grafana will use to reduce many fields to a single value. For a list of available calculations, refer to [List of calculations]({{< relref "../panels/calculations-list.md" >}}). +- **Calculation -** Select a reducer function that Grafana will use to reduce many fields to a single value. For a list of available calculations, refer to [List of calculations]({{< relref "../panels/reference-calculation-types.md" >}}). - **Fields -** Select the fields display in the panel. #### All values diff --git a/docs/sources/visualizations/gauge-panel.md b/docs/sources/visualizations/gauge-panel.md index 8aa8f14aecf..045d1d6ee8e 100644 --- a/docs/sources/visualizations/gauge-panel.md +++ b/docs/sources/visualizations/gauge-panel.md @@ -24,7 +24,7 @@ Choose how Grafana displays your data. Show a calculated value based on all rows. -- **Calculation -** Select a reducer function that Grafana will use to reduce many fields to a single value. For a list of available calculations, refer to [List of calculations]({{< relref "../panels/calculations-list.md" >}}). +- **Calculation -** Select a reducer function that Grafana will use to reduce many fields to a single value. For a list of available calculations, refer to [List of calculations]({{< relref "../panels/reference-calculation-types.md" >}}). - **Fields -** Select the fields display in the panel. #### All values diff --git a/docs/sources/visualizations/graph-panel.md b/docs/sources/visualizations/graph-panel.md index ac1bde7c7a6..d2006ea5916 100644 --- a/docs/sources/visualizations/graph-panel.md +++ b/docs/sources/visualizations/graph-panel.md @@ -16,9 +16,9 @@ The graph panel can render metrics as a line, a path of dots, or a series of bar Graph visualizations allow you to apply: - [Alerts]({{< relref "../alerting/_index.md" >}}) - This is the only type of visualization that allows you to set alerts. -- [Data transformations]({{< relref "../panels/transformations/_index.md" >}}) -- [Field overrides]({{< relref "../panels/field-overrides.md" >}}) -- [Thresholds]({{< relref "../panels/thresholds.md" >}}) +- [Transform data]({{< relref "../panels/transform-data/apply-transformation-to-data.md" >}}) +- [Add a field override]({{< relref "../panels/override-field-values/add-a-field-override.md" >}}) +- [Add a threshold]({{< relref "../panels/specify-thresholds/add-a-threshold.md" >}}) ## Display options diff --git a/docs/sources/visualizations/histogram.md b/docs/sources/visualizations/histogram.md index 7def217c3e4..548b272c7e1 100644 --- a/docs/sources/visualizations/histogram.md +++ b/docs/sources/visualizations/histogram.md @@ -42,7 +42,7 @@ Controls the fill opacity bars. ### Gradient mode -Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard [color scheme]({{< relref "../panels/standard-options.md#color-scheme" >}}) field option. +Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard [color scheme]({{< relref "../panels/working-with-panels/apply-color-to-series.md" >}}) field option. Gradient display is influenced by the **Fill opacity** setting. @@ -64,4 +64,4 @@ Gradient color is generated based on the hue of the line color. ### Legend calculations -Choose a [standard calculations]({{< relref "../panels/calculations-list.md">}}) to show in the legend. You can select more than one. +Choose a [standard calculations]({{< relref "../panels/reference-calculation-types.md">}}) to show in the legend. You can select more than one. diff --git a/docs/sources/visualizations/pie-chart-panel.md b/docs/sources/visualizations/pie-chart-panel.md index 77a389120cc..2720bfb0321 100644 --- a/docs/sources/visualizations/pie-chart-panel.md +++ b/docs/sources/visualizations/pie-chart-panel.md @@ -24,7 +24,7 @@ Choose how much information to show. ### Calculation -Select a calculation to reduce each series when Calculate has been selected. For information about available calculations, refer to the [Calculation list]({{< relref "../panels/calculations-list.md" >}}). +Select a calculation to reduce each series when Calculate has been selected. For information about available calculations, refer to the [Calculation list]({{< relref "../panels/reference-calculation-types.md" >}}). ### Limit diff --git a/docs/sources/visualizations/stat-panel.md b/docs/sources/visualizations/stat-panel.md index c78d73661df..a1608afaab0 100644 --- a/docs/sources/visualizations/stat-panel.md +++ b/docs/sources/visualizations/stat-panel.md @@ -41,7 +41,7 @@ Choose how Grafana displays your data. Show a calculated value based on all rows. -- **Calculation -** Select a reducer function that Grafana will use to reduce many fields to a single value. For a list of available calculations, refer to [List of calculations]({{< relref "../panels/calculations-list.md" >}}). +- **Calculation -** Select a reducer function that Grafana will use to reduce many fields to a single value. For a list of available calculations, refer to [List of calculations]({{< relref "../panels/reference-calculation-types.md" >}}). - **Fields -** Select the fields display in the panel. #### All values diff --git a/docs/sources/visualizations/state-timeline.md b/docs/sources/visualizations/state-timeline.md index 099b8e87eb7..4dbe6d12cce 100644 --- a/docs/sources/visualizations/state-timeline.md +++ b/docs/sources/visualizations/state-timeline.md @@ -42,7 +42,7 @@ Controls the opacity of state regions. ## Value mappings -To assign colors to boolean or string values, use [Value mappings]({{< relref "../panels/value-mappings.md" >}}). +To assign colors to boolean or string values, use [Value mappings]({{< relref "../panels/format-data/about-value-mapping.md" >}}). {{< figure src="/static/img/docs/v8/value_mappings_side_editor.png" max-width="300px" caption="Value mappings side editor" >}} @@ -54,6 +54,6 @@ The panel can be used with time series data as well. In this case, the threshold ## Legend options -When the legend option is enabled it can show either the value mappings or the threshold brackets. To show the value mappings in the legend, it's important that the `Color scheme` option under [Standard options]({{< relref "../panels/standard-options.md" >}}) is set to `Single color` or `Classic palette`. To see the threshold brackets in the legend set the `Color scheme` to `From thresholds`. +When the legend option is enabled it can show either the value mappings or the threshold brackets. To show the value mappings in the legend, it's important that the `Color scheme` as referenced in [Apply color to a series and fields]({{< relref "../panels/working-with-panels/apply-color-to-series.md" >}}) is set to `Single color` or `Classic palette`. To see the threshold brackets in the legend set the `Color scheme` to `From thresholds`. {{< docs/shared "visualizations/legend-mode.md" >}} diff --git a/docs/sources/visualizations/status-history.md b/docs/sources/visualizations/status-history.md index 6c44fdd3ca6..37f5748c43e 100644 --- a/docs/sources/visualizations/status-history.md +++ b/docs/sources/visualizations/status-history.md @@ -51,6 +51,6 @@ use gradient color schemes to color values. ## Legend options -When the legend option is enabled it can show either the value mappings or the threshold brackets. To show the value mappings in the legend, it's important that the `Color scheme` option under [Standard options]({{< relref "../panels/standard-options.md" >}}) is set to `Single color` or `Classic palette`. To see the threshold brackets in the legend set the `Color scheme` to `From thresholds`. +When the legend option is enabled it can show either the value mappings or the threshold brackets. To show the value mappings in the legend, it's important that the `Color scheme` as referenced in [Apply color to a series and fields]({{< relref "../panels/working-with-panels/apply-color-to-series.md" >}}) is set to `Single color` or `Classic palette`. To see the threshold brackets in the legend set the `Color scheme` to `From thresholds`. {{< docs/shared "visualizations/legend-mode.md" >}} diff --git a/docs/sources/visualizations/table/filter-table-columns.md b/docs/sources/visualizations/table/filter-table-columns.md index c92460a913e..475b8035074 100644 --- a/docs/sources/visualizations/table/filter-table-columns.md +++ b/docs/sources/visualizations/table/filter-table-columns.md @@ -12,7 +12,7 @@ If you turn on the **Column filter**, then you can filter table options. ## Turn on column filtering 1. In Grafana, navigate to the dashboard with the table with the columns that you want to filter. -1. On the table panel you want to filter, [open the panel editor]({{< relref "../../panels/panel-editor.md#open-the-panel-editor" >}}). +1. On the table panel you want to filter, open the panel editor. 1. Click the **Field** tab. 1. In Table options, turn on the **Column filter** option. diff --git a/docs/sources/visualizations/time-series/_index.md b/docs/sources/visualizations/time-series/_index.md index de68030d94d..cf384933397 100644 --- a/docs/sources/visualizations/time-series/_index.md +++ b/docs/sources/visualizations/time-series/_index.md @@ -9,7 +9,7 @@ weight = 1200 {{< figure src="/static/img/docs/time-series-panel/time_series_small_example.png" max-width="1200px" caption="Time series" >}} -Time series visualization is the default and primary way to visualize time series data. It can render as a line, a path of dots, or a series of bars. It is versatile enough to display almost any time-series data. [ This public demo dashboard](https://play.grafana.org/d/000000016/1-time-series-graphs?orgId=1) contains many different examples for how this visualization can be configured and styled. +Time series visualization is the default and primary way to visualize time series data. It can render as a line, a path of dots, or a series of bars. It is versatile enough to display almost any time-series data. [This public demo dashboard](https://play.grafana.org/d/000000016/1-time-series-graphs?orgId=1) contains many different examples for how this visualization can be configured and styled. > **Note:** You can migrate Graph panel visualizations to Time series visualizations. To migrate, open the panel and then select the **Time series** visualization. Grafana transfers all applicable settings. @@ -23,7 +23,7 @@ These options are available whether you are graphing your time series as lines, ### Legend calculations -Choose which of the [standard calculations]({{< relref "../../panels/calculations-list.md">}}) to show in the legend. You can have more than one. +Choose which of the [standard calculations]({{< relref "../../panels/reference-calculation-types.md">}}) to show in the legend. You can have more than one. ## Graph styles diff --git a/docs/sources/visualizations/time-series/change-axis-display.md b/docs/sources/visualizations/time-series/change-axis-display.md index 3c370ad3901..6b893602d29 100644 --- a/docs/sources/visualizations/time-series/change-axis-display.md +++ b/docs/sources/visualizations/time-series/change-axis-display.md @@ -61,7 +61,7 @@ Set a **Soft min** or **soft max** option for better control of Y-axis limits. B **Soft min** and **soft max** settings can prevent blips from turning into mountains when the data is mostly flat, and hard min or max derived from standard min and max field options can prevent intermittent spikes from flattening useful detail by clipping the spikes past a defined point. -You can set standard min/max options to define hard limits of the Y-axis. For more information, refer to [Standard field options]({{< relref "../../panels/standard-options.md#max" >}}). +You can set standard min/max options to define hard limits of the Y-axis. For more information, refer to [Reference: Standard field definitions]({{< relref "../../panels/reference-standard-field-definitions.md#max" >}}). ![Label example](/static/img/docs/time-series-panel/axis-soft-min-max-7-4.png) diff --git a/docs/sources/visualizations/time-series/graph-color-scheme.md b/docs/sources/visualizations/time-series/graph-color-scheme.md index f73b30dc148..9bf451dd049 100644 --- a/docs/sources/visualizations/time-series/graph-color-scheme.md +++ b/docs/sources/visualizations/time-series/graph-color-scheme.md @@ -9,7 +9,7 @@ weight = 400 # Graph and color schemes -Under [Standard options]({{< relref "../../panels/standard-options.md" >}}) you find the [Color scheme]({{< relref "../../panels/standard-options.md#color-scheme" >}}) option. This option controls series are assigned their color. +To set the graph and color schemes, refer to [Apply color to series and fields]({{< relref "../../panels/working-with-panels/apply-color-to-series.md" >}}). ## Classic palette diff --git a/docs/sources/visualizations/time-series/graph-time-series-as-bars.md b/docs/sources/visualizations/time-series/graph-time-series-as-bars.md index a95e2091b07..99d9772fc31 100644 --- a/docs/sources/visualizations/time-series/graph-time-series-as-bars.md +++ b/docs/sources/visualizations/time-series/graph-time-series-as-bars.md @@ -9,11 +9,14 @@ weight = 200 This section explains how to use Time series field options to visualize time series data as bars and illustrates what the options do. +For more information about the time series visualization, refer to [Time series]({{< relref "_index.md" >}}). + ## Create the panel -1. [Add a panel]({{< relref "../../panels/add-a-panel.md" >}}). Select the [Time series]({{< relref "_index.md" >}}) visualization. -2. In the [Panel editor]({{< relref "../../panels/panel-editor.md" >}}) side pane, click **Graph styles** to expand it. -3. In Style, click **Bars**. +1. [Add a panel]({{< relref "../../panels/working-with-panels/add-panel.md" >}}). +1. Select the **Time series** visualization. +1. In the Panel editor side pane, click **Graph styles** to expand it. +1. In Style, click **Bars**. ## Style the bars @@ -75,7 +78,7 @@ Fill opacity set to 95: ### Gradient mode -Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard [color scheme]({{< relref "../../panels/standard-options.md#color-scheme" >}}) field option. +Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard color scheme field option. For more information, refer to [Apply color to series and fields]({{< relref "../../panels/working-with-panels/apply-color-to-series.md" >}}). Gradient appearance is influenced by the **Fill opacity** setting. In the screenshots below, **Fill opacity** is set to 50. @@ -99,7 +102,7 @@ Gradient color is generated based on the hue of the line color. #### Scheme -In this mode the whole bar will use a color gradient defined by your [Color scheme]({{< relref "../../panels/standard-options.md#color-scheme" >}}) option. There is more information on this option in [Graph and color scheme]({{< relref "./graph-color-scheme.md" >}}). +In this mode the whole bar will use a color gradient defined by your Color scheme. For more information, refer to [Apply color to series and fields]({{< relref "../../panels/working-with-panels/apply-color-to-series.md" >}}). There is more information on this option in [Graph and color scheme]({{< relref "./graph-color-scheme.md" >}}). {{< figure src="/static/img/docs/time-series-panel/gradient_mode_scheme_bars.png" max-width="1200px" caption="Gradient color scheme mode" >}} diff --git a/docs/sources/visualizations/time-series/graph-time-series-as-lines.md b/docs/sources/visualizations/time-series/graph-time-series-as-lines.md index dbb4760d859..a9bb7d34315 100644 --- a/docs/sources/visualizations/time-series/graph-time-series-as-lines.md +++ b/docs/sources/visualizations/time-series/graph-time-series-as-lines.md @@ -11,8 +11,8 @@ This section explains how to use Time series field options to visualize time ser ## Create the panel -1. [Add a panel]({{< relref "../../panels/add-a-panel.md" >}}). Select the [Time series]({{< relref "_index.md" >}}) visualization. -1. In the [Panel editor]({{< relref "../../panels/panel-editor.md" >}}) side pane, click **Graph styles** to expand it. +1. [Add a panel]({{< relref "../../panels/working-with-panels/add-panel.md" >}}). Select the [Time series]({{< relref "_index.md" >}}) visualization. +1. In the Panel editor side pane, click **Graph styles** to expand it. 1. In Style, click **Lines**. ## Style the lines @@ -83,7 +83,7 @@ Fill opacity set to 95: ### Gradient mode -Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard [color scheme]({{< relref "../../panels/standard-options.md#color-scheme" >}}) field option. +Set the mode of the gradient fill. Fill gradient is based on the line color. To change the color, use the standard color scheme field option. For more information, refer to [Apply color to series and fields]({{< relref "../../panels/working-with-panels/apply-color-to-series.md" >}}) . Gradient appearance is influenced by the **Fill opacity** setting. In the screenshots below, **Fill opacity** is set to 50. @@ -107,13 +107,13 @@ Gradient color is generated based on the hue of the line color. #### Scheme -In this mode the whole line will use a color gradient defined by your [Color scheme]({{< relref "../../panels/standard-options.md#color-scheme" >}}) option. There is more information on this option in [Graph and color scheme]({{< relref "./graph-color-scheme.md" >}}). +In this mode the whole line will use a color gradient defined by your Color scheme. For more information, refer to [Apply color to series and fields]({{< relref "../../panels/working-with-panels/apply-color-to-series.md" >}}). There is more information on this option in [Graph and color scheme]({{< relref "./graph-color-scheme.md" >}}). {{< figure src="/static/img/docs/time-series-panel/gradient_mode_scheme_line.png" max-width="1200px" caption="Gradient mode scheme" >}} ### Line style -Set the style of the line. To change the color, use the standard [color scheme]({{< relref "../../panels/standard-options.md#color-scheme" >}}) field option. +Set the style of the line. To change the color, use the standard color scheme field option. For more information, refer to [Apply color to series and fields]({{< relref "../../panels/working-with-panels/apply-color-to-series.md" >}}) Line style appearance is influenced by the **Line width** and **Fill opacity** settings. In the screenshots below, **Line width** is set to 3 and **Fill opacity** is set to 20. diff --git a/docs/sources/visualizations/time-series/graph-time-series-as-points.md b/docs/sources/visualizations/time-series/graph-time-series-as-points.md index ac5f0852506..d92b56be547 100644 --- a/docs/sources/visualizations/time-series/graph-time-series-as-points.md +++ b/docs/sources/visualizations/time-series/graph-time-series-as-points.md @@ -11,8 +11,8 @@ This section explains how to use Time series field options to visualize time ser ## Create the panel -1. [Add a panel]({{< relref "../../panels/add-a-panel.md" >}}). Select the [Time series]({{< relref "_index.md" >}}) visualization. -1. In the [Panel editor]({{< relref "../../panels/panel-editor.md" >}}) side pane, click **Graph styles** to expand it. +1. [Add a panel]({{< relref "../../panels/working-with-panels/add-panel.md" >}}). Select the [Time series]({{< relref "_index.md" >}}) visualization. +1. In the Panel editor side pane, click **Graph styles** to expand it. 1. In Style, click **Points**. ## Style the points diff --git a/docs/sources/visualizations/time-series/graph-time-series-stacking.md b/docs/sources/visualizations/time-series/graph-time-series-stacking.md index 9b2ccdba022..36866bc1571 100644 --- a/docs/sources/visualizations/time-series/graph-time-series-stacking.md +++ b/docs/sources/visualizations/time-series/graph-time-series-stacking.md @@ -37,7 +37,7 @@ Enable stacking series on top of each other. The stacking group option is only available as an override. -For more information about creating field overrides, refer to [Field overrides]({{< relref "../../panels/field-overrides.md" >}}). +For more information about creating field overrides, refer to [About field overrides]({{< relref "../../panels/override-field-values/about-field-overrides.md" >}}). Stack series in the same group. In the Overrides section: diff --git a/docs/sources/whatsnew/whats-new-in-v2-0.md b/docs/sources/whatsnew/whats-new-in-v2-0.md index 4882bcca4ee..67107573ac0 100644 --- a/docs/sources/whatsnew/whats-new-in-v2-0.md +++ b/docs/sources/whatsnew/whats-new-in-v2-0.md @@ -57,7 +57,7 @@ They're a great way to communicate about a particular incident with specific peo ### Publish snapshots -You can publish snapshots locally or to [snapshot.raintank.io](http://snapshot.raintank.io). snapshot.raintank is a free service provided by [raintank](http://raintank.io) for hosting external Grafana snapshots. +You can publish snapshots locally or to [snapshots.raintank.io](https://snapshots.raintank.io). snapshots.raintank.io is a free service provided by [Grafana Labs](https://grafana.com) for hosting external Grafana snapshots. Either way, anyone with the link (and access to your Grafana instance for local snapshots) can view it. @@ -82,11 +82,11 @@ Currently you can only override the dashboard time with relative time ranges, no You can embed a single panel on another web page or your own application using the panel share dialog. -Below you should see an iframe with a graph panel (taken from a Dashboard snapshot at [snapshot.raintank.io](http://snapshot.raintank.io). +Below you should see an iframe with a graph panel (taken from a Dashboard snapshot at [snapshots.raintank.io](http://snapshots.raintank.io). Try hovering or zooming on the panel below! - + This feature makes it easy to include interactive visualizations from your Grafana instance anywhere you want. diff --git a/docs/sources/whatsnew/whats-new-in-v7-0.md b/docs/sources/whatsnew/whats-new-in-v7-0.md index b7212bb359c..ba2bc196f5c 100644 --- a/docs/sources/whatsnew/whats-new-in-v7-0.md +++ b/docs/sources/whatsnew/whats-new-in-v7-0.md @@ -42,7 +42,7 @@ In previous versions of Grafana, each visualization had slightly different ways We have yet to migrate all core panels to this new architecture so in 7.0 there will be some inconsistencies in the UI between panels. This will be fixed soon in future releases as we update all the core panels and help the community update the community panel plugins. -Learn more about this feature in [Panel editor]({{< relref "../panels/panel-editor.md" >}}). +Learn more about this feature in [Panel editor]({{< relref "../panels/working-with-panels/_index.md" >}}). ## New tracing UI @@ -58,9 +58,9 @@ In the future we will add more workflows and integrations so that correlating be The data you want to visualize can come from many different places and it is usually not in exactly the right form. Users can now transform non-time series data into tables (e.g., JSON files or even simple lookup tables) in seconds without any customization or additional overhead. They can then combine non-time series data with any other data in Grafana; data from an external database or a panel that already exists in one of their current dashboards. -By chaining a simple set of point and click [transformations]({{< relref "../panels/transformations/_index.md" >}}), users will be able join, pivot, filter, re-name and do calculations to get the results they need. Perfect for operations across queries or data sources missing essential data transformations. +By chaining a simple set of point and click [transformations]({{< relref "../panels/reference-transformation-functions.md" >}}), users will be able join, pivot, filter, re-name and do calculations to get the results they need. Perfect for operations across queries or data sources missing essential data transformations. -[Transformations]({{< relref "../panels/transformations/_index.md" >}}) also adds the ability to do maths across queries. Lots of data sources do not support this natively, so being able to do it in Grafana is a powerful feature. +[Transformations]({{< relref "../panels/transform-data/about-transformation.md" >}}) also adds the ability to do math across queries. Lots of data sources do not support this natively, so being able to do it in Grafana is a powerful feature. For users with large dashboards or with heavy queries, being able to reuse the query result from one panel in another panel can be a huge performance gain for slow queries (e.g log or sql queries). From the data source menu in the query editor, you can choose the `--dashboard--` option and then choose the query result from another panel on the same dashboard. @@ -78,7 +78,7 @@ We are also introducing a new shared data model for both time series and table d - **Outer join:** Joins many time series/tables by a field. This can be used to outer join multiple time series on the _time_ field to show many time series in one table. - **Add field from calculation:** This is a powerful transformation that allows you perform many different types of math operations and add the result as a new field. Can be used to calculate the difference between two series or fields and add the result to a new field. Or multiply one field with another and add the result to a new field. -Learn more about this feature in [Transformations]({{< relref "../panels/transformations/_index.md" >}}). +Learn more about this feature in [Transformations]({{< relref "../panels/reference-transformation-functions.md" >}}). ## Field options and overrides @@ -88,7 +88,7 @@ Up until now the overrides were available only for Graph and Table panel(via Col This feature enables even more powerful visualizations and fine grained control over how the data is displayed. -Learn more about this feature in [Field overrides]({{< relref "../panels/field-overrides.md" >}}). +Learn more about this feature in [Field overrides]({{< relref "../panels/override-field-values/about-field-overrides.md" >}}). ## Inspect panels and export data to CSV @@ -96,7 +96,7 @@ Learn more about this feature in [Field overrides]({{< relref "../panels/field-o Another new feature of Grafana 7.0 is the panel inspector. Inspect allows you to view the raw data for any Grafana panel as well as export that data to a CSV file. With Panel inspect you will also be able to perform simple raw data transformations like join, view query stats or detailed execution data. -Learn more about this feature in [Inspect a panel]({{< relref "../panels/inspect-panel.md" >}}). +Learn more about this feature in [Inspect a panel]({{< relref "../panels/query-a-data-source/inspect-request-and-response-data.md" >}}).
diff --git a/docs/sources/whatsnew/whats-new-in-v7-2.md b/docs/sources/whatsnew/whats-new-in-v7-2.md index e3d7f227d19..2417cbfb298 100644 --- a/docs/sources/whatsnew/whats-new-in-v7-2.md +++ b/docs/sources/whatsnew/whats-new-in-v7-2.md @@ -73,14 +73,12 @@ You can now dynamically apply value filters to any table column. This option can You can now add override rules that use a regex matcher to choose which fields to apply rules to. -[Field overrides]({{< relref "../panels/field-overrides.md" >}}) has been updated as a result of these changes. - ## New transformations and enhancements Grafana 7.2 includes the following transformation enhancements: -- A new [Group By]({{< relref "../panels/transformations/types-options.md#group-by">}}) transformation that allows you to group by multiple fields and add any number of aggregations for other fields. -- The [Labels to fields]({{< relref "../panels/transformations/types-options.md#labels-to-fields">}}) transformation now allows you to pick one label and use that as the name of the value field. +- A new [Group By]({{< relref "../panels/reference-transformation-functions.md#group-by">}}) transformation that allows you to group by multiple fields and add any number of aggregations for other fields. +- The [Labels to fields]({{< relref "../panels/reference-transformation-functions.md#labels-to-fields">}}) transformation now allows you to pick one label and use that as the name of the value field. - You can drag transformations to reorder them. Remember that transformations are processed in the order they are listed in the UI, so think before you move something! {{< figure src="/static/img/docs/v72/transformations.gif" max-width="800px" caption="Group by and reordering of transformations" >}} @@ -91,7 +89,7 @@ The up and down arrows, which were previously the only way to change query order {{< figure src="/static/img/docs/v72/drag-queries.gif" max-width="800px" caption="Drag to reorder queries" >}} -The [Queries]({{< relref "../panels/queries.md" >}}) topic has been updated as a result of this change. +The [Queries]({{< relref "../panels/query-a-data-source/add-a-query.md" >}}) topic has been updated as a result of this change. ## Inspect queries in Explore diff --git a/docs/sources/whatsnew/whats-new-in-v7-3.md b/docs/sources/whatsnew/whats-new-in-v7-3.md index bab3cf37b16..bd4e1d7cb26 100644 --- a/docs/sources/whatsnew/whats-new-in-v7-3.md +++ b/docs/sources/whatsnew/whats-new-in-v7-3.md @@ -34,7 +34,7 @@ Another new feature that can be seen in the image above is the new image cell di {{< figure src="/static/img/docs/v73/color_scheme_dropdown.png" max-width="450px" caption="Color scheme" class="pull-right" >}} -A new standard field [color scheme]({{< relref "../panels/standard-options.md#color-scheme" >}}) option has been added. This new option will provide a unified way for all new panels to specify how colors should be assigned. +A new standard field color scheme option has been added. This new option will provide a unified way for all new panels to specify how colors should be assigned. For more information, refer to [Apply color to series and fields](../panels/working-with-panels/apply-color-to-series.md) - **Single color**: Specifies a single color. Useful in an override rule. - **From thresholds**: Informs Grafana to take color from the matching threshold. @@ -61,7 +61,7 @@ As this new option is a standard field option it works in every panel. Here is a In v7.0, we introduced a new table panel and inspect mode with Download CSV enabled. However, CSV export to Excel was removed. Due to a large number of inquiries and requests, this [community contribution from tomdaly](https://github.com/grafana/grafana/pull/27284) brought the feature back. -For more information, refer to [Download raw query results as CSV]({{< relref "../panels/inspect-panel/#download-raw-query-results-as-csv" >}}) in the Grafana documentation. +For more information, refer to [Download raw query results]({{< relref "../panels/query-a-data-source/download-raw-query-results.md" >}}). ## Google Cloud monitoring out-of-the-box dashboards diff --git a/docs/sources/whatsnew/whats-new-in-v7-4.md b/docs/sources/whatsnew/whats-new-in-v7-4.md index d71053fba1b..4fff3b039e4 100644 --- a/docs/sources/whatsnew/whats-new-in-v7-4.md +++ b/docs/sources/whatsnew/whats-new-in-v7-4.md @@ -54,7 +54,7 @@ The following transformations were added in Grafana 7.4. The _Sort by_ transformation allows you to sort data before sending it to the visualization. -For more information, refer to [Sort by]({{< relref "../panels/transformations/types-options.md#sort-by" >}}) in [Transformation types and options]({{< relref "../panels/transformations/types-options.md" >}}). +For more information, refer to [Sort by]({{< relref "../panels/reference-transformation-functions.md#sort-by" >}}). #### Filter data by value transform @@ -62,13 +62,13 @@ The new _Filter data by value_ transformation allows you to filter your data dir This transformation is very useful if your data source does not natively filter by values. You might also use this to narrow values to display if you are using a shared query. -For more information, refer to [Filter data by value]({{< relref "../panels/transformations/types-options.md#filter-data-by-value" >}}) in [Transformation types and options]({{< relref "../panels/transformations/types-options.md" >}}). +For more information, refer to [Filter data by value]({{< relref "../panels/reference-transformation-functions.md#filter-data-by-value" >}}). ### New override option On the Overrides tab, you can now set properties for fields returned by a specific query. -For more information, refer to [Field overrides]({{< relref "../panels/field-overrides.md" >}}). +For more information, refer to [About field overrides]({{< relref "../panels/override-field-values/about-field-overrides.md" >}}). ### Exemplar support @@ -99,7 +99,7 @@ The main use case is for [multi-dimensional]({{< relref "../basics/timeseries-di > **Note:** Queries built with this feature might break with minor version upgrades until Grafana 8 is released. This feature does not work with the current Grafana alerting. -For more information, refer to [Expressions]({{< relref "../panels/expressions.md" >}}). [Queries]({{< relref "../panels/queries.md" >}}) was also updated as a result of this feature. +For more information, refer to [About expressions]({{< relref "../panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions.md" >}}). [About queries]({{< relref "../panels/query-a-data-source/about-queries.md" >}}) was also updated as a result of this feature. ### Alert notification query label interpolation @@ -170,7 +170,7 @@ For more information, refer to the [Google Cloud Monitoring docs]({{< relref ".. The feature previously referred to as DataSource Start Pages or Cheat Sheets has been renamed to Query Editor Help, and is now supported in panel query editors (depending on the data source), as well as in Explore. -[Queries]({{< relref "../panels/queries.md" >}}) was updated as a result of this feature. +[Queries]({{< relref "../panels/query-a-data-source/manage-queries.md" >}}) was updated as a result of this feature. For more information on adding a query editor help component to your plugin, refer to [Add a query editor help component]({{< relref "../developers/plugins/add-query-editor-help.md" >}}). diff --git a/docs/sources/whatsnew/whats-new-in-v8-0.md b/docs/sources/whatsnew/whats-new-in-v8-0.md index a5e73e63e28..dc7d4725684 100644 --- a/docs/sources/whatsnew/whats-new-in-v8-0.md +++ b/docs/sources/whatsnew/whats-new-in-v8-0.md @@ -28,7 +28,7 @@ To learn more about the differences between new alerts and the legacy alerts, re ### Library panels -Library panels allow users to build panels that can be used in multiple dashboards. Any updates made to that shared panel will then automatically be applied to all the dashboards that have that panel. For instructions on how to create, add, unlink and manage library panels, refer to [Library panels]({{< relref "../panels/panel-library.md" >}}). +Library panels allow users to build panels that can be used in multiple dashboards. Any updates made to that shared panel will then automatically be applied to all the dashboards that have that panel. For instructions on how to create, add, unlink and manage library panels, refer to [Library panels]({{< relref "../panels/library-panels/" >}}). ### Real-time streaming @@ -114,7 +114,7 @@ Lots of panel editor improvements, heavily informed by user research and communi - All options are now shown in a single pane. - You can now search panel options. -- The Value mappings feature has been completely redesigned. For more information, refer to [Value mappings]({{< relref "../panels/value-mappings.md" >}}). +- The Value mappings feature has been completely redesigned. For more information, refer to [About value mappings]({{< relref "../panels/format-data/about-value-mapping.md" >}}). - New **Table view** option is always available. The [Panels]({{< relref "../panels/_index.md" >}}) section has been updated to reflect these changes. @@ -129,7 +129,7 @@ Under the hood, the new theme architecture enables us to bring more sophisticate When you inspect a panel, you can now download log results as a text (.txt) file. -[Download log results]({{< relref "../panels/inspect-panel.md#download-log-results" >}}) in [Inspect a panel]({{< relref "../panels/inspect-panel.md" >}}) was added as a result of this feature. +[Download log results]({{< relref "../panels/working-with-panels/download-log-results.md" >}}) was added as a result of this feature. ### Inspector in Explore diff --git a/docs/sources/whatsnew/whats-new-in-v8-1.md b/docs/sources/whatsnew/whats-new-in-v8-1.md index c675aec1b25..085f6caeabb 100644 --- a/docs/sources/whatsnew/whats-new-in-v8-1.md +++ b/docs/sources/whatsnew/whats-new-in-v8-1.md @@ -53,7 +53,7 @@ Grafana 8.1 includes some significant enhancements to transformations, including #### Config from query (Beta) -This transformation enables panel config (Threshold, Min, Max, etc.) to be derived from query results. For more information, refer to [Config from query results transform]({{< relref "../panels/transformations/config-from-query.md" >}}). +This transformation enables panel config (Threshold, Min, Max, etc.) to be derived from query results. For more information, refer to [Config from query results]({{< relref "../panels/reference-transformation-functions.md#config-from-query-results" >}}). #### Rows to fields (Beta) @@ -75,7 +75,7 @@ Output: As you can see each row in the source data becomes a separate field. Each field now also has a max config option set. Options like **Min**, **Max**, **Unit** and **Thresholds** are all part of field configuration and if set like this will be used by the visualization instead of any options manually configured in the panel editor options pane. -For more on how to use this transformation, refer to [Rows to fields transform]({{< relref "../panels/transformations/rows-to-fields.md" >}}). +For more on how to use this transformation, refer to [Rows to fields transform]({{< relref "../panels/reference-transformation-functions.md#rows-to-fields" >}}). #### Contextual & Inline Help diff --git a/docs/sources/whatsnew/whats-new-in-v8-3.md b/docs/sources/whatsnew/whats-new-in-v8-3.md index d2cbec7c658..a8a15a30a5c 100644 --- a/docs/sources/whatsnew/whats-new-in-v8-3.md +++ b/docs/sources/whatsnew/whats-new-in-v8-3.md @@ -20,7 +20,7 @@ We’ve summarized what’s new in the release here, but you might also be inter ## Community Contributions -Grafana 8.3 includes many community contributions including the ability to interpolate variables in tags ([#42318](https://github.com/grafana/grafana/pull/42318)), fly-out sub-menu support ([#41647](https://github.com/grafana/grafana/pull/42318)), the addition of AWS RoboMaker & AWS Global Accelerator metrics to the AWS CloudWatch data source ([#41450](https://github.com/grafana/grafana/pull/41450), [#41236](https://github.com/grafana/grafana/pull/41236)), as well as updates to the news panel ([#41193](https://github.com/grafana/grafana/pull/41193)), the plugin catalog ([#40917](https://github.com/grafana/grafana/pull/40917)), and many more. The Grafana Labs team is deeply grateful for all those who take the time to contribute and participate in the development of Grafana. +Grafana 8.3 includes many community contributions including the ability to interpolate variables in tags ([#42318](https://github.com/grafana/grafana/pull/42318)), fly-out sub-menu support ([#41647](https://github.com/grafana/grafana/pull/41647)), the addition of AWS RoboMaker & AWS Global Accelerator metrics to the AWS CloudWatch data source ([#41450](https://github.com/grafana/grafana/pull/41450), [#41236](https://github.com/grafana/grafana/pull/41236)), as well as updates to the news panel ([#41193](https://github.com/grafana/grafana/pull/41193)), the plugin catalog ([#40917](https://github.com/grafana/grafana/pull/40917)), and many more. The Grafana Labs team is deeply grateful for all those who take the time to contribute and participate in the development of Grafana. ## Accessibility diff --git a/e2e/build_results.zip b/e2e/build_results.zip deleted file mode 100644 index 10ef9d01697..00000000000 Binary files a/e2e/build_results.zip and /dev/null differ diff --git a/e2e/smoke-tests-suite/variables/load-options-from-url.ts b/e2e/dashboards-suite/load-options-from-url.spec.ts similarity index 100% rename from e2e/smoke-tests-suite/variables/load-options-from-url.ts rename to e2e/dashboards-suite/load-options-from-url.spec.ts diff --git a/e2e/smoke-tests-suite/variables/new-query-variable.ts b/e2e/dashboards-suite/new-query-variable.spec.ts similarity index 100% rename from e2e/smoke-tests-suite/variables/new-query-variable.ts rename to e2e/dashboards-suite/new-query-variable.spec.ts diff --git a/e2e/smoke-tests-suite/variables/set-options-from-ui.ts b/e2e/dashboards-suite/set-options-from-ui.spec.ts similarity index 100% rename from e2e/smoke-tests-suite/variables/set-options-from-ui.ts rename to e2e/dashboards-suite/set-options-from-ui.spec.ts diff --git a/e2e/dashboards-suite/templating-dashboard-links-and-variables.ts b/e2e/dashboards-suite/templating-dashboard-links-and-variables.spec.ts similarity index 100% rename from e2e/dashboards-suite/templating-dashboard-links-and-variables.ts rename to e2e/dashboards-suite/templating-dashboard-links-and-variables.spec.ts diff --git a/e2e/smoke-tests-suite/variables/textbox-variables.ts b/e2e/dashboards-suite/textbox-variables.spec.ts similarity index 100% rename from e2e/smoke-tests-suite/variables/textbox-variables.ts rename to e2e/dashboards-suite/textbox-variables.spec.ts diff --git a/e2e/run-suite b/e2e/run-suite index ce9707169c3..9aa620b20f3 100755 --- a/e2e/run-suite +++ b/e2e/run-suite @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -xeo pipefail -. e2e/variables +. scripts/grafana-server/variables if ((BASH_VERSINFO[0] < 4)); then echo "Bash ver >= 4 is needed to run this script" diff --git a/e2e/start-and-run-suite b/e2e/start-and-run-suite index 9aac995b61f..2c5c7d1d996 100755 --- a/e2e/start-and-run-suite +++ b/e2e/start-and-run-suite @@ -1,13 +1,13 @@ #!/bin/bash -. e2e/variables +. scripts/grafana-server/variables if [ "$BASE_URL" != "" ]; then echo -e "BASE_URL set, skipping starting server" else # Start it in the background - ./e2e/start-server 2>&1 > e2e/server.log & - ./e2e/wait-for-grafana + ./scripts/grafana-server/start-server 2>&1 > scripts/grafana-server/server.log & + ./scripts/grafana-server/wait-for-grafana fi ./e2e/run-suite "$@" diff --git a/e2e/various-suite/exemplars.spec.ts b/e2e/various-suite/exemplars.spec.ts index f75faa114d2..13abcbefc6b 100644 --- a/e2e/various-suite/exemplars.spec.ts +++ b/e2e/various-suite/exemplars.spec.ts @@ -1,4 +1,5 @@ import { e2e } from '@grafana/e2e'; +import { selectors } from '@grafana/e2e-selectors'; const dataSourceName = 'PromExemplar'; const addDataSource = () => { @@ -9,6 +10,7 @@ const addDataSource = () => { form: () => { e2e.components.DataSource.Prometheus.configPage.exemplarsAddButton().click(); e2e.components.DataSource.Prometheus.configPage.internalLinkSwitch().check({ force: true }); + e2e.components.DataSource.DataSourceHttpSettings.urlInput().type('http://prom-url:9090'); e2e.components.DataSourcePicker.inputV2().should('be.visible').click({ force: true }); e2e().contains('gdev-tempo').scrollIntoView().should('be.visible').click(); diff --git a/e2e/verify-release b/e2e/verify-release index 3e40a635741..2bb9df037fa 100755 --- a/e2e/verify-release +++ b/e2e/verify-release @@ -1,5 +1,5 @@ #!/bin/bash -. e2e/variables +. scripts/grafana-server/variables ./e2e/run-suite verify/specs diff --git a/emails/templates/alert_notification.html b/emails/templates/alert_notification.html index d41104e61e9..98d2f7ff56c 100644 --- a/emails/templates/alert_notification.html +++ b/emails/templates/alert_notification.html @@ -111,7 +111,7 @@
- View your Alert rule + View your Alert rule
@@ -120,7 +120,7 @@
- Go to the Alerts page + Go to the Alerts page
diff --git a/emails/templates/invited_to_org.html b/emails/templates/invited_to_org.html index 681830604b9..69ef61587df 100644 --- a/emails/templates/invited_to_org.html +++ b/emails/templates/invited_to_org.html @@ -34,7 +34,7 @@ - +
Log in nowLog in now
diff --git a/emails/templates/new_user_invite.html b/emails/templates/new_user_invite.html index cafbf4a4af8..b2c0b431fd3 100644 --- a/emails/templates/new_user_invite.html +++ b/emails/templates/new_user_invite.html @@ -28,11 +28,11 @@

You've been invited to join the [[.OrgName]] organization by [[.InvitedBy]]. To accept your invitation and join the team, please click the link below:

- + - +
Accept InvitationAccept Invitation
@@ -40,10 +40,10 @@

You can also copy and paste this link into your browser directly: [[.LinkUrl]]

- + - \ No newline at end of file + diff --git a/emails/templates/signup_started.html b/emails/templates/signup_started.html index 80b4c327ba4..3e8b3e0b976 100644 --- a/emails/templates/signup_started.html +++ b/emails/templates/signup_started.html @@ -33,7 +33,7 @@ - +
Complete Sign UpComplete Sign Up
diff --git a/go.mod b/go.mod index a6b10ddbae9..df9a267e849 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/fatih/color v1.10.0 github.com/gchaincl/sqlhooks v1.3.0 github.com/getsentry/sentry-go v0.10.0 - github.com/go-kit/kit v0.11.0 + github.com/go-kit/kit v0.11.0 // indirect github.com/go-openapi/strfmt v0.20.2 github.com/go-redis/redis/v8 v8.11.4 github.com/go-sourcemap/sourcemap v2.1.3+incompatible @@ -47,21 +47,20 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/golang/snappy v0.0.4 - github.com/google/go-cmp v0.5.6 + github.com/google/go-cmp v0.5.7 github.com/google/uuid v1.3.0 github.com/google/wire v0.5.0 github.com/gorilla/websocket v1.4.2 github.com/gosimple/slug v1.9.0 github.com/grafana/cuetsy v0.0.0-20211119211437-8c25464cc9bf - github.com/grafana/grafana-aws-sdk v0.7.0 - github.com/grafana/grafana-plugin-sdk-go v0.121.0 + github.com/grafana/grafana-aws-sdk v0.9.1 + github.com/grafana/grafana-plugin-sdk-go v0.125.0 github.com/grafana/loki v1.6.2-0.20211015002020-7832783b1caa github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/hashicorp/go-hclog v0.16.1 github.com/hashicorp/go-plugin v1.4.3 github.com/hashicorp/go-version v1.3.0 - github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec - github.com/influxdata/influxdb-client-go/v2 v2.3.1-0.20210518120617-5d1fff431040 + github.com/influxdata/influxdb-client-go/v2 v2.6.0 github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.12 @@ -81,7 +80,7 @@ require ( github.com/pkg/browser v0.0.0-20210904010418-6d279e18f982 // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/alertmanager v0.23.1-0.20211116083607-e2a10119aaf7 - github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/client_golang v1.12.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.32.1 github.com/prometheus/prometheus v1.8.2-0.20211011171444-354d8d2ecfac @@ -94,7 +93,7 @@ require ( github.com/uber/jaeger-client-go v2.29.1+incompatible github.com/unknwon/com v1.0.1 github.com/urfave/cli/v2 v2.3.0 - github.com/vectordotdev/go-datemath v0.1.1-0.20211214152759-c03a58217724 + github.com/vectordotdev/go-datemath v0.1.1-0.20220110192739-f9ce83ec349f github.com/weaveworks/common v0.0.0-20210913144402-035033b78a78 github.com/xorcare/pointer v1.1.0 github.com/yudai/gojsondiff v1.0.0 @@ -111,7 +110,7 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac golang.org/x/tools v0.1.5 - gonum.org/v1/gonum v0.9.1 + gonum.org/v1/gonum v0.9.3 google.golang.org/api v0.58.0 google.golang.org/grpc v1.41.0 google.golang.org/protobuf v1.27.1 @@ -142,7 +141,7 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alecthomas/units v0.0.0-20210912230133-d1bdfacee922 // indirect github.com/andybalholm/brotli v1.0.3 - github.com/apache/arrow/go/arrow v0.0.0-20210223225224-5bea62493d91 // indirect + github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect github.com/armon/go-metrics v0.3.8 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -154,7 +153,7 @@ require ( github.com/cheekybits/genny v1.0.0 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect - github.com/deepmap/oapi-codegen v1.6.0 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dennwc/varint v1.0.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect @@ -184,7 +183,7 @@ require ( github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/google/btree v1.0.1 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect - github.com/googleapis/gax-go/v2 v2.1.1 + github.com/googleapis/gax-go/v2 v2.1.1 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect @@ -193,7 +192,7 @@ require ( github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/go-multierror v1.1.0 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/memberlist v0.2.4 // indirect github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect github.com/igm/sockjs-go/v3 v3.0.1 // indirect @@ -222,13 +221,13 @@ require ( github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/exporter-toolkit v0.7.0 // indirect github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/rs/cors v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect - github.com/segmentio/encoding v0.3.2 // indirect + github.com/segmentio/encoding v0.3.2 github.com/sercand/kuberesolver v2.4.0+incompatible // indirect github.com/sergi/go-diff v1.0.0 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect @@ -245,24 +244,24 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/goleak v1.1.10 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect + golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2 + google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect ) -require ( - cloud.google.com/go/kms v1.1.0 - github.com/Azure/go-autorest/autorest/adal v0.9.15 - github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f -) +require cloud.google.com/go/kms v1.1.0 require ( - github.com/containerd/containerd v1.5.8 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect + github.com/containerd/containerd v1.5.9 // indirect + github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/klauspost/compress v1.13.1 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/pierrec/lz4/v4 v4.1.8 // indirect github.com/segmentio/asm v1.1.1 // indirect ) diff --git a/go.sum b/go.sum index 8ee7b4e3291..8df53ae14bb 100644 --- a/go.sum +++ b/go.sum @@ -175,6 +175,7 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= @@ -276,10 +277,10 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/apache/arrow/go/arrow v0.0.0-20200629181129-68b1273cbbf7/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= -github.com/apache/arrow/go/arrow v0.0.0-20210223225224-5bea62493d91 h1:rbe942bXzd2vnds4y9fYQL8X4yFltXoZsKW7KtG+TFM= github.com/apache/arrow/go/arrow v0.0.0-20210223225224-5bea62493d91/go.mod h1:c9sxoIT3YgLxH4UhLOCKaBlEojuMhVYpk4Ntv3opUTQ= +github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ= +github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= github.com/apache/thrift v0.14.1/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= github.com/aristanetworks/goarista v0.0.0-20190325233358-a123909ec740/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= @@ -483,8 +484,8 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.4/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= -github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4= +github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -606,8 +607,9 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= -github.com/deepmap/oapi-codegen v1.6.0 h1:w/d1ntwh91XI0b/8ja7+u5SvA4IFfM0UNNLmiDR1gg0= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= @@ -770,6 +772,7 @@ github.com/gchaincl/sqlhooks v1.3.0/go.mod h1:9BypXnereMT0+Ys8WGWHqzgkkOfHIhyeUC github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/sentry-go v0.10.0 h1:6gwY+66NHKqyZrdi6O2jGdo7wGdo9b3B69E01NFgT5g= github.com/getsentry/sentry-go v0.10.0/go.mod h1:kELm/9iCblqUYh+ZRML7PNdCvEuw24wBvJPYyi86cws= @@ -1116,8 +1119,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -1232,23 +1236,23 @@ github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f h1:FvvSVEbnGeM2bUivG github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f/go.mod h1:uPG2nyK4CtgNDmWv7qyzYcdI+S90kHHRWvHnBtEMBXM= github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036 h1:GplhUk6Xes5JIhUUrggPcPBhOn+eT8+WsHiebvq7GgA= github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/grafana/grafana-aws-sdk v0.7.0 h1:D+Lhxi3P/7vpyDHUK/fdX9bL2mRz8hLG04ucNf1E02o= -github.com/grafana/grafana-aws-sdk v0.7.0/go.mod h1:+pPo5U+pX0zWimR7YBc7ASeSQfbRkcTyQYqMiAj7G5U= +github.com/grafana/grafana-aws-sdk v0.9.1 h1:jMZlsLsWnqOwLt2UNcLUsJ2z6289hLYlscK35QgS158= +github.com/grafana/grafana-aws-sdk v0.9.1/go.mod h1:6KaQ8uUD4KpXr/b7bAC7zbfSXTVOiTk4XhIrwkGWn4w= github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58 h1:2ud7NNM7LrGPO4x0NFR8qLq68CqI4SmB7I2yRN2w9oE= github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58/go.mod h1:Vo2TKWfDVmNTELBUM+3lkrZvFtBws0qSZdXhQxRdJrE= -github.com/grafana/grafana-plugin-sdk-go v0.79.0/go.mod h1:NvxLzGkVhnoBKwzkst6CFfpMFKwAdIUZ1q8ssuLeF60= +github.com/grafana/grafana-plugin-sdk-go v0.94.0/go.mod h1:3VXz4nCv6wH5SfgB3mlW39s+c+LetqSCjFj7xxPC5+M= github.com/grafana/grafana-plugin-sdk-go v0.114.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk= -github.com/grafana/grafana-plugin-sdk-go v0.121.0 h1:4+dXoezL9L40iu7ym4u7ZJ4OE57NaVc4WSHlbxtCtGM= -github.com/grafana/grafana-plugin-sdk-go v0.121.0/go.mod h1:Mhy+5mC6rSqEhnzop1wEh//n/fgkzNK5pRgg3DfCp4g= +github.com/grafana/grafana-plugin-sdk-go v0.125.0 h1:wK2zopAaKhVIMkXzgbExKqZtt+x2ZTGfcY+3wvOuyYQ= +github.com/grafana/grafana-plugin-sdk-go v0.125.0/go.mod h1:9YiJ5GUxIsIEUC0qR9+BJVP5M7mCSP6uc6Ne62YKkgc= github.com/grafana/loki v1.6.2-0.20211015002020-7832783b1caa h1:+pXjAxavVR2FKKNsuuCXGCWEj8XGc1Af6SPiyBpzU2A= github.com/grafana/loki v1.6.2-0.20211015002020-7832783b1caa/go.mod h1:0O8o/juxNSKN/e+DzWDTRkl7Zm8CkZcz0NDqEdojlrk= github.com/grafana/saml v0.0.0-20211007135653-aed1b2edd86b h1:YiSGp34F4V0G08HHx1cJBf2GVgwYAkXQjzuVs1t8jYk= github.com/grafana/saml v0.0.0-20211007135653-aed1b2edd86b/go.mod h1:q83kyQoMD0vhy+RzFLlbw0UgHJ6TAihQpuXvdFmm4s4= +github.com/grafana/sqlds/v2 v2.3.2/go.mod h1:34uyqPBWsEvg4V/xxh6V4uIqwu1qLfOfsmScll/ukrk= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= -github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= @@ -1422,8 +1426,6 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec h1:CGkYB1Q7DSsH/ku+to+foV4agt2F2miquaLUgF6L178= -github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.0/go.mod h1:BwN2XG2lMszOoquQaFdPET8FRQfrXiZsWmcMO9rkaVY= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= @@ -1441,8 +1443,9 @@ github.com/influxdata/influxdb v1.8.4/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbf github.com/influxdata/influxdb v1.8.5/go.mod h1:oFH+pbEyDln/1TKwa98oJzVrkZwdjrJOwIDGYZj7Ma0= github.com/influxdata/influxdb v1.9.2/go.mod h1:UEe3MeD9AaP5rlPIes102IhYua3FhIWZuOXNHxDjSrI= github.com/influxdata/influxdb v1.9.3/go.mod h1:xD4ZjAgEJQO9/bX3NhFrssKtdNPi+ki1kjrttJRDhGc= -github.com/influxdata/influxdb-client-go/v2 v2.3.1-0.20210518120617-5d1fff431040 h1:MBLCfcSsUyFPDJp6T7EoHp/Ph3Jkrm4EuUKLD2rUWHg= github.com/influxdata/influxdb-client-go/v2 v2.3.1-0.20210518120617-5d1fff431040/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxdb-client-go/v2 v2.6.0 h1:bIOaGTgvvv1Na2hG+nIvqyv7PK2UiU2WrJN1ck1ykyM= +github.com/influxdata/influxdb-client-go/v2 v2.6.0/go.mod h1:Y/0W1+TZir7ypoQZYd2IrnVOKB3Tq6oegAQeSVN/+EU= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxql v1.1.0/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= @@ -1561,6 +1564,7 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -1631,8 +1635,6 @@ github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkV github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/m3db/prometheus_remote_client_golang v0.4.4 h1:DsAIjVKoCp7Ym35tAOFL1OuMLIdIikAEHeNPHY+yyM8= github.com/m3db/prometheus_remote_client_golang v0.4.4/go.mod h1:wHfVbA3eAK6dQvKjCkHhusWYegCk3bDGkA15zymSHdc= -github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls= github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -1657,7 +1659,6 @@ github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwM github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= -github.com/mattetti/filebuffer v1.0.0/go.mod h1:X6nyAIge2JGVmuJt2MFCqmHrb/5IHiphfHtot0s5cnI= github.com/mattetti/filebuffer v1.0.1 h1:gG7pyfnSIZCxdoKq+cPa8T0hhYtD9NxCdI4D7PTjRLM= github.com/mattetti/filebuffer v1.0.1/go.mod h1:YdMURNDOttIiruleeVr6f56OrMc+MydEnTcXwtkxNVs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -1684,7 +1685,6 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1841,7 +1841,6 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/olivere/elastic v6.2.35+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= @@ -1946,8 +1945,11 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.7/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= +github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= @@ -2002,8 +2004,9 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -2065,8 +2068,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0= github.com/prometheus/prometheus v0.0.0-20200609090129-a6600f564e3c/go.mod h1:S5n0C6tSgdnwWshBUceRx5G1OsjLv/EeZ9t3wIfEtsY= @@ -2349,8 +2353,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vectordotdev/go-datemath v0.1.1-0.20211214152759-c03a58217724 h1:v2/Zact72KD/a+HJNb0ES/8JV0Lhlp9PjPdZQGiaBoQ= -github.com/vectordotdev/go-datemath v0.1.1-0.20211214152759-c03a58217724/go.mod h1:PnwzbSst7KD3vpBzzlntZU5gjVa455Uqa5QPiKSYJzQ= +github.com/vectordotdev/go-datemath v0.1.1-0.20220110192739-f9ce83ec349f h1:2upw/ZfjkCKpc4k6DXg7lMfCSLkfw/8epV5/y2ZUQ8U= +github.com/vectordotdev/go-datemath v0.1.1-0.20220110192739-f9ce83ec349f/go.mod h1:PnwzbSst7KD3vpBzzlntZU5gjVa455Uqa5QPiKSYJzQ= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5/go.mod h1:ppEjwdhyy7Y31EnHRDm1JkChoC7LXIJ7Ex0VYLWtZtQ= github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= @@ -2913,8 +2917,9 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -3064,8 +3069,8 @@ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.9.1 h1:HCWmqqNoELL0RAQeKBXWtkp04mGk8koafcB4He6+uhc= -gonum.org/v1/gonum v0.9.1/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.9.3 h1:DnoIG+QAMaF5NvxnGe/oKsgKcAc6PcUyl8q0VetfQ8s= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= @@ -3193,6 +3198,7 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxH google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= diff --git a/latest.json b/latest.json index d403ab4ea75..91df80dab6b 100644 --- a/latest.json +++ b/latest.json @@ -1,4 +1,4 @@ { - "stable": "8.3.3", - "testing": "8.3.3" + "stable": "8.3.4", + "testing": "8.3.4" } diff --git a/package.json b/package.json index 63891ce2606..ddb51ba5b27 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "@grafana/e2e": "workspace:*", "@grafana/eslint-config": "2.5.1", "@grafana/toolkit": "workspace:*", - "@grafana/tsconfig": "^1.0.0-rc1", + "@grafana/tsconfig": "^1.1.0-rc1", "@lingui/cli": "3.13.0", "@lingui/macro": "3.13.0", "@microsoft/api-extractor": "7.19.4", @@ -125,10 +125,9 @@ "@types/lru-cache": "^5.1.0", "@types/mousetrap": "1.6.3", "@types/node": "16.11.19", - "@types/ol": "^6.5.1", "@types/papaparse": "5.3.1", "@types/pluralize": "^0.0.29", - "@types/prismjs": "1.16.6", + "@types/prismjs": "1.26.0", "@types/rc-time-picker": "^3", "@types/react": "17.0.38", "@types/react-beautiful-dnd": "13.1.2", @@ -147,29 +146,28 @@ "@types/semver": "7.3.9", "@types/slate": "0.47.2", "@types/slate-plain-serializer": "0.7.2", - "@types/slate-react": "0.22.5", "@types/testing-library__jest-dom": "5.14.2", "@types/testing-library__react-hooks": "^3.2.0", "@types/tinycolor2": "1.4.3", "@types/uuid": "8.3.4", - "@typescript-eslint/eslint-plugin": "5.9.0", - "@typescript-eslint/parser": "5.9.0", + "@typescript-eslint/eslint-plugin": "5.10.0", + "@typescript-eslint/parser": "5.10.0", "@wojtekmaj/enzyme-adapter-react-17": "0.6.6", "autoprefixer": "10.4.1", - "axios": "0.24.0", + "axios": "0.25.0", "babel-jest": "27.4.6", "babel-loader": "8.2.3", "babel-plugin-angularjs-annotate": "0.10.0", "babel-plugin-macros": "3.1.0", "copy-webpack-plugin": "9.0.1", "css-loader": "6.5.1", - "css-minimizer-webpack-plugin": "3.3.1", - "cypress": "9.2.0", + "css-minimizer-webpack-plugin": "3.4.1", + "cypress": "9.3.1", "enzyme": "3.11.0", "enzyme-to-json": "3.6.2", "eslint": "7.28.0", "eslint-config-prettier": "8.3.0", - "eslint-plugin-jsdoc": "37.6.1", + "eslint-plugin-jsdoc": "37.7.0", "eslint-plugin-lodash": "^7.2.0", "eslint-plugin-no-only-tests": "2.6.0", "eslint-plugin-prettier": "4.0.0", @@ -192,8 +190,8 @@ "jest-junit": "13.0.0", "jest-matcher-utils": "27.4.6", "lerna": "^4.0.0", - "lint-staged": "12.1.6", - "mini-css-extract-plugin": "2.4.6", + "lint-staged": "12.3.1", + "mini-css-extract-plugin": "2.5.2", "mutationobserver-shim": "0.3.7", "ngtemplate-loader": "2.1.0", "node-notifier": "^10.0.0", @@ -209,11 +207,11 @@ "react-test-renderer": "17.0.2", "redux-mock-store": "1.5.4", "rimraf": "3.0.2", - "sass": "1.47.0", + "sass": "1.49.0", "sass-loader": "12.4.0", - "sinon": "12.0.1", + "sinon": "13.0.0", "style-loader": "3.3.1", - "stylelint": "14.2.0", + "stylelint": "14.3.0", "stylelint-config-prettier": "9.0.3", "stylelint-config-sass-guidelines": "9.0.1", "terser-webpack-plugin": "5.3.0", @@ -221,9 +219,9 @@ "ts-jest": "27.1.2", "ts-loader": "9.2.6", "ts-node": "10.4.0", - "typescript": "4.5.4", + "typescript": "4.4.4", "wait-on": "6.0.0", - "webpack": "5.66.0", + "webpack": "5.68.0", "webpack-bundle-analyzer": "4.5.0", "webpack-cli": "4.9.1", "webpack-dev-server": "4.7.2", @@ -236,6 +234,7 @@ "@grafana/data": "workspace:*", "@grafana/e2e-selectors": "workspace:*", "@grafana/experimental": "0.0.2-canary.11", + "@grafana/google-sdk": "0.0.2", "@grafana/runtime": "workspace:*", "@grafana/schema": "workspace:*", "@grafana/slate-react": "0.22.10-grafana", @@ -244,7 +243,7 @@ "@kusto/monaco-kusto": "4.1.2", "@lingui/core": "3.13.0", "@lingui/react": "3.13.0", - "@opentelemetry/api": "1.0.4", + "@opentelemetry/api": "1.1.0", "@opentelemetry/exporter-collector": "0.25.0", "@opentelemetry/semantic-conventions": "1.0.1", "@popperjs/core": "2.11.2", @@ -258,15 +257,15 @@ "@react-stately/menu": "3.2.3", "@react-stately/tree": "3.2.0", "@reduxjs/toolkit": "1.7.1", - "@sentry/browser": "6.16.1", - "@sentry/types": "6.16.1", - "@sentry/utils": "6.16.1", - "@visx/event": "2.1.2", + "@sentry/browser": "6.17.2", + "@sentry/types": "6.17.2", + "@sentry/utils": "6.17.2", + "@visx/event": "2.6.0", "@visx/gradient": "2.1.0", "@visx/group": "2.1.0", "@visx/scale": "2.2.2", "@visx/shape": "2.4.0", - "@visx/tooltip": "2.2.2", + "@visx/tooltip": "2.8.0", "@welldone-software/why-did-you-render": "6.2.3", "angular": "1.8.2", "angular-bindonce": "0.3.1", @@ -313,12 +312,12 @@ "mousetrap": "1.6.5", "mousetrap-global-bind": "1.1.0", "moveable": "0.27.3", - "ol": "6.9.0", + "ol": "6.12.0", "papaparse": "5.3.1", "pluralize": "^8.0.0", "prismjs": "1.26.0", "prop-types": "15.8.1", - "rc-cascader": "1.5.0", + "rc-cascader": "3.2.1", "rc-drawer": "4.4.3", "rc-slider": "9.7.5", "rc-time-picker": "3.7.3", @@ -375,7 +374,7 @@ "@rushstack/node-core-library": "3.45.0", "@rushstack/rig-package": "0.3.7", "@rushstack/ts-command-line": "4.10.6", - "@storybook/react/webpack": "5.66.0" + "@storybook/react/webpack": "5.68.0" }, "workspaces": { "packages": [ diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 047ceef1562..1360df99d8d 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -28,10 +28,10 @@ "date-fns": "2.28.0", "eventemitter3": "4.0.7", "lodash": "4.17.21", - "marked": "4.0.9", + "marked": "4.0.10", "moment": "2.29.1", "moment-timezone": "0.5.34", - "ol": "6.9.0", + "ol": "6.12.0", "papaparse": "5.3.1", "react": "17.0.2", "react-dom": "17.0.2", @@ -67,11 +67,11 @@ "@types/tinycolor2": "1.4.3", "react-test-renderer": "17.0.2", "rimraf": "3.0.2", - "rollup": "2.63.0", + "rollup": "2.66.1", "rollup-plugin-sourcemaps": "0.6.3", "rollup-plugin-terser": "7.0.2", - "sinon": "12.0.1", + "sinon": "13.0.0", "tinycolor2": "1.4.2", - "typescript": "4.5.4" + "typescript": "4.4.4" } } diff --git a/packages/grafana-data/src/dataframe/DataFrameJSON.ts b/packages/grafana-data/src/dataframe/DataFrameJSON.ts index ae80255cf90..98fbd050903 100644 --- a/packages/grafana-data/src/dataframe/DataFrameJSON.ts +++ b/packages/grafana-data/src/dataframe/DataFrameJSON.ts @@ -153,7 +153,6 @@ export function dataFrameFromJSON(dto: DataFrameJSON): DataFrame { // Find the longest field length const length = data ? data.values.reduce((max, vals) => Math.max(max, vals.length), 0) : 0; - const fields = schema.fields.map((f, index) => { let buffer = data ? data.values[index] : []; let origLen = buffer.length; diff --git a/packages/grafana-data/src/dataframe/MutableDataFrame.ts b/packages/grafana-data/src/dataframe/MutableDataFrame.ts index a76873b64dc..2597aa17d8f 100644 --- a/packages/grafana-data/src/dataframe/MutableDataFrame.ts +++ b/packages/grafana-data/src/dataframe/MutableDataFrame.ts @@ -155,6 +155,27 @@ export class MutableDataFrame extends FunctionalVector implements Da } } + private parsers: Map any> | undefined = undefined; + + /** + * @deprecated unclear if this is actually used + */ + setParser(field: Field, parser: (v: string) => any) { + if (!this.parsers) { + this.parsers = new Map any>(); + } + this.parsers.set(field, parser); + return parser; + } + + private parseValue(field: Field, v: any): any { + let p = this.parsers?.get(field); + if (!p) { + p = this.setParser(field, makeFieldParser(v, field)); + } + return p(v); + } + /** * This will add each value to the corresponding column */ @@ -181,10 +202,7 @@ export class MutableDataFrame extends FunctionalVector implements Da const f = this.fields[i]; let v = row[i]; if (f.type !== FieldType.string && isString(v)) { - if (!f.parse) { - f.parse = makeFieldParser(v, f); - } - v = f.parse(v); + v = this.parseValue(f, v); } f.values.add(v); } @@ -200,10 +218,7 @@ export class MutableDataFrame extends FunctionalVector implements Da let val = obj[field.name]; if (field.type !== FieldType.string && isString(val)) { - if (!field.parse) { - field.parse = makeFieldParser(val, field); - } - val = field.parse(val); + val = this.parseValue(field, val); } if (val === undefined) { diff --git a/packages/grafana-data/src/dataframe/processDataFrame.ts b/packages/grafana-data/src/dataframe/processDataFrame.ts index 954e6f29780..d3478cac95b 100644 --- a/packages/grafana-data/src/dataframe/processDataFrame.ts +++ b/packages/grafana-data/src/dataframe/processDataFrame.ts @@ -308,7 +308,7 @@ export const isDataFrame = (data: any): data is DataFrame => data && data.hasOwn export function toDataFrame(data: any): DataFrame { if ('fields' in data) { // DataFrameDTO does not have length - if ('length' in data) { + if ('length' in data && data.fields[0]?.values?.get) { return data as DataFrame; } diff --git a/packages/grafana-data/src/field/displayProcessor.test.ts b/packages/grafana-data/src/field/displayProcessor.test.ts index 0c301915ba8..3bae9138192 100644 --- a/packages/grafana-data/src/field/displayProcessor.test.ts +++ b/packages/grafana-data/src/field/displayProcessor.test.ts @@ -145,6 +145,39 @@ describe('Format value', () => { expect(result.text).toEqual('10.0'); }); + it('should return icon value if there are matching value mappings', () => { + const valueMappings: ValueMapping[] = [ + { type: MappingType.ValueToText, options: { '11': { text: 'elva', icon: 'windmill.svg' } } }, + ]; + + const display = getDisplayProcessorFromConfig({ decimals: 1, mappings: valueMappings }); + const result = display('11'); + + expect(result.icon).toEqual('windmill.svg'); + }); + + it('should return icon value if there are matching value mappings in a range', () => { + const valueMappings: ValueMapping[] = [ + { type: MappingType.RangeToText, options: { from: 1, to: 9, result: { text: '1-9', icon: 'drone.svg' } } }, + ]; + + const display = getDisplayProcessorFromConfig({ decimals: 1, mappings: valueMappings }); + const result = display('8'); + + expect(result.icon).toEqual('drone.svg'); + }); + + it('should return undefined icon value if there are no matching value mappings in a range', () => { + const valueMappings: ValueMapping[] = [ + { type: MappingType.ValueToText, options: { '11': { text: 'elva', icon: 'windmill.svg' } } }, + ]; + + const display = getDisplayProcessorFromConfig({ decimals: 1, mappings: valueMappings }); + const result = display('10'); + + expect(result.icon).toEqual(undefined); + }); + it('should return mapped value if there are matching value mappings', () => { const valueMappings: ValueMapping[] = [ { type: MappingType.ValueToText, options: { '11': { text: 'elva' } } }, diff --git a/packages/grafana-data/src/field/displayProcessor.ts b/packages/grafana-data/src/field/displayProcessor.ts index 6abc7202714..32c92e08072 100644 --- a/packages/grafana-data/src/field/displayProcessor.ts +++ b/packages/grafana-data/src/field/displayProcessor.ts @@ -84,6 +84,7 @@ export function getDisplayProcessor(options?: DisplayProcessorOptions): DisplayP let prefix: string | undefined; let suffix: string | undefined; let color: string | undefined; + let icon: string | undefined; let percent: number | undefined; if (mappings && mappings.length > 0) { @@ -97,6 +98,10 @@ export function getDisplayProcessor(options?: DisplayProcessorOptions): DisplayP if (mappingResult.color != null) { color = options.theme.visualization.getColorByName(mappingResult.color); } + + if (mappingResult.icon != null) { + icon = mappingResult.icon; + } } } @@ -137,7 +142,23 @@ export function getDisplayProcessor(options?: DisplayProcessorOptions): DisplayP percent = scaleResult.percent; } - return { text, numeric, prefix, suffix, color, percent }; + const display: DisplayValue = { + text, + numeric, + prefix, + suffix, + color, + }; + + if (icon != null) { + display.icon = icon; + } + + if (percent != null) { + display.percent = percent; + } + + return display; }; } diff --git a/packages/grafana-data/src/field/fieldOverrides.test.ts b/packages/grafana-data/src/field/fieldOverrides.test.ts index 3aa26b74b28..ca67d49af23 100644 --- a/packages/grafana-data/src/field/fieldOverrides.test.ts +++ b/packages/grafana-data/src/field/fieldOverrides.test.ts @@ -644,7 +644,7 @@ describe('getLinksSupplier', () => { expect(links[0]).toEqual( expect.objectContaining({ title: 'testDS', - href: `/explore?left=${encodeURIComponent('{"datasource":"testDS","queries":["12345"]}')}`, + href: `/explore?left=${encodeURIComponent('{"datasource":"testDS","queries":["12345"],"panelsState":{}}')}`, onClick: undefined, }) ); diff --git a/packages/grafana-data/src/geo/layer.ts b/packages/grafana-data/src/geo/layer.ts index 48757bcebd4..080f0035a4d 100644 --- a/packages/grafana-data/src/geo/layer.ts +++ b/packages/grafana-data/src/geo/layer.ts @@ -1,10 +1,10 @@ import { RegistryItemWithOptions } from '../utils/Registry'; -import BaseLayer from 'ol/layer/Base'; -import Map from 'ol/Map'; import { PanelData } from '../types'; import { GrafanaTheme2 } from '../themes'; import { PanelOptionsEditorBuilder } from '../utils'; import { ReactNode } from 'react'; +import { PluggableMap } from 'ol'; +import BaseLayer from 'ol/layer/Base'; /** * @alpha @@ -105,5 +105,5 @@ export interface MapLayerRegistryItem extends Registr * Function that configures transformation and returns a transformer * @param options */ - create: (map: Map, options: MapLayerOptions, theme: GrafanaTheme2) => Promise; + create: (map: PluggableMap, options: MapLayerOptions, theme: GrafanaTheme2) => Promise; } diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts index 967bdccaa38..dbfd943df6e 100644 --- a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts @@ -82,7 +82,7 @@ export function convertFieldTypes(options: ConvertFieldTypeTransformerOptions, f } /** - * Convert a single field type to specifed field type. + * Convert a single field type to specified field type. * @param field - field to convert * @param opts - field conversion options * @returns converted field diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index 6652199abb6..ed2066063e6 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -28,5 +28,6 @@ export enum DataTransformerID { prepareTimeSeries = 'prepareTimeSeries', convertFieldType = 'convertFieldType', fieldLookup = 'fieldLookup', + spatial = 'spatial', extractFields = 'extractFields', } diff --git a/packages/grafana-data/src/types/dataFrame.ts b/packages/grafana-data/src/types/dataFrame.ts index 0190e75fbfc..bb4e9784e34 100644 --- a/packages/grafana-data/src/types/dataFrame.ts +++ b/packages/grafana-data/src/types/dataFrame.ts @@ -15,6 +15,7 @@ export enum FieldType { boolean = 'boolean', // Used to detect that the value is some kind of trace data to help with the visualisation and processing. trace = 'trace', + geo = 'geo', other = 'other', // Object, Array, etc } @@ -121,11 +122,6 @@ export interface Field> { */ state?: FieldState | null; - /** - * Convert text to the field value - */ - parse?: (value: any) => T; - /** * Convert a value for display */ diff --git a/packages/grafana-data/src/types/dataLink.ts b/packages/grafana-data/src/types/dataLink.ts index 2d69aedfc7a..a0c6ac312e2 100644 --- a/packages/grafana-data/src/types/dataLink.ts +++ b/packages/grafana-data/src/types/dataLink.ts @@ -1,5 +1,6 @@ import { DataQuery } from './query'; import { InterpolateFunction } from './panel'; +import { ExplorePanelsState } from './explore'; /** * Callback info for DataLink click events @@ -44,6 +45,7 @@ export interface InternalDataLink { query: T; datasourceUid: string; datasourceName: string; + panelsState?: ExplorePanelsState; } export type LinkTarget = '_blank' | '_self' | undefined; diff --git a/packages/grafana-data/src/types/displayValue.ts b/packages/grafana-data/src/types/displayValue.ts index 7fb97183e8e..2effad55d78 100644 --- a/packages/grafana-data/src/types/displayValue.ts +++ b/packages/grafana-data/src/types/displayValue.ts @@ -12,9 +12,13 @@ export interface DisplayValue extends FormattedValue { */ percent?: number; /** - * Color based on configs or Threshold + * Color based on mappings or threshold */ color?: string; + /** + * Icon based on mappings or threshold + */ + icon?: string; title?: string; /** diff --git a/packages/grafana-data/src/types/explore.ts b/packages/grafana-data/src/types/explore.ts index 335c65e60e2..cd115a42a74 100644 --- a/packages/grafana-data/src/types/explore.ts +++ b/packages/grafana-data/src/types/explore.ts @@ -1,3 +1,4 @@ +import { PreferredVisualisationType } from './data'; import { DataQuery } from './query'; import { RawTimeRange, TimeRange } from './time'; @@ -10,11 +11,20 @@ export interface ExploreUrlState { range: RawTimeRange; originPanelId?: number; context?: string; + panelsState?: ExplorePanelsState; +} + +export interface ExplorePanelsState extends Partial> { + trace?: ExploreTracePanelState; +} + +export interface ExploreTracePanelState { + spanId?: string; } /** * SplitOpen type is used in Explore and related components. */ export type SplitOpen = ( - options?: { datasourceUid: string; query: T; range?: TimeRange } | undefined + options?: { datasourceUid: string; query: T; range?: TimeRange; panelsState?: ExplorePanelsState } | undefined ) => void; diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 94d9a8b6eb3..971014422c9 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -1,3 +1,9 @@ +// NOTE: This file was auto generated. DO NOT EDIT DIRECTLY! +// To change feature flags, edit: +// pkg/services/featuremgmt/registry.go +// Then run tests in: +// pkg/services/featuremgmt/toggles_gen_test.go + /** * Describes available feature toggles in Grafana. These can be configured via * conf/custom.ini to enable features under development or not yet available in @@ -10,13 +16,6 @@ export interface FeatureToggles { [name: string]: boolean | undefined; // support any string value - recordedQueries?: boolean; - teamsync?: boolean; - ldapsync?: boolean; - caching?: boolean; - dspermissions?: boolean; - analytics?: boolean; - ['enterprise.plugins']?: boolean; trimDefaults?: boolean; envelopeEncryption?: boolean; httpclientprovider_azure_auth?: boolean; @@ -36,4 +35,5 @@ export interface FeatureToggles { newNavigation?: boolean; showFeatureFlagsInUI?: boolean; disable_http_request_histogram?: boolean; + validatedQueries?: boolean; } diff --git a/packages/grafana-data/src/types/navModel.ts b/packages/grafana-data/src/types/navModel.ts index 23457500d37..d4c885f11a5 100644 --- a/packages/grafana-data/src/types/navModel.ts +++ b/packages/grafana-data/src/types/navModel.ts @@ -1,3 +1,5 @@ +import { ComponentType } from 'react'; + export interface NavModelItem { text: string; url?: string; @@ -17,6 +19,8 @@ export interface NavModelItem { showOrgSwitcher?: boolean; onClick?: () => void; menuItemType?: NavMenuItemType; + highlightText?: string; + tabSuffix?: ComponentType<{ className?: string }>; } export enum NavSection { diff --git a/packages/grafana-data/src/types/valueMapping.ts b/packages/grafana-data/src/types/valueMapping.ts index 6a4b373d305..463137c6bc8 100644 --- a/packages/grafana-data/src/types/valueMapping.ts +++ b/packages/grafana-data/src/types/valueMapping.ts @@ -14,6 +14,7 @@ export enum MappingType { export interface ValueMappingResult { text?: string; color?: string; + icon?: string; index?: number; } diff --git a/packages/grafana-data/src/utils/dataLinks.test.ts b/packages/grafana-data/src/utils/dataLinks.test.ts index daab192e387..b609ed2827b 100644 --- a/packages/grafana-data/src/utils/dataLinks.test.ts +++ b/packages/grafana-data/src/utils/dataLinks.test.ts @@ -1,5 +1,5 @@ import { mapInternalLinkToExplore } from './dataLinks'; -import { FieldType } from '../types'; +import { DataLink, FieldType } from '../types'; import { ArrayVector } from '../vector'; describe('mapInternalLinkToExplore', () => { @@ -31,7 +31,52 @@ describe('mapInternalLinkToExplore', () => { expect(link).toEqual( expect.objectContaining({ title: 'dsName', - href: `/explore?left=${encodeURIComponent('{"datasource":"dsName","queries":[{"query":"12344"}]}')}`, + href: `/explore?left=${encodeURIComponent( + '{"datasource":"dsName","queries":[{"query":"12344"}],"panelsState":{}}' + )}`, + onClick: undefined, + }) + ); + }); + + it('includes panels state', () => { + const panelsState = { + trace: { + spanId: 'abcdef', + }, + }; + + const dataLink: DataLink = { + url: '', + title: '', + internal: { + datasourceUid: 'uid', + datasourceName: 'dsName', + query: { query: '12344' }, + panelsState, + }, + }; + + const link = mapInternalLinkToExplore({ + link: dataLink, + internalLink: dataLink.internal!, + scopedVars: {}, + range: {} as any, + field: { + name: 'test', + type: FieldType.number, + config: {}, + values: new ArrayVector([2]), + }, + replaceVariables: (val) => val, + }); + + expect(link).toEqual( + expect.objectContaining({ + title: 'dsName', + href: `/explore?left=${encodeURIComponent( + '{"datasource":"dsName","queries":[{"query":"12344"}],"panelsState":{"trace":{"spanId":"abcdef"}}}' + )}`, onClick: undefined, }) ); diff --git a/packages/grafana-data/src/utils/dataLinks.ts b/packages/grafana-data/src/utils/dataLinks.ts index 593799d293d..ce279fe5421 100644 --- a/packages/grafana-data/src/utils/dataLinks.ts +++ b/packages/grafana-data/src/utils/dataLinks.ts @@ -1,11 +1,13 @@ import { DataLink, DataQuery, + ExplorePanelsState, Field, InternalDataLink, InterpolateFunction, LinkModel, ScopedVars, + SplitOpen, TimeRange, } from '../types'; import { locationUtil } from './location'; @@ -33,26 +35,28 @@ export type LinkToExploreOptions = { range: TimeRange; field: Field; internalLink: InternalDataLink; - onClickFn?: (options: { datasourceUid: string; query: any; range?: TimeRange }) => void; + onClickFn?: SplitOpen; replaceVariables: InterpolateFunction; }; export function mapInternalLinkToExplore(options: LinkToExploreOptions): LinkModel { const { onClickFn, replaceVariables, link, scopedVars, range, field, internalLink } = options; - const interpolatedQuery = interpolateQuery(link, scopedVars, replaceVariables); + const interpolatedQuery = interpolateObject(link.internal?.query, scopedVars, replaceVariables); + const interpolatedPanelsState = interpolateObject(link.internal?.panelsState, scopedVars, replaceVariables); const title = link.title ? link.title : internalLink.datasourceName; return { title: replaceVariables(title, scopedVars), // In this case this is meant to be internal link (opens split view by default) the href will also points // to explore but this way you can open it in new tab. - href: generateInternalHref(internalLink.datasourceName, interpolatedQuery, range), + href: generateInternalHref(internalLink.datasourceName, interpolatedQuery, range, interpolatedPanelsState), onClick: onClickFn ? () => { onClickFn({ datasourceUid: internalLink.datasourceUid, query: interpolatedQuery, + panelsState: interpolatedPanelsState, range, }); } @@ -65,26 +69,32 @@ export function mapInternalLinkToExplore(options: LinkToExploreOptions): LinkMod /** * Generates href for internal derived field link. */ -function generateInternalHref(datasourceName: string, query: T, range: TimeRange): string { +function generateInternalHref( + datasourceName: string, + query: T, + range: TimeRange, + panelsState?: ExplorePanelsState +): string { return locationUtil.assureBaseUrl( `/explore?left=${encodeURIComponent( serializeStateToUrlParam({ range: range.raw, datasource: datasourceName, queries: [query], + panelsState: panelsState, }) )}` ); } -function interpolateQuery( - link: DataLink, +function interpolateObject( + object: T | undefined, scopedVars: ScopedVars, replaceVariables: InterpolateFunction ): T { let stringifiedQuery = ''; try { - stringifiedQuery = JSON.stringify(link.internal?.query || ''); + stringifiedQuery = JSON.stringify(object || {}); } catch (err) { // should not happen and not much to do about this, possibly something non stringifiable in the query console.error(err); diff --git a/packages/grafana-data/src/utils/url.ts b/packages/grafana-data/src/utils/url.ts index 3e53a71578d..7cf29ed8e7f 100644 --- a/packages/grafana-data/src/utils/url.ts +++ b/packages/grafana-data/src/utils/url.ts @@ -195,9 +195,16 @@ export const urlUtil = { parseKeyValue, }; +/** + * Create an string that is used in URL to represent the Explore state. This is basically just a stringified json + * that is that used as a state of a single Explore pane so it does not represent full Explore URL. + * + * @param urlState + * @param compact this parameter is deprecated and will be removed in a future release. + */ export function serializeStateToUrlParam(urlState: ExploreUrlState, compact?: boolean): string { - if (compact) { - return JSON.stringify([urlState.range.from, urlState.range.to, urlState.datasource, ...urlState.queries]); + if (compact !== undefined) { + console.warn('`compact` parameter is deprecated and will be removed in a future release'); } return JSON.stringify(urlState); } diff --git a/packages/grafana-data/src/valueFormats/categories.ts b/packages/grafana-data/src/valueFormats/categories.ts index 1bc51510107..e7c2aa8aef9 100644 --- a/packages/grafana-data/src/valueFormats/categories.ts +++ b/packages/grafana-data/src/valueFormats/categories.ts @@ -171,8 +171,8 @@ export const getCategories = (): ValueFormatCategory[] => [ { name: 'kibibits/sec', id: 'Kibits', fn: binaryPrefix('b/s', 1) }, { name: 'kilobytes/sec', id: 'KBs', fn: SIPrefix('B/s', 1) }, { name: 'kilobits/sec', id: 'Kbits', fn: SIPrefix('b/s', 1) }, - { name: 'mibibytes/sec', id: 'MiBs', fn: binaryPrefix('B/s', 2) }, - { name: 'mibibits/sec', id: 'Mibits', fn: binaryPrefix('b/s', 2) }, + { name: 'mebibytes/sec', id: 'MiBs', fn: binaryPrefix('B/s', 2) }, + { name: 'mebibits/sec', id: 'Mibits', fn: binaryPrefix('b/s', 2) }, { name: 'megabytes/sec', id: 'MBs', fn: SIPrefix('B/s', 2) }, { name: 'megabits/sec', id: 'Mbits', fn: SIPrefix('b/s', 2) }, { name: 'gibibytes/sec', id: 'GiBs', fn: binaryPrefix('B/s', 3) }, diff --git a/packages/grafana-data/src/valueFormats/dateTimeFormatters.ts b/packages/grafana-data/src/valueFormats/dateTimeFormatters.ts index c56dd5fe34a..3f95589d5c3 100644 --- a/packages/grafana-data/src/valueFormats/dateTimeFormatters.ts +++ b/packages/grafana-data/src/valueFormats/dateTimeFormatters.ts @@ -20,6 +20,17 @@ export enum Interval { Millisecond = 'millisecond', } +const UNITS = [ + Interval.Year, + Interval.Month, + Interval.Week, + Interval.Day, + Interval.Hour, + Interval.Minute, + Interval.Second, + Interval.Millisecond, +]; + const INTERVALS_IN_SECONDS: IntervalsInSeconds = { [Interval.Year]: 31536000, [Interval.Month]: 2592000, @@ -206,17 +217,6 @@ export function toDuration(size: number, decimals: DecimalCount, timeScale: Inte return v; } - const units = [ - { long: Interval.Year }, - { long: Interval.Month }, - { long: Interval.Week }, - { long: Interval.Day }, - { long: Interval.Hour }, - { long: Interval.Minute }, - { long: Interval.Second }, - { long: Interval.Millisecond }, - ]; - // convert $size to milliseconds // intervals_in_seconds uses seconds (duh), convert them to milliseconds here to minimize floating point errors size *= INTERVALS_IN_SECONDS[timeScale] * 1000; @@ -231,13 +231,13 @@ export function toDuration(size: number, decimals: DecimalCount, timeScale: Inte decimalsCount = decimals as number; } - for (let i = 0; i < units.length && decimalsCount >= 0; i++) { - const interval = INTERVALS_IN_SECONDS[units[i].long] * 1000; + for (let i = 0; i < UNITS.length && decimalsCount >= 0; i++) { + const interval = INTERVALS_IN_SECONDS[UNITS[i]] * 1000; const value = size / interval; if (value >= 1 || decrementDecimals) { decrementDecimals = true; const floor = Math.floor(value); - const unit = units[i].long + (floor !== 1 ? 's' : ''); + const unit = UNITS[i] + (floor !== 1 ? 's' : ''); strings.push(floor + ' ' + unit); size = size % interval; decimalsCount--; diff --git a/packages/grafana-e2e-selectors/package.json b/packages/grafana-e2e-selectors/package.json index cbdda684be3..195c0b8d964 100644 --- a/packages/grafana-e2e-selectors/package.json +++ b/packages/grafana-e2e-selectors/package.json @@ -28,7 +28,7 @@ "@rollup/plugin-node-resolve": "13.1.3", "@types/node": "16.11.19", "rimraf": "3.0.2", - "rollup": "2.63.0", + "rollup": "2.66.1", "rollup-plugin-sourcemaps": "0.6.3", "rollup-plugin-terser": "7.0.2" }, @@ -36,6 +36,6 @@ "dependencies": { "@grafana/tsconfig": "^1.0.0-rc1", "tslib": "2.3.1", - "typescript": "4.5.4" + "typescript": "4.4.4" } } diff --git a/packages/grafana-e2e-selectors/src/selectors/components.ts b/packages/grafana-e2e-selectors/src/selectors/components.ts index a69a6b0c36a..26f2fc7bd91 100644 --- a/packages/grafana-e2e-selectors/src/selectors/components.ts +++ b/packages/grafana-e2e-selectors/src/selectors/components.ts @@ -35,6 +35,9 @@ export const Components = { startValue: 'TestData start value', }, }, + DataSourceHttpSettings: { + urlInput: 'Datasource HTTP settings url', + }, Jaeger: { traceIDInput: 'Trace ID', }, @@ -322,4 +325,10 @@ export const Components = { DashboardRow: { title: (title: string) => `data-testid dashboard-row-title-${title}`, }, + UserProfile: { + profileSaveButton: 'data-testid-user-profile-save', + preferencesSaveButton: 'data-testid-shared-prefs-save', + orgsTable: 'data-testid-user-orgs-table', + sessionsTable: 'data-testid-user-sessions-table', + }, }; diff --git a/packages/grafana-e2e/package.json b/packages/grafana-e2e/package.json index ad965d1c0c8..337a953666e 100644 --- a/packages/grafana-e2e/package.json +++ b/packages/grafana-e2e/package.json @@ -37,11 +37,11 @@ "@types/lodash": "4.14.149", "@types/node": "16.11.19", "@types/uuid": "8.3.4", - "rollup": "2.63.0", + "rollup": "2.66.1", "rollup-plugin-copy": "3.4.0", "rollup-plugin-sourcemaps": "0.6.3", "rollup-plugin-terser": "7.0.2", - "webpack": "5.66.0" + "webpack": "5.68.0" }, "types": "src/index.ts", "dependencies": { @@ -55,18 +55,18 @@ "blink-diff": "1.0.13", "chrome-remote-interface": "0.31.1", "commander": "8.3.0", - "cypress": "9.2.0", + "cypress": "9.3.1", "cypress-file-upload": "5.0.8", "devtools-protocol": "0.0.927104", "execa": "5.1.1", "lodash": "4.17.21", - "mocha": "9.1.3", + "mocha": "9.2.0", "resolve-as-bin": "2.1.0", "rimraf": "3.0.2", "tracelib": "1.0.1", "ts-loader": "6.2.1", "tslib": "2.3.1", - "typescript": "4.5.4", + "typescript": "4.4.4", "uuid": "8.3.2", "yaml": "^1.8.3" } diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index 9a573c347e2..8de4dbe08e2 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -25,7 +25,7 @@ "@grafana/data": "8.4.0-pre", "@grafana/e2e-selectors": "8.4.0-pre", "@grafana/ui": "8.4.0-pre", - "@sentry/browser": "6.16.1", + "@sentry/browser": "6.17.2", "history": "4.10.1", "lodash": "4.17.21", "react": "17.0.2", @@ -50,10 +50,10 @@ "@types/systemjs": "^0.20.6", "lodash": "4.17.21", "rimraf": "3.0.2", - "rollup": "2.63.0", + "rollup": "2.66.1", "rollup-plugin-sourcemaps": "0.6.3", "rollup-plugin-terser": "7.0.2", - "typescript": "4.5.4" + "typescript": "4.4.4" }, "types": "src/index.ts" } diff --git a/packages/grafana-runtime/src/config.ts b/packages/grafana-runtime/src/config.ts index da3989ad381..ec4825f27c1 100644 --- a/packages/grafana-runtime/src/config.ts +++ b/packages/grafana-runtime/src/config.ts @@ -93,7 +93,7 @@ export class GrafanaBootConfig implements GrafanaConfig { applicationInsightsConnectionString?: string; applicationInsightsEndpointUrl?: string; recordedQueries = { - enabled: false, + enabled: true, }; featureHighlights = { enabled: false, diff --git a/packages/grafana-runtime/src/index.ts b/packages/grafana-runtime/src/index.ts index 4e2b6f35a1f..d7e297a97e0 100644 --- a/packages/grafana-runtime/src/index.ts +++ b/packages/grafana-runtime/src/index.ts @@ -7,7 +7,7 @@ export * from './services'; export * from './config'; export * from './types'; export { loadPluginCss, SystemJS, PluginCssOptions } from './utils/plugin'; -export { reportMetaAnalytics, reportInteraction, reportPageview } from './utils/analytics'; +export { reportMetaAnalytics, reportInteraction, reportPageview, reportExperimentView } from './utils/analytics'; export { featureEnabled } from './utils/licensing'; export { logInfo, logDebug, logWarning, logError } from './utils/logging'; export { diff --git a/packages/grafana-runtime/src/services/EchoSrv.ts b/packages/grafana-runtime/src/services/EchoSrv.ts index 7e8ae524e64..9be1d332723 100644 --- a/packages/grafana-runtime/src/services/EchoSrv.ts +++ b/packages/grafana-runtime/src/services/EchoSrv.ts @@ -81,6 +81,7 @@ export enum EchoEventType { Sentry = 'sentry', Pageview = 'pageview', Interaction = 'interaction', + ExperimentView = 'experimentview', } /** diff --git a/packages/grafana-runtime/src/services/dataSourceSrv.ts b/packages/grafana-runtime/src/services/dataSourceSrv.ts index 31baf9a9f5e..e23c09d5835 100644 --- a/packages/grafana-runtime/src/services/dataSourceSrv.ts +++ b/packages/grafana-runtime/src/services/dataSourceSrv.ts @@ -28,6 +28,11 @@ export interface DataSourceSrv { ref?: DataSourceRef | string | null, scopedVars?: ScopedVars ): DataSourceInstanceSettings | undefined; + + /** + * Reloads the DataSourceSrv + */ + reload(): void; } /** @public */ diff --git a/packages/grafana-runtime/src/types/analytics.ts b/packages/grafana-runtime/src/types/analytics.ts index 99e158e2d6d..7a658d07d3f 100644 --- a/packages/grafana-runtime/src/types/analytics.ts +++ b/packages/grafana-runtime/src/types/analytics.ts @@ -104,6 +104,24 @@ export interface InteractionEchoEventPayload { */ export type InteractionEchoEvent = EchoEvent; +/** + * Describes the payload of an experimentview event. + * + * @public + */ +export interface ExperimentViewEchoEventPayload { + experimentId: string; + experimentGroup: string; + experimentVariant: string; +} + +/** + * Describes experimentview event with predefined {@link EchoEventType.EchoEventType} type. + * + * @public + */ +export type ExperimentViewEchoEvent = EchoEvent; + /** * Pageview event typeguard. * @@ -121,3 +139,12 @@ export const isPageviewEvent = (event: EchoEvent): event is PageviewEchoEvent => export const isInteractionEvent = (event: EchoEvent): event is InteractionEchoEvent => { return Boolean(event.payload.interactionName); }; + +/** + * Experimentview event typeguard. + * + * @public + */ +export const isExperimentViewEvent = (event: EchoEvent): event is ExperimentViewEchoEvent => { + return Boolean(event.payload.experimentId); +}; diff --git a/packages/grafana-runtime/src/utils/analytics.ts b/packages/grafana-runtime/src/utils/analytics.ts index e864cd5c4b2..3488de8eaf2 100644 --- a/packages/grafana-runtime/src/utils/analytics.ts +++ b/packages/grafana-runtime/src/utils/analytics.ts @@ -1,5 +1,6 @@ import { getEchoSrv, EchoEventType } from '../services/EchoSrv'; import { + ExperimentViewEchoEvent, InteractionEchoEvent, MetaAnalyticsEvent, MetaAnalyticsEventPayload, @@ -50,3 +51,19 @@ export const reportInteraction = (interactionName: string, properties?: Record { + getEchoSrv().addEvent({ + type: EchoEventType.ExperimentView, + payload: { + experimentId: id, + experimentGroup: group, + experimentVariant: variant, + }, + }); +}; diff --git a/packages/grafana-schema/package.json b/packages/grafana-schema/package.json index 174aff5e0c7..204550b89ed 100644 --- a/packages/grafana-schema/package.json +++ b/packages/grafana-schema/package.json @@ -28,10 +28,10 @@ "@rollup/plugin-node-resolve": "13.1.3", "@swc/helpers": "0.3.2", "rimraf": "3.0.2", - "rollup": "2.63.0", + "rollup": "2.66.1", "rollup-plugin-sourcemaps": "0.6.3", "rollup-plugin-terser": "7.0.2", - "typescript": "4.5.4" + "typescript": "4.4.4" }, "dependencies": { "tslib": "2.3.1" diff --git a/packages/grafana-toolkit/README.md b/packages/grafana-toolkit/README.md index bbb06243b36..67f653a11a5 100644 --- a/packages/grafana-toolkit/README.md +++ b/packages/grafana-toolkit/README.md @@ -15,6 +15,8 @@ yarn install yarn dev ``` +> **Note:** If running NPM 7+ the `npx` commands mentioned in this article may hang. The workaround is to use `npx --legacy-peer-deps `. + ## Update your plugin to use grafana-toolkit Follow the steps below to start using grafana-toolkit in your existing plugin. diff --git a/packages/grafana-toolkit/src/cli/index.ts b/packages/grafana-toolkit/src/cli/index.ts index 797684d4d9d..f8ec7bc945d 100644 --- a/packages/grafana-toolkit/src/cli/index.ts +++ b/packages/grafana-toolkit/src/cli/index.ts @@ -1,7 +1,6 @@ // @ts-ignore import chalk from 'chalk'; import { program } from 'commander'; -import { promises as fs } from 'fs'; import { execTask } from './utils/execTask'; import { startTask } from './tasks/core.start'; import { changelogTask } from './tasks/changelog'; @@ -13,7 +12,7 @@ import { pluginTestTask } from './tasks/plugin.tests'; import { searchTestDataSetupTask } from './tasks/searchTestDataSetup'; import { closeMilestoneTask } from './tasks/closeMilestone'; import { pluginDevTask } from './tasks/plugin.dev'; -import { githubPublishTask } from './tasks/plugin.utils'; +import { getToolkitVersion, githubPublishTask } from './tasks/plugin.utils'; import { pluginUpdateTask } from './tasks/plugin.update'; import { ciBuildPluginTask, ciPackagePluginTask, ciPluginReportTask } from './tasks/plugin.ci'; import { buildPackageTask } from './tasks/package.build'; @@ -132,8 +131,7 @@ export const run = (includeInternalScripts = false) => { } program.option('-v, --version', 'Toolkit version').action(async () => { - const pkg = await fs.readFile(`${__dirname}/../../package.json`, 'utf8'); - const { version } = JSON.parse(pkg); + const version = getToolkitVersion(); console.log(`v${version}`); }); diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.sign.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.sign.ts index 1a9d2a5b925..c8c566a88b6 100644 --- a/packages/grafana-toolkit/src/cli/tasks/plugin.sign.ts +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.sign.ts @@ -1,7 +1,7 @@ import path from 'path'; import { buildManifest, signManifest, saveManifest } from '../../plugins/manifest'; import { Task, TaskRunner } from './task'; - +import { getToolkitVersion } from './plugin.utils'; interface PluginSignOptions { signatureType?: string; rootUrls?: string[]; @@ -23,8 +23,8 @@ const pluginSignRunner: TaskRunner = async ({ signatureType, manifest.rootUrls = rootUrls; } + manifest.toolkit = { version: getToolkitVersion() }; const signedManifest = await signManifest(manifest); - // console.log(signedManifest); console.log('Saving signed manifest...'); await saveManifest(distContentDir, signedManifest); diff --git a/packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts b/packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts index 2ffeabfc784..ee32cf54177 100755 --- a/packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts +++ b/packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts @@ -4,6 +4,8 @@ import { GitHubRelease } from '../utils/githubRelease'; import { getPluginId } from '../../config/utils/getPluginId'; import { getCiFolder } from '../../plugins/env'; import { useSpinner } from '../utils/useSpinner'; +import { readFileSync } from 'fs'; + import path = require('path'); import execa = require('execa'); @@ -140,6 +142,14 @@ const prepareRelease = ({ dryrun, verbose }: any) => } }); +export const getToolkitVersion = () => { + const pkg = readFileSync(`${__dirname}/../../../package.json`, 'utf8'); + const { version } = JSON.parse(pkg); + if (!version) { + throw `Could not find the toolkit version`; + } + return version; +}; interface GithubPublishReleaseOptions { commitHash?: string; githubToken: string; diff --git a/packages/grafana-toolkit/src/config/webpack.plugin.config.ts b/packages/grafana-toolkit/src/config/webpack.plugin.config.ts index 9e6e323814c..9764abc0070 100644 --- a/packages/grafana-toolkit/src/config/webpack.plugin.config.ts +++ b/packages/grafana-toolkit/src/config/webpack.plugin.config.ts @@ -97,8 +97,9 @@ const getCommonPlugins = (options: WebpackConfigurationOptions) => { new CopyWebpackPlugin( [ // If src/README.md exists use it; otherwise the root README - { from: hasREADME ? 'README.md' : '../README.md', to: '.', force: true }, + { from: hasREADME ? 'README.md' : '../README.md', to: '.', force: true, prority: 1 }, { from: 'plugin.json', to: '.' }, + { from: '**/README.md', to: '[path]README.md', priority: 0 }, { from: '../LICENSE', to: '.' }, { from: '../CHANGELOG.md', to: '.', force: true }, { from: '**/*.json', to: '.' }, diff --git a/packages/grafana-toolkit/src/plugins/types.ts b/packages/grafana-toolkit/src/plugins/types.ts index 792fd09e4fd..296cbc8252a 100644 --- a/packages/grafana-toolkit/src/plugins/types.ts +++ b/packages/grafana-toolkit/src/plugins/types.ts @@ -98,4 +98,7 @@ export interface ManifestInfo { plugin: string; version: string; files: Record; + toolkit?: { + version: string; + }; } diff --git a/packages/grafana-ui/.storybook/main.ts b/packages/grafana-ui/.storybook/main.ts index a3fa12a8178..d7e254c7915 100644 --- a/packages/grafana-ui/.storybook/main.ts +++ b/packages/grafana-ui/.storybook/main.ts @@ -1,4 +1,5 @@ const path = require('path'); +const { ProvidePlugin } = require('webpack'); const TerserPlugin = require('terser-webpack-plugin'); const CssMinimizerPlugin = require('css-minimizer-webpack-plugin'); const FilterWarningsPlugin = require('webpack-filter-warnings-plugin'); @@ -48,11 +49,6 @@ module.exports = { webpackFinal: async (config: any, { configType }: any) => { const isProductionBuild = configType === 'PRODUCTION'; - config.resolve.fallback = { - ...(config.resolve.fallback || {}), - process: false, - }; - // remove svg from default storybook webpack 5 config so we can use `raw-loader` config.module.rules = config.module.rules.map((rule: any) => { if ( @@ -170,6 +166,34 @@ module.exports = { }) ); + config.plugins = fixProcessResolution(config.plugins); + + return config; + }, + // This webpack config only affects the manager (storybook UI) and generally shouldn't need to be changed. + // If stories are failing to build check `webpackFinal` property above. + managerWebpack: async (config: any) => { + config.plugins = fixProcessResolution(config.plugins); + return config; }, }; + +// Storybook 6.4.13 introduced https://github.com/storybookjs/storybook/pull/17213 +// which appears to prevent resolving correctly with yarn pnp. +// We can probably remove this in the next patch release. +function fixProcessResolution(plugins: any[]) { + plugins.forEach((p: any, i: number) => { + if (p.constructor.name === 'ProvidePlugin') { + plugins.splice( + i, + 1, + new ProvidePlugin({ + process: require.resolve('process/browser.js'), + }) + ); + } + }); + + return plugins; +} diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 5aa1434199c..6f51f50ec0e 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -44,7 +44,7 @@ "@react-aria/menu": "3.3.0", "@react-aria/overlays": "3.7.3", "@react-stately/menu": "3.2.3", - "@sentry/browser": "6.16.1", + "@sentry/browser": "6.17.2", "ansicolor": "1.1.95", "calculate-size": "1.1.1", "classnames": "2.3.1", @@ -59,14 +59,15 @@ "memoize-one": "6.0.0", "moment": "2.29.1", "monaco-editor": "^0.31.1", + "ol": "6.12.0", "prismjs": "1.26.0", - "rc-cascader": "1.5.0", + "rc-cascader": "3.2.1", "rc-drawer": "4.4.3", "rc-slider": "9.7.5", "rc-time-picker": "^3.7.3", "react": "17.0.2", "react-beautiful-dnd": "13.1.0", - "react-calendar": "3.5.0", + "react-calendar": "3.6.0", "react-colorful": "5.5.1", "react-custom-scrollbars-2": "4.4.0", "react-dom": "17.0.2", @@ -97,20 +98,20 @@ "@rollup/plugin-commonjs": "21.0.1", "@rollup/plugin-image": "2.1.1", "@rollup/plugin-node-resolve": "13.1.3", - "@storybook/addon-a11y": "6.4.9", - "@storybook/addon-actions": "6.4.9", - "@storybook/addon-docs": "6.4.9", - "@storybook/addon-essentials": "6.4.9", + "@storybook/addon-a11y": "6.4.13", + "@storybook/addon-actions": "6.4.13", + "@storybook/addon-docs": "6.4.13", + "@storybook/addon-essentials": "6.4.13", "@storybook/addon-knobs": "6.4.0", - "@storybook/addon-storysource": "6.4.9", - "@storybook/addons": "6.4.9", - "@storybook/api": "6.4.9", - "@storybook/builder-webpack5": "6.4.9", - "@storybook/components": "6.4.9", - "@storybook/core-events": "6.4.9", - "@storybook/manager-webpack5": "6.4.9", - "@storybook/react": "6.4.9", - "@storybook/theming": "6.4.9", + "@storybook/addon-storysource": "6.4.13", + "@storybook/addons": "6.4.13", + "@storybook/api": "6.4.13", + "@storybook/builder-webpack5": "6.4.13", + "@storybook/components": "6.4.13", + "@storybook/core-events": "6.4.13", + "@storybook/manager-webpack5": "6.4.13", + "@storybook/react": "6.4.13", + "@storybook/theming": "6.4.13", "@swc/helpers": "0.3.2", "@testing-library/dom": "8.11.1", "@testing-library/jest-dom": "5.16.1", @@ -130,7 +131,7 @@ "@types/lodash": "4.14.149", "@types/mock-raf": "1.0.3", "@types/node": "16.11.19", - "@types/prismjs": "1.16.6", + "@types/prismjs": "1.26.0", "@types/react": "17.0.38", "@types/react-beautiful-dnd": "13.1.2", "@types/react-calendar": "3.4.4", @@ -143,7 +144,6 @@ "@types/react-window": "1.8.5", "@types/slate": "0.47.2", "@types/slate-plain-serializer": "0.7.2", - "@types/slate-react": "0.22.5", "@types/testing-library__jest-dom": "5.14.2", "@types/testing-library__react-hooks": "^3.2.0", "@types/tinycolor2": "1.4.3", @@ -152,18 +152,19 @@ "babel-loader": "8.2.3", "common-tags": "1.8.2", "css-loader": "6.5.1", - "css-minimizer-webpack-plugin": "3.3.1", + "css-minimizer-webpack-plugin": "3.4.1", "csstype": "3.0.10", "enzyme": "3.11.0", "expose-loader": "3.1.0", "mock-raf": "1.0.1", "postcss": "8.4.5", "postcss-loader": "6.2.1", + "process": "^0.11.10", "raw-loader": "4.0.2", "react-docgen-typescript-loader": "3.7.2", "react-test-renderer": "17.0.2", "rimraf": "3.0.2", - "rollup": "2.63.0", + "rollup": "2.66.1", "rollup-plugin-sourcemaps": "0.6.3", "rollup-plugin-terser": "7.0.2", "sass-loader": "12.4.0", @@ -171,8 +172,8 @@ "style-loader": "3.3.1", "terser-webpack-plugin": "5.3.0", "ts-loader": "8.0.11", - "typescript": "4.5.4", - "webpack": "5.66.0", + "typescript": "4.4.4", + "webpack": "5.68.0", "webpack-filter-warnings-plugin": "1.2.1" }, "types": "src/index.ts" diff --git a/packages/grafana-ui/src/components/Badge/Badge.mdx b/packages/grafana-ui/src/components/Badge/Badge.mdx index 8782bceb0c1..f246c69d5c0 100644 --- a/packages/grafana-ui/src/components/Badge/Badge.mdx +++ b/packages/grafana-ui/src/components/Badge/Badge.mdx @@ -1,5 +1,10 @@ +import { Meta, Props } from '@storybook/addon-docs/blocks'; +import { Badge } from './Badge'; + -# Badge +## Badge The badge component adds meta information to other content, for example about release status or new elements. You can add any `Icon` component or use the badge without an icon. + + diff --git a/packages/grafana-ui/src/components/Badge/Badge.story.tsx b/packages/grafana-ui/src/components/Badge/Badge.story.tsx index 4d790acf15a..32aa3a63969 100644 --- a/packages/grafana-ui/src/components/Badge/Badge.story.tsx +++ b/packages/grafana-ui/src/components/Badge/Badge.story.tsx @@ -3,13 +3,14 @@ import { Story } from '@storybook/react'; import { Badge, BadgeProps } from '@grafana/ui'; import { iconOptions } from '../../utils/storybook/knobs'; import { withCenteredStory } from '../../utils/storybook/withCenteredStory'; +import mdx from './Badge.mdx'; export default { title: 'Data Display/Badge', component: Badge, decorators: [withCenteredStory], parameters: { - docs: {}, + docs: { page: mdx }, }, argTypes: { icon: { options: iconOptions, control: { type: 'select' } }, diff --git a/packages/grafana-ui/src/components/Button/Button.tsx b/packages/grafana-ui/src/components/Button/Button.tsx index 714808d9b88..d1cf2db8dd4 100644 --- a/packages/grafana-ui/src/components/Button/Button.tsx +++ b/packages/grafana-ui/src/components/Button/Button.tsx @@ -82,7 +82,15 @@ export const LinkButton = React.forwardRef( iconOnly: !children, }); - const linkButtonStyles = cx(styles.button, { [styles.disabled]: disabled }, className); + const linkButtonStyles = cx( + styles.button, + { + [css(styles.disabled, { + pointerEvents: 'none', + })]: disabled, + }, + className + ); deprecatedPropWarning( variant === 'link', @@ -228,7 +236,6 @@ function getPropertiesForDisabled(theme: GrafanaTheme2, variant: ButtonVariant, const disabledStyles: CSSObject = { cursor: 'not-allowed', boxShadow: 'none', - pointerEvents: 'none', color: theme.colors.text.disabled, transition: 'none', }; diff --git a/packages/grafana-ui/src/components/ButtonCascader/ButtonCascader.tsx b/packages/grafana-ui/src/components/ButtonCascader/ButtonCascader.tsx index a485813bfcf..0349f93c989 100644 --- a/packages/grafana-ui/src/components/ButtonCascader/ButtonCascader.tsx +++ b/packages/grafana-ui/src/components/ButtonCascader/ButtonCascader.tsx @@ -1,18 +1,18 @@ import React from 'react'; -import { Icon } from '../Icon/Icon'; import { IconName } from '../../types/icon'; -import { css, cx } from '@emotion/css'; +import { css } from '@emotion/css'; -// @ts-ignore import RCCascader from 'rc-cascader'; import { CascaderOption } from '../Cascader/Cascader'; import { onChangeCascader, onLoadDataCascader } from '../Cascader/optionMappings'; import { stylesFactory, useTheme2 } from '../../themes'; import { GrafanaTheme2 } from '@grafana/data'; +import { Button, ButtonProps } from '../Button'; +import { Icon } from '../Icon/Icon'; export interface ButtonCascaderProps { options: CascaderOption[]; - children: string; + children?: string; icon?: IconName; disabled?: boolean; value?: string[]; @@ -21,6 +21,9 @@ export interface ButtonCascaderProps { onChange?: (value: string[], selectedOptions: CascaderOption[]) => void; onPopupVisibleChange?: (visible: boolean) => void; className?: string; + variant?: ButtonProps['variant']; + buttonProps?: ButtonProps; + hideDownIcon?: boolean; } const getStyles = stylesFactory((theme: GrafanaTheme2) => { @@ -41,10 +44,17 @@ const getStyles = stylesFactory((theme: GrafanaTheme2) => { }); export const ButtonCascader: React.FC = (props) => { - const { onChange, className, loadData, icon, ...rest } = props; + const { onChange, className, loadData, icon, buttonProps, hideDownIcon, variant, disabled, ...rest } = props; const theme = useTheme2(); const styles = getStyles(theme); + // Weird way to do this bit it goes around a styling issue in Button where even null/undefined child triggers + // styling change which messes up the look if there is only single icon content. + let content: any = props.children; + if (!hideDownIcon) { + content = [props.children, ]; + } + return ( = (props) => { {...rest} expandIcon={null} > - + ); }; diff --git a/packages/grafana-ui/src/components/ButtonCascader/_ButtonCascader.scss b/packages/grafana-ui/src/components/ButtonCascader/_ButtonCascader.scss index 8793c831678..63ba29bdd31 100644 --- a/packages/grafana-ui/src/components/ButtonCascader/_ButtonCascader.scss +++ b/packages/grafana-ui/src/components/ButtonCascader/_ButtonCascader.scss @@ -1,19 +1,24 @@ .rc-cascader { font-size: 12px; - &-menus { - font-size: 12px; - overflow: hidden; - background: $page-bg; + &-dropdown { position: absolute; - border: $panel-border; - border-radius: $border-radius; - box-shadow: $typeahead-shadow; - white-space: nowrap; + // Required, otherwise the portal that the popup is shown in will render under other components + z-index: 9999; &-hidden { display: none; } + } + + &-menus { + //font-size: 12px; + overflow: hidden; + background: $page-bg; + border: $panel-border; + border-radius: $border-radius; + box-shadow: $typeahead-shadow; + white-space: nowrap; &.slide-up-enter, &.slide-up-appear { @@ -87,7 +92,7 @@ position: relative; &:hover { - background: $typeahead-selected-bg; + background: $colors-action-hover; } &-disabled { @@ -108,12 +113,11 @@ } &-active { - color: $typeahead-selected-color; - background: $typeahead-selected-bg; + color: $text-color-strong; + background: $colors-action-selected; &:hover { - color: $typeahead-selected-color; - background: $typeahead-selected-bg; + background: $colors-action-hover; } } @@ -126,6 +130,7 @@ color: $text-color-weak; position: absolute; right: 16px; + top: 0; line-height: 32px; } } diff --git a/packages/grafana-ui/src/components/Cascader/Cascader.tsx b/packages/grafana-ui/src/components/Cascader/Cascader.tsx index 3ef8ca4880e..0fbfa65da20 100644 --- a/packages/grafana-ui/src/components/Cascader/Cascader.tsx +++ b/packages/grafana-ui/src/components/Cascader/Cascader.tsx @@ -184,7 +184,7 @@ export class Cascader extends React.PureComponent }; render() { - const { allowCustomValue, placeholder, width, changeOnSelect, options } = this.props; + const { allowCustomValue, formatCreateLabel, placeholder, width, changeOnSelect, options } = this.props; const { focusCascade, isSearching, rcValue, activeLabel } = this.state; const searchableOptions = this.getSearchableOptions(options); @@ -201,21 +201,17 @@ export class Cascader extends React.PureComponent onBlur={this.onBlur} options={searchableOptions} onCreateOption={this.onCreateOption} - formatCreateLabel={this.props.formatCreateLabel} + formatCreateLabel={formatCreateLabel} width={width} /> ) : (
void) | undefined; -export const onChangeCascader = (onChanged: onChangeType) => ( - values: CascaderValueType, - options: RCCascaderOption[] -) => { +export const onChangeCascader = (onChanged: onChangeType) => (values: SingleValueType, options: RCCascaderOption[]) => { if (onChanged) { // map values to strings for backwards compatibility with Cascader components onChanged( diff --git a/packages/grafana-ui/src/components/Collapse/CollapsableSection.tsx b/packages/grafana-ui/src/components/Collapse/CollapsableSection.tsx index 3e2ac0adc99..89d2300b033 100644 --- a/packages/grafana-ui/src/components/Collapse/CollapsableSection.tsx +++ b/packages/grafana-ui/src/components/Collapse/CollapsableSection.tsx @@ -1,8 +1,10 @@ -import React, { FC, ReactNode, useState } from 'react'; -import { css } from '@emotion/css'; +import React, { FC, ReactNode, useRef, useState } from 'react'; +import { uniqueId } from 'lodash'; +import { css, cx } from '@emotion/css'; import { useStyles2 } from '../../themes'; -import { Icon } from '..'; +import { Icon, Spinner } from '..'; import { GrafanaTheme2 } from '@grafana/data'; +import { getFocusStyles } from '../../themes/mixins'; export interface Props { label: ReactNode; @@ -10,46 +12,102 @@ export interface Props { /** Callback for the toggle functionality */ onToggle?: (isOpen: boolean) => void; children: ReactNode; + className?: string; + contentClassName?: string; + loading?: boolean; + labelId?: string; } -export const CollapsableSection: FC = ({ label, isOpen, onToggle, children }) => { +export const CollapsableSection: FC = ({ + label, + isOpen, + onToggle, + className, + contentClassName, + children, + labelId, + loading = false, +}) => { const [open, toggleOpen] = useState(isOpen); const styles = useStyles2(collapsableSectionStyles); - const headerStyle = open ? styles.header : styles.headerCollapsed; const tooltip = `Click to ${open ? 'collapse' : 'expand'}`; - const onClick = () => { + const onClick = (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + onToggle?.(!open); toggleOpen(!open); }; + const { current: id } = useRef(uniqueId()); + + const buttonLabelId = labelId ?? `collapse-label-${id}`; return ( -
-
- {label} - + <> +
+ +
+ {label} +
- {open &&
{children}
} -
+ {open && ( +
+ {children} +
+ )} + ); }; -const collapsableSectionStyles = (theme: GrafanaTheme2) => { - const header = css({ +const collapsableSectionStyles = (theme: GrafanaTheme2) => ({ + header: css({ display: 'flex', + cursor: 'pointer', + boxSizing: 'border-box', + flexDirection: 'row-reverse', + position: 'relative', justifyContent: 'space-between', fontSize: theme.typography.size.lg, padding: `${theme.spacing(0.5)} 0`, - cursor: 'pointer', - }); - const headerCollapsed = css(header, { + '&:focus-within': getFocusStyles(theme), + }), + headerClosed: css({ borderBottom: `1px solid ${theme.colors.border.weak}`, - }); - const icon = css({ + }), + button: css({ + all: 'unset', + '&:focus-visible': { + outline: 'none', + outlineOffset: 'unset', + transition: 'none', + boxShadow: 'none', + }, + }), + icon: css({ color: theme.colors.text.secondary, - }); - const content = css({ + }), + content: css({ padding: `${theme.spacing(2)} 0`, - }); - - return { header, headerCollapsed, icon, content }; -}; + }), + spinner: css({ + display: 'flex', + alignItems: 'center', + width: theme.v1.spacing.md, + }), + label: css({ + display: 'flex', + }), +}); diff --git a/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx b/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx index 5303c738adc..6e44460f34f 100644 --- a/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx +++ b/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx @@ -1,6 +1,7 @@ import React, { useState, useCallback } from 'react'; import { css, cx } from '@emotion/css'; import { DataSourceSettings, SelectableValue } from '@grafana/data'; +import { selectors } from '@grafana/e2e-selectors'; import { BasicAuthSettings } from './BasicAuthSettings'; import { HttpProxySettings } from './HttpProxySettings'; import { TLSAuthSettings } from './TLSAuthSettings'; @@ -125,10 +126,14 @@ export const DataSourceHttpSettings: React.FC = (props) => { className={inputStyle} placeholder={defaultUrl} value={dataSourceConfig.url} + aria-label={selectors.components.DataSource.DataSourceHttpSettings.urlInput} onChange={(event) => onSettingsChange({ url: event.currentTarget.value })} /> ); + const azureAuthEnabled: boolean = + (azureAuthSettings?.azureAuthSupported && azureAuthSettings.getAzureAuthEnabled(dataSourceConfig)) || false; + return (
<> @@ -219,16 +224,16 @@ export const DataSourceHttpSettings: React.FC = (props) => { />
- {azureAuthSettings?.azureAuthEnabled && ( + {azureAuthSettings?.azureAuthSupported && (
{ - onSettingsChange({ - jsonData: { ...dataSourceConfig.jsonData, azureAuth: event!.currentTarget.checked }, - }); + onSettingsChange( + azureAuthSettings.setAzureAuthEnabled(dataSourceConfig, event!.currentTarget.checked) + ); }} tooltip="Use Azure authentication for Azure endpoint." /> @@ -267,11 +272,9 @@ export const DataSourceHttpSettings: React.FC = (props) => { )} - {azureAuthSettings?.azureAuthEnabled && - azureAuthSettings?.azureSettingsUI && - dataSourceConfig.jsonData.azureAuth && ( - - )} + {azureAuthSettings?.azureAuthSupported && azureAuthEnabled && azureAuthSettings.azureSettingsUI && ( + + )} {dataSourceConfig.jsonData.sigV4Auth && sigV4AuthToggleEnabled && } diff --git a/packages/grafana-ui/src/components/DataSourceSettings/types.ts b/packages/grafana-ui/src/components/DataSourceSettings/types.ts index f298651b3e7..ae71ad191d5 100644 --- a/packages/grafana-ui/src/components/DataSourceSettings/types.ts +++ b/packages/grafana-ui/src/components/DataSourceSettings/types.ts @@ -2,8 +2,20 @@ import React from 'react'; import { DataSourceSettings } from '@grafana/data'; export interface AzureAuthSettings { - azureAuthEnabled: boolean; - azureSettingsUI?: React.ComponentType; + /** Set to true if Azure authentication supported by the datasource */ + readonly azureAuthSupported: boolean; + + /** Gets whether the Azure authentication currently enabled for the datasource */ + readonly getAzureAuthEnabled: (config: DataSourceSettings) => boolean; + + /** Enables/disables the Azure authentication from the datasource */ + readonly setAzureAuthEnabled: ( + config: DataSourceSettings, + enabled: boolean + ) => Partial>; + + /** Optional React component of additional Azure settings UI if authentication is enabled */ + readonly azureSettingsUI?: React.ComponentType; } export interface HttpSettingsBaseProps { diff --git a/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx b/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx index 348914dc44e..b8df2c4d405 100644 --- a/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx +++ b/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx @@ -23,7 +23,7 @@ import { ThemeContext } from '../../../../themes'; * - noOptionsMessage & loadingMessage is of string type * - isDisabled is renamed to disabled */ -type LegacyCommonProps = Omit, 'noOptionsMessage' | 'disabled' | 'value'>; +type LegacyCommonProps = Omit, 'noOptionsMessage' | 'disabled' | 'value' | 'loadingMessage'>; interface AsyncProps extends LegacyCommonProps, Omit, 'loadingMessage'> { loadingMessage?: () => string; @@ -237,6 +237,7 @@ export class AsyncSelect extends PureComponent> { getOptionLabel={getOptionLabel} getOptionValue={getOptionValue} menuShouldScrollIntoView={false} + //@ts-expect-error onChange={onChange} loadOptions={loadOptions} isLoading={isLoading} diff --git a/packages/grafana-ui/src/components/GraphNG/utils.test.ts b/packages/grafana-ui/src/components/GraphNG/utils.test.ts index 7552cd1974f..0cc6b59533c 100644 --- a/packages/grafana-ui/src/components/GraphNG/utils.test.ts +++ b/packages/grafana-ui/src/components/GraphNG/utils.test.ts @@ -197,7 +197,7 @@ describe('GraphNG utils', () => { timeZone: DefaultTimeZone, getTimeRange: getDefaultTimeRange, eventBus: new EventBusSrv(), - sync: DashboardCursorSync.Tooltip, + sync: () => DashboardCursorSync.Tooltip, allFrames: [frame!], }).getConfig(); expect(result).toMatchSnapshot(); diff --git a/packages/grafana-ui/src/components/JSONFormatter/json_explorer/helpers.test.ts b/packages/grafana-ui/src/components/JSONFormatter/json_explorer/helpers.test.ts new file mode 100644 index 00000000000..f4d388acdb3 --- /dev/null +++ b/packages/grafana-ui/src/components/JSONFormatter/json_explorer/helpers.test.ts @@ -0,0 +1,45 @@ +import { formatString } from './helpers'; + +describe('JSONFormatter helpers', () => { + describe('escapeString', () => { + it("does not escape strings that don't contain quotes", () => { + const input = 'hello world'; + const expected = 'hello world'; + + const result = formatString(input); + expect(result).toEqual(expected); + }); + + it('does not escape strings that contain single quotes', () => { + const input = `'hello world'`; + const expected = `'hello world'`; + + const result = formatString(input); + expect(result).toEqual(expected); + }); + + it('does escapes strings that contain one double quote', () => { + const input = `"hello world`; + const expected = `\\"hello world`; + + const result = formatString(input); + expect(result).toEqual(expected); + }); + + it('does escapes strings that contain two double quotes', () => { + const input = `"hello world"`; + const expected = `\\"hello world\\"`; + + const result = formatString(input); + expect(result).toEqual(expected); + }); + + it('does escapes a string that looks like JSON', () => { + const input = `{"hello": "world"}`; + const expected = `{\\"hello\\": \\"world\\"}`; + + const result = formatString(input); + expect(result).toEqual(expected); + }); + }); +}); diff --git a/packages/grafana-ui/src/components/JSONFormatter/json_explorer/helpers.ts b/packages/grafana-ui/src/components/JSONFormatter/json_explorer/helpers.ts index 64f40ddcfb1..72c410a1feb 100644 --- a/packages/grafana-ui/src/components/JSONFormatter/json_explorer/helpers.ts +++ b/packages/grafana-ui/src/components/JSONFormatter/json_explorer/helpers.ts @@ -4,8 +4,8 @@ /* * Escapes `"` characters from string */ -function escapeString(str: string): string { - return str.replace('"', '"'); +export function formatString(str: string): string { + return str.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); } /* @@ -62,7 +62,7 @@ export function getValuePreview(object: object, value: string): string { } if (type === 'string') { - value = '"' + escapeString(value) + '"'; + value = '"' + formatString(value) + '"'; } if (type === 'function') { // Remove content of the function diff --git a/packages/grafana-ui/src/components/Modal/Modal.tsx b/packages/grafana-ui/src/components/Modal/Modal.tsx index f650278a294..cdeb318ce24 100644 --- a/packages/grafana-ui/src/components/Modal/Modal.tsx +++ b/packages/grafana-ui/src/components/Modal/Modal.tsx @@ -1,13 +1,14 @@ -import React, { PropsWithChildren, useCallback, useEffect } from 'react'; import { cx } from '@emotion/css'; -import { useTheme2 } from '../../themes'; -import { IconName } from '../../types'; -import { getModalStyles } from './getModalStyles'; -import { ModalHeader } from './ModalHeader'; -import { IconButton } from '../IconButton/IconButton'; -import { HorizontalGroup } from '../Layout/Layout'; import { FocusScope } from '@react-aria/focus'; import { OverlayContainer } from '@react-aria/overlays'; +import React, { PropsWithChildren, useCallback, useEffect } from 'react'; + +import { useTheme2 } from '../../themes'; +import { IconName } from '../../types'; +import { IconButton } from '../IconButton/IconButton'; +import { HorizontalGroup } from '../Layout/Layout'; +import { getModalStyles } from './getModalStyles'; +import { ModalHeader } from './ModalHeader'; export interface Props { /** @deprecated no longer used */ @@ -20,6 +21,7 @@ export interface Props { contentClassName?: string; closeOnEscape?: boolean; closeOnBackdropClick?: boolean; + trapFocus?: boolean; isOpen?: boolean; onDismiss?: () => void; @@ -39,6 +41,7 @@ export function Modal(props: PropsWithChildren) { contentClassName, onDismiss: propsOnDismiss, onClickBackdrop, + trapFocus = true, } = props; const theme = useTheme2(); const styles = getModalStyles(theme); @@ -76,7 +79,7 @@ export function Modal(props: PropsWithChildren) { className={styles.modalBackdrop} onClick={onClickBackdrop || (closeOnBackdropClick ? onDismiss : undefined)} /> - +
{typeof title === 'string' && } diff --git a/packages/grafana-ui/src/components/Modal/ModalTabsHeader.tsx b/packages/grafana-ui/src/components/Modal/ModalTabsHeader.tsx index c34a7696576..a4a2820ab2c 100644 --- a/packages/grafana-ui/src/components/Modal/ModalTabsHeader.tsx +++ b/packages/grafana-ui/src/components/Modal/ModalTabsHeader.tsx @@ -1,4 +1,5 @@ import React from 'react'; +import { NavModelItem } from '@grafana/data'; import { IconName } from '../../types'; import { TabsBar } from '../Tabs/TabsBar'; import { Tab } from '../Tabs/Tab'; @@ -8,7 +9,7 @@ interface ModalTab { value: string; label: string; icon?: IconName; - labelSuffix?: () => JSX.Element; + tabSuffix?: NavModelItem['tabSuffix']; } interface Props { @@ -29,7 +30,7 @@ export const ModalTabsHeader: React.FC = ({ icon, title, tabs, activeTab, key={`${t.value}-${index}`} label={t.label} icon={t.icon} - suffix={t.labelSuffix} + suffix={t.tabSuffix} active={t.value === activeTab} onChangeTab={() => onChangeTab(t)} /> diff --git a/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts b/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts index b1581a2414f..a83f5fb0cdc 100644 --- a/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts +++ b/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts @@ -15,7 +15,7 @@ export interface PanelContext { eventBus: EventBus; /** Dashboard panels sync */ - sync?: DashboardCursorSync; + sync?: () => DashboardCursorSync; /** Information on what the outer container is */ app?: CoreApp | 'string'; diff --git a/packages/grafana-ui/src/components/Select/MultiValue.tsx b/packages/grafana-ui/src/components/Select/MultiValue.tsx index 5d022a62f8a..c7166459e68 100644 --- a/packages/grafana-ui/src/components/Select/MultiValue.tsx +++ b/packages/grafana-ui/src/components/Select/MultiValue.tsx @@ -1,7 +1,8 @@ import React from 'react'; + import { useTheme2 } from '../../themes'; +import { IconButton } from '../IconButton/IconButton'; import { getSelectStyles } from './getSelectStyles'; -import { Icon } from '../Icon/Icon'; interface MultiValueContainerProps { innerProps: any; @@ -25,9 +26,5 @@ export type MultiValueRemoveProps = { export const MultiValueRemove: React.FC = ({ children, innerProps }) => { const theme = useTheme2(); const styles = getSelectStyles(theme); - return ( -
- -
- ); + return ; }; diff --git a/packages/grafana-ui/src/components/Select/types.ts b/packages/grafana-ui/src/components/Select/types.ts index c7493a69866..989d293bfdd 100644 --- a/packages/grafana-ui/src/components/Select/types.ts +++ b/packages/grafana-ui/src/components/Select/types.ts @@ -79,6 +79,8 @@ export interface SelectCommonProps { value: SelectableValue | null, options: OptionsOrGroups> ) => boolean; + /** Message to display isLoading=true*/ + loadingMessage?: string; } export interface SelectAsyncProps { diff --git a/packages/grafana-ui/src/components/Table/FooterRow.tsx b/packages/grafana-ui/src/components/Table/FooterRow.tsx index 3f77f923f47..5bdc13b8b61 100644 --- a/packages/grafana-ui/src/components/Table/FooterRow.tsx +++ b/packages/grafana-ui/src/components/Table/FooterRow.tsx @@ -9,30 +9,14 @@ import { EmptyCell, FooterCell } from './FooterCell'; export interface FooterRowProps { totalColumnsWidth: number; footerGroups: HeaderGroup[]; - footerValues?: FooterItem[]; + footerValues: FooterItem[]; + height: number; } export const FooterRow = (props: FooterRowProps) => { - const { totalColumnsWidth, footerGroups, footerValues } = props; + const { totalColumnsWidth, footerGroups, height } = props; const e2eSelectorsTable = selectors.components.Panels.Visualization.Table; const tableStyles = useStyles2(getTableStyles); - const EXTENDED_ROW_HEIGHT = 27; - - if (!footerValues) { - return null; - } - - let length = 0; - for (const fv of footerValues) { - if (Array.isArray(fv) && fv.length > length) { - length = fv.length; - } - } - - let height: number | undefined; - if (footerValues && length > 1) { - height = EXTENDED_ROW_HEIGHT * length; - } return ( +
+ {disp} +
+ + ); +} diff --git a/packages/grafana-ui/src/components/Table/HeaderRow.tsx b/packages/grafana-ui/src/components/Table/HeaderRow.tsx index 434852eec4c..a935b961249 100644 --- a/packages/grafana-ui/src/components/Table/HeaderRow.tsx +++ b/packages/grafana-ui/src/components/Table/HeaderRow.tsx @@ -55,11 +55,7 @@ function renderHeaderCell(column: any, tableStyles: TableStyles, field?: Field,
{column.canSort && ( <> -
+
+ {column.canFilter && } )} diff --git a/packages/grafana-ui/src/components/Table/Table.tsx b/packages/grafana-ui/src/components/Table/Table.tsx index 2df351259ac..a875aa82195 100644 --- a/packages/grafana-ui/src/components/Table/Table.tsx +++ b/packages/grafana-ui/src/components/Table/Table.tsx @@ -127,7 +127,30 @@ export const Table: FC = memo((props: Props) => { footerValues, showTypeIcons, } = props; + const tableStyles = useStyles2(getTableStyles); + const headerHeight = noHeader ? 0 : tableStyles.cellHeight; + + const footerHeight = useMemo(() => { + const EXTENDED_ROW_HEIGHT = 33; + let length = 0; + + if (!footerValues) { + return 0; + } + + for (const fv of footerValues) { + if (Array.isArray(fv) && fv.length > length) { + length = fv.length; + } + } + + if (length > 1) { + return EXTENDED_ROW_HEIGHT * length; + } + + return EXTENDED_ROW_HEIGHT; + }, [footerValues]); // React table data array. This data acts just like a dummy array to let react-table know how many rows exist // The cells use the field to look up values @@ -197,7 +220,7 @@ export const Table: FC = memo((props: Props) => { [onCellFilterAdded, prepareRow, rows, tableStyles] ); - const headerHeight = noHeader ? 0 : tableStyles.cellHeight; + const listHeight = height - (headerHeight + footerHeight); return (
@@ -206,7 +229,7 @@ export const Table: FC = memo((props: Props) => { {!noHeader && } {rows.length > 0 ? ( = memo((props: Props) => { No data
)} - + {footerValues && ( + + )}
diff --git a/packages/grafana-ui/src/components/Table/styles.ts b/packages/grafana-ui/src/components/Table/styles.ts index 9182ec63bd1..76adbf40729 100644 --- a/packages/grafana-ui/src/components/Table/styles.ts +++ b/packages/grafana-ui/src/components/Table/styles.ts @@ -92,6 +92,9 @@ export const getTableStyles = (theme: GrafanaTheme2) => { } `, headerCellLabel: css` + border: none; + padding: 0; + background: inherit; cursor: pointer; white-space: nowrap; overflow: hidden; @@ -131,6 +134,10 @@ export const getTableStyles = (theme: GrafanaTheme2) => { &:hover { background-color: ${rowHoverBg}; } + + &:last-child { + border-bottom: 0; + } `, imageCell: css` height: 100%; diff --git a/packages/grafana-ui/src/components/Table/utils.test.ts b/packages/grafana-ui/src/components/Table/utils.test.ts index f710638701c..bb2ff5b70d9 100644 --- a/packages/grafana-ui/src/components/Table/utils.test.ts +++ b/packages/grafana-ui/src/components/Table/utils.test.ts @@ -186,7 +186,6 @@ describe('Table utils', () => { title: '1.0', text: '1.0', }), - parse: (value: any) => '1.0', }; const rows: any[] = []; diff --git a/packages/grafana-ui/src/components/Table/utils.ts b/packages/grafana-ui/src/components/Table/utils.ts index 360e2e2f76f..872904219e6 100644 --- a/packages/grafana-ui/src/components/Table/utils.ts +++ b/packages/grafana-ui/src/components/Table/utils.ts @@ -14,6 +14,7 @@ import { DefaultCell } from './DefaultCell'; import { BarGaugeCell } from './BarGaugeCell'; import { CellComponent, TableCellDisplayMode, TableFieldOptions, FooterItem, GrafanaTableColumn } from './types'; import { JSONViewCell } from './JSONViewCell'; +import { GeoCell } from './GeoCell'; import { ImageCell } from './ImageCell'; import { getFooterValue } from './FooterRow'; @@ -132,6 +133,10 @@ function getCellComponent(displayMode: TableCellDisplayMode, field: Field): Cell return JSONViewCell; } + if (field.type === FieldType.geo) { + return GeoCell; + } + // Default or Auto if (field.type === FieldType.other) { return JSONViewCell; diff --git a/packages/grafana-ui/src/components/Tabs/Tab.tsx b/packages/grafana-ui/src/components/Tabs/Tab.tsx index f5839ee1e94..562eb2669ee 100644 --- a/packages/grafana-ui/src/components/Tabs/Tab.tsx +++ b/packages/grafana-ui/src/components/Tabs/Tab.tsx @@ -1,6 +1,6 @@ import React, { HTMLProps } from 'react'; import { css, cx } from '@emotion/css'; -import { GrafanaTheme2 } from '@grafana/data'; +import { GrafanaTheme2, NavModelItem } from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; import { Icon } from '../Icon/Icon'; @@ -18,11 +18,12 @@ export interface TabProps extends HTMLProps { onChangeTab?: (event?: React.MouseEvent) => void; /** A number rendered next to the text. Usually used to display the number of items in a tab's view. */ counter?: number | null; - suffix?: () => JSX.Element; + /** Extra content, displayed after the tab label and counter */ + suffix?: NavModelItem['tabSuffix']; } export const Tab = React.forwardRef( - ({ label, active, icon, onChangeTab, counter, suffix, className, href, ...otherProps }, ref) => { + ({ label, active, icon, onChangeTab, counter, suffix: Suffix, className, href, ...otherProps }, ref) => { const theme = useTheme2(); const tabsStyles = getTabStyles(theme); const content = () => ( @@ -30,7 +31,7 @@ export const Tab = React.forwardRef( {icon && } {label} {typeof counter === 'number' && } - {suffix && suffix()} + {Suffix && } ); @@ -116,5 +117,8 @@ const getTabStyles = stylesFactory((theme: GrafanaTheme2) => { background-image: ${theme.colors.gradients.brandHorizontal} !important; } `, + suffix: css` + margin-left: ${theme.spacing(1)}; + `, }; }); diff --git a/packages/grafana-ui/src/components/Tags/Tag.tsx b/packages/grafana-ui/src/components/Tags/Tag.tsx index c4c05dc9c48..01fc15e68cc 100644 --- a/packages/grafana-ui/src/components/Tags/Tag.tsx +++ b/packages/grafana-ui/src/components/Tags/Tag.tsx @@ -3,6 +3,8 @@ import { cx, css } from '@emotion/css'; import { GrafanaTheme } from '@grafana/data'; import { useTheme } from '../../themes'; import { getTagColor, getTagColorsFromName } from '../../utils'; +import { IconName } from '../../types/icon'; +import { Icon } from '../Icon/Icon'; /** * @public @@ -12,12 +14,13 @@ export type OnTagClick = (name: string, event: React.MouseEvent) => export interface Props extends Omit, 'onClick'> { /** Name of the tag to display */ name: string; + icon?: IconName; /** Use constant color from TAG_COLORS. Using index instead of color directly so we can match other styling. */ colorIndex?: number; onClick?: OnTagClick; } -export const Tag = forwardRef(({ name, onClick, className, colorIndex, ...rest }, ref) => { +export const Tag = forwardRef(({ name, onClick, icon, className, colorIndex, ...rest }, ref) => { const theme = useTheme(); const styles = getTagStyles(theme, name, colorIndex); @@ -32,10 +35,12 @@ export const Tag = forwardRef(({ name, onClick, className, c return onClick ? ( ) : ( + {icon && } {name} ); diff --git a/packages/grafana-ui/src/components/Tags/TagList.tsx b/packages/grafana-ui/src/components/Tags/TagList.tsx index 6d82cc2a64b..25e6825f1f6 100644 --- a/packages/grafana-ui/src/components/Tags/TagList.tsx +++ b/packages/grafana-ui/src/components/Tags/TagList.tsx @@ -1,8 +1,9 @@ -import React, { FC, memo } from 'react'; +import React, { forwardRef, memo } from 'react'; import { css, cx } from '@emotion/css'; import { OnTagClick, Tag } from './Tag'; import { useTheme2 } from '../../themes'; import { GrafanaTheme2 } from '@grafana/data'; +import { IconName } from '../../types/icon'; export interface Props { displayMax?: number; @@ -12,24 +13,30 @@ export interface Props { className?: string; /** aria-label for the `i`-th Tag component */ getAriaLabel?: (name: string, i: number) => string; + /** Icon to show next to tag label */ + icon?: IconName; } -export const TagList: FC = memo(({ displayMax, tags, onClick, className, getAriaLabel }) => { - const theme = useTheme2(); - const styles = getStyles(theme, Boolean(displayMax && displayMax > 0)); - const numTags = tags.length; - const tagsToDisplay = displayMax ? tags.slice(0, displayMax) : tags; - return ( -
    - {tagsToDisplay.map((tag, i) => ( -
  • - -
    • - ))} - {displayMax && displayMax > 0 && numTags - 1 > 0 && + {numTags - 1}} -
- ); -}); +export const TagList = memo( + forwardRef(({ displayMax, tags, icon, onClick, className, getAriaLabel }, ref) => { + const theme = useTheme2(); + const styles = getStyles(theme, Boolean(displayMax && displayMax > 0)); + const numTags = tags.length; + const tagsToDisplay = displayMax ? tags.slice(0, displayMax) : tags; + return ( +
    + {tagsToDisplay.map((tag, i) => ( +
  • + +
    • + ))} + {displayMax && displayMax > 0 && numTags - 1 > 0 && ( + + {numTags - 1} + )} +
+ ); + }) +); TagList.displayName = 'TagList'; diff --git a/packages/grafana-ui/src/components/ThresholdsEditorNew/ThresholdsEditor.story.tsx b/packages/grafana-ui/src/components/ThresholdsEditorNew/ThresholdsEditor.story.tsx deleted file mode 100644 index 8bbefbaba66..00000000000 --- a/packages/grafana-ui/src/components/ThresholdsEditorNew/ThresholdsEditor.story.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import React from 'react'; -import { action } from '@storybook/addon-actions'; -import { withCenteredStory } from '../../utils/storybook/withCenteredStory'; -import { ThresholdsEditor } from './ThresholdsEditor'; -import { ThresholdsMode, ThresholdsConfig } from '@grafana/data'; - -const thresholds: ThresholdsConfig = { - mode: ThresholdsMode.Absolute, - steps: [ - { value: -Infinity, color: 'green' }, - { value: 50, color: 'red' }, - { value: 60, color: 'blue' }, - ], -}; - -export default { - title: 'Pickers and Editors/ThresholdsEditorNew', - component: ThresholdsEditor, - decorators: [withCenteredStory], - parameters: { - docs: {}, - }, -}; - -export const Default = () => { - return ; -}; - -export const WithThreshold = () => { - return ; -}; diff --git a/packages/grafana-ui/src/components/TimeSeries/TimeSeries.tsx b/packages/grafana-ui/src/components/TimeSeries/TimeSeries.tsx index a62fc1777dd..2b0813d2ee4 100644 --- a/packages/grafana-ui/src/components/TimeSeries/TimeSeries.tsx +++ b/packages/grafana-ui/src/components/TimeSeries/TimeSeries.tsx @@ -18,7 +18,7 @@ export class UnthemedTimeSeries extends React.Component { panelContext: PanelContext = {} as PanelContext; prepConfig = (alignedFrame: DataFrame, allFrames: DataFrame[], getTimeRange: () => TimeRange) => { - const { eventBus, sync } = this.context; + const { eventBus, sync } = this.context as PanelContext; const { theme, timeZone, legend, renderers, tweakAxis, tweakScale } = this.props; return preparePlotConfigBuilder({ diff --git a/packages/grafana-ui/src/components/TimeSeries/utils.ts b/packages/grafana-ui/src/components/TimeSeries/utils.ts index 2e7bfb1a513..650e748a284 100644 --- a/packages/grafana-ui/src/components/TimeSeries/utils.ts +++ b/packages/grafana-ui/src/components/TimeSeries/utils.ts @@ -38,7 +38,10 @@ const defaultConfig: GraphFieldConfig = { axisPlacement: AxisPlacement.Auto, }; -export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ sync: DashboardCursorSync; legend?: VizLegendOptions }> = ({ +export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ + sync?: () => DashboardCursorSync; + legend?: VizLegendOptions; +}> = ({ frame, theme, timeZone, @@ -67,6 +70,10 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ sync: DashboardCursor let xScaleUnit = '_x'; let yScaleKey = ''; + const xFieldAxisPlacement = + xField.config.custom?.axisPlacement !== AxisPlacement.Hidden ? AxisPlacement.Bottom : AxisPlacement.Hidden; + const xFieldAxisShow = xField.config.custom?.axisPlacement !== AxisPlacement.Hidden; + if (xField.type === FieldType.time) { xScaleUnit = 'time'; builder.addScale({ @@ -83,7 +90,8 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ sync: DashboardCursor builder.addAxis({ scaleKey: xScaleKey, isTime: true, - placement: AxisPlacement.Bottom, + placement: xFieldAxisPlacement, + show: xFieldAxisShow, label: xField.config.custom?.axisLabel, timeZone, theme, @@ -103,7 +111,8 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ sync: DashboardCursor builder.addAxis({ scaleKey: xScaleKey, - placement: AxisPlacement.Bottom, + placement: xFieldAxisPlacement, + show: xFieldAxisShow, label: xField.config.custom?.axisLabel, theme, grid: { show: xField.config.custom?.axisGridShow }, @@ -263,9 +272,12 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ sync: DashboardCursor series: [t, b], fill: undefined, // using null will have the band use fill options from `t` }); - } - if (!fillOpacity) { - fillOpacity = 35; // default from flot + + if (!fillOpacity) { + fillOpacity = 35; // default from flot + } + } else { + fillOpacity = 0; } } } @@ -399,7 +411,7 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ sync: DashboardCursor }, }; - if (sync !== DashboardCursorSync.Off) { + if (sync && sync() !== DashboardCursorSync.Off) { const payload: DataHoverPayload = { point: { [xScaleKey]: null, @@ -412,6 +424,10 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn<{ sync: DashboardCursor key: '__global_', filters: { pub: (type: string, src: uPlot, x: number, y: number, w: number, h: number, dataIdx: number) => { + if (sync && sync() === DashboardCursorSync.Off) { + return false; + } + payload.rowIndex = dataIdx; if (x < 0 && y < 0) { payload.point[xScaleUnit] = null; diff --git a/packages/grafana-ui/src/components/ValueMappingsEditor/ValueMappingsEditor.story.tsx b/packages/grafana-ui/src/components/ValueMappingsEditor/ValueMappingsEditor.story.tsx deleted file mode 100644 index 3a2e78b9389..00000000000 --- a/packages/grafana-ui/src/components/ValueMappingsEditor/ValueMappingsEditor.story.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import React, { useState } from 'react'; -import { ValueMappingsEditor } from './ValueMappingsEditor'; -import { MappingType, ValueMapping } from '@grafana/data'; - -export default { - title: 'Pickers and Editors/ValueMappingsEditor', - component: ValueMappingsEditor, -}; - -export function Example() { - const [mappings, setMappings] = useState([ - { - type: MappingType.ValueToText, - options: { - LowLow: { color: 'red' }, - Low: { text: 'not good', color: 'orange' }, - Ok: { text: 'all good', color: 'green' }, - NoColor: { text: 'Unknown' }, - }, - }, - { - type: MappingType.RangeToText, - options: { - from: 10, - to: 15, - result: { - index: 5, - text: 'bad', - color: 'red', - }, - }, - }, - { - type: MappingType.RegexToText, - options: { - pattern: '(.*).example.com', - result: { - index: 5, - text: '$1', - color: 'green', - }, - }, - }, - ]); - - return ; -} diff --git a/packages/grafana-ui/src/components/VizLayout/VizLayout.tsx b/packages/grafana-ui/src/components/VizLayout/VizLayout.tsx index d171e0889ad..83444158386 100644 --- a/packages/grafana-ui/src/components/VizLayout/VizLayout.tsx +++ b/packages/grafana-ui/src/components/VizLayout/VizLayout.tsx @@ -1,7 +1,11 @@ import React, { FC, CSSProperties, ComponentType } from 'react'; import { useMeasure } from 'react-use'; -import { CustomScrollbar } from '../CustomScrollbar/CustomScrollbar'; +import { css } from '@emotion/css'; import { LegendPlacement } from '@grafana/schema'; +import { GrafanaTheme2 } from '@grafana/data'; +import { CustomScrollbar } from '../CustomScrollbar/CustomScrollbar'; +import { getFocusStyles } from '../../themes/mixins'; +import { useStyles2 } from '../../themes/ThemeContext'; /** * @beta @@ -24,6 +28,7 @@ export interface VizLayoutComponentType extends FC { * @beta */ export const VizLayout: VizLayoutComponentType = ({ width, height, legend, children }) => { + const styles = useStyles2(getVizStyles); const containerStyle: CSSProperties = { display: 'flex', width: `${width}px`, @@ -32,17 +37,17 @@ export const VizLayout: VizLayoutComponentType = ({ width, height, legend, child const [legendRef, legendMeasure] = useMeasure(); if (!legend) { - return
{children(width, height)}
; + return ( +
+ {children(width, height)} +
+ ); } const { placement, maxHeight = '35%', maxWidth = '60%' } = legend.props; let size: VizSize | null = null; - const vizStyle: CSSProperties = { - flexGrow: 2, - }; - const legendStyle: CSSProperties = {}; switch (placement) { @@ -76,7 +81,9 @@ export const VizLayout: VizLayoutComponentType = ({ width, height, legend, child return (
-
{size && children(size.width, size.height)}
+
+ {size && children(size.width, size.height)} +
{legend}
@@ -84,6 +91,15 @@ export const VizLayout: VizLayoutComponentType = ({ width, height, legend, child ); }; +export const getVizStyles = (theme: GrafanaTheme2) => { + return { + viz: css({ + flexGrow: 2, + borderRadius: theme.shape.borderRadius(1), + '&:focus-visible': getFocusStyles(theme), + }), + }; +}; interface VizSize { width: number; height: number; diff --git a/packages/grafana-ui/src/components/uPlot/config/UPlotThresholds.ts b/packages/grafana-ui/src/components/uPlot/config/UPlotThresholds.ts index 21029f98f07..a682824599f 100644 --- a/packages/grafana-ui/src/components/uPlot/config/UPlotThresholds.ts +++ b/packages/grafana-ui/src/components/uPlot/config/UPlotThresholds.ts @@ -1,4 +1,4 @@ -import { GrafanaTheme2, ThresholdsConfig, ThresholdsMode } from '@grafana/data'; +import { GrafanaTheme2, Threshold, ThresholdsConfig, ThresholdsMode } from '@grafana/data'; import { GraphThresholdsStyleConfig, GraphTresholdsStyleMode } from '@grafana/schema'; import { getGradientRange, scaleGradient } from './gradientFills'; import tinycolor from 'tinycolor2'; @@ -16,9 +16,79 @@ export interface UPlotThresholdOptions { } export function getThresholdsDrawHook(options: UPlotThresholdOptions) { + function addLines(u: uPlot, steps: Threshold[], theme: GrafanaTheme2, xMin: number, xMax: number, yScaleKey: string) { + let ctx = u.ctx; + + // Thresholds below a transparent threshold is treated like "less than", and line drawn previous threshold + let transparentIndex = 0; + + for (let idx = 0; idx < steps.length; idx++) { + const step = steps[idx]; + if (step.color === 'transparent') { + transparentIndex = idx; + break; + } + } + + ctx.lineWidth = 2; + + // Ignore the base -Infinity threshold by always starting on index 1 + for (let idx = 1; idx < steps.length; idx++) { + const step = steps[idx]; + let color: tinycolor.Instance; + + // if we are below a transparent index treat this a less then threshold, use previous thresholds color + if (transparentIndex >= idx && idx > 0) { + color = tinycolor(theme.visualization.getColorByName(steps[idx - 1].color)); + } else { + color = tinycolor(theme.visualization.getColorByName(step.color)); + } + + // Unless alpha specififed set to default value + if (color.getAlpha() === 1) { + color.setAlpha(0.7); + } + + let x0 = Math.round(u.valToPos(xMin!, 'x', true)); + let y0 = Math.round(u.valToPos(step.value, yScaleKey, true)); + let x1 = Math.round(u.valToPos(xMax!, 'x', true)); + let y1 = Math.round(u.valToPos(step.value, yScaleKey, true)); + + ctx.beginPath(); + ctx.moveTo(x0, y0); + ctx.lineTo(x1, y1); + + ctx.strokeStyle = color.toString(); + ctx.stroke(); + } + } + + function addAreas(u: uPlot, steps: Threshold[], theme: GrafanaTheme2) { + let ctx = u.ctx; + + let grd = scaleGradient( + u, + u.series[1].scale!, + steps.map((step) => { + let color = tinycolor(theme.visualization.getColorByName(step.color)); + + if (color.getAlpha() === 1) { + color.setAlpha(0.15); + } + + return [step.value, color.toString()]; + }), + true + ); + + ctx.fillStyle = grd; + ctx.fillRect(u.bbox.left, u.bbox.top, u.bbox.width, u.bbox.height); + } + + const { scaleKey, thresholds, theme, config, hardMin, hardMax, softMin, softMax } = options; + return (u: uPlot) => { const ctx = u.ctx; - const { scaleKey, thresholds, theme, config, hardMin, hardMax, softMin, softMax } = options; const { min: xMin, max: xMax } = u.scales.x; const { min: yMin, max: yMax } = u.scales[scaleKey]; @@ -38,82 +108,20 @@ export function getThresholdsDrawHook(options: UPlotThresholdOptions) { })); } - function addLines() { - // Thresholds below a transparent threshold is treated like "less than", and line drawn previous threshold - let transparentIndex = 0; - - for (let idx = 0; idx < steps.length; idx++) { - const step = steps[idx]; - if (step.color === 'transparent') { - transparentIndex = idx; - break; - } - } - - // Ignore the base -Infinity threshold by always starting on index 1 - for (let idx = 1; idx < steps.length; idx++) { - const step = steps[idx]; - let color: tinycolor.Instance; - - // if we are below a transparent index treat this a less then threshold, use previous thresholds color - if (transparentIndex >= idx && idx > 0) { - color = tinycolor(theme.visualization.getColorByName(steps[idx - 1].color)); - } else { - color = tinycolor(theme.visualization.getColorByName(step.color)); - } - - // Unless alpha specififed set to default value - if (color.getAlpha() === 1) { - color.setAlpha(0.7); - } - - let x0 = Math.round(u.valToPos(xMin!, 'x', true)); - let y0 = Math.round(u.valToPos(step.value, scaleKey, true)); - let x1 = Math.round(u.valToPos(xMax!, 'x', true)); - let y1 = Math.round(u.valToPos(step.value, scaleKey, true)); - - ctx.beginPath(); - ctx.lineWidth = 2; - ctx.strokeStyle = color.toString(); - ctx.moveTo(x0, y0); - ctx.lineTo(x1, y1); - ctx.stroke(); - ctx.closePath(); - } - } - - function addAreas() { - let grd = scaleGradient( - u, - u.series[1].scale!, - steps.map((step) => { - let color = tinycolor(theme.visualization.getColorByName(step.color)); - - if (color.getAlpha() === 1) { - color.setAlpha(0.15); - } - - return [step.value, color.toString()]; - }), - true - ); - - ctx.save(); - ctx.fillStyle = grd; - ctx.fillRect(u.bbox.left, u.bbox.top, u.bbox.width, u.bbox.height); - ctx.restore(); - } + ctx.save(); switch (config.mode) { case GraphTresholdsStyleMode.Line: - addLines(); + addLines(u, steps, theme, xMin, xMax, scaleKey); break; case GraphTresholdsStyleMode.Area: - addAreas(); + addAreas(u, steps, theme); break; case GraphTresholdsStyleMode.LineAndArea: - addLines(); - addAreas(); + addAreas(u, steps, theme); + addLines(u, steps, theme, xMin, xMax, scaleKey); } + + ctx.restore(); }; } diff --git a/packages/grafana-ui/src/components/uPlot/plugins/KeyboardPlugin.tsx b/packages/grafana-ui/src/components/uPlot/plugins/KeyboardPlugin.tsx new file mode 100644 index 00000000000..975f3b8dab2 --- /dev/null +++ b/packages/grafana-ui/src/components/uPlot/plugins/KeyboardPlugin.tsx @@ -0,0 +1,160 @@ +import React, { useLayoutEffect } from 'react'; +import { clamp } from 'lodash'; +import uPlot from 'uplot'; +import { UPlotConfigBuilder } from '../config/UPlotConfigBuilder'; + +interface KeyboardPluginProps { + config: UPlotConfigBuilder; // onkeypress, onkeyup, onkeydown (triggered by vizlayout handlers) +} + +const PIXELS_PER_MS = 0.1 as const; +const SHIFT_MULTIPLIER = 2 as const; +const KNOWN_KEYS = new Set(['ArrowRight', 'ArrowLeft', 'ArrowUp', 'ArrowDown', 'Shift', ' ']); + +const initHook = (u: uPlot) => { + let vizLayoutViz: HTMLElement | null = u.root.closest('[tabindex]'); + let pressedKeys = new Set(); + let dragStartX: number | null = null; + let keysLastHandledAt: number | null = null; + + if (!vizLayoutViz) { + return; + } + + const moveCursor = (dx: number, dy: number) => { + const { cursor } = u; + if (cursor.left === undefined || cursor.top === undefined) { + return; + } + + const { width, height } = u.over.style; + const [maxX, maxY] = [Math.floor(parseFloat(width)), Math.floor(parseFloat(height))]; + + u.setCursor({ + left: clamp(cursor.left + dx, 0, maxX), + top: clamp(cursor.top + dy, 0, maxY), + }); + }; + + const handlePressedKeys = (time: number) => { + const nothingPressed = pressedKeys.size === 0; + if (nothingPressed || !u) { + keysLastHandledAt = null; + return; + } + + const dt = time - (keysLastHandledAt ?? time); + const dx = dt * PIXELS_PER_MS; + let horValue = 0; + let vertValue = 0; + + if (pressedKeys.has('ArrowUp')) { + vertValue -= dx; + } + if (pressedKeys.has('ArrowDown')) { + vertValue += dx; + } + if (pressedKeys.has('ArrowLeft')) { + horValue -= dx; + } + if (pressedKeys.has('ArrowRight')) { + horValue += dx; + } + if (pressedKeys.has('Shift')) { + horValue *= SHIFT_MULTIPLIER; + vertValue *= SHIFT_MULTIPLIER; + } + + moveCursor(horValue, vertValue); + + const { cursor } = u; + if (pressedKeys.has(' ') && cursor) { + const drawHeight = Number(u.over.style.height.slice(0, -2)); + + u.setSelect( + { + left: cursor.left! < dragStartX! ? cursor.left! : dragStartX!, + top: 0, + width: Math.abs(cursor.left! - (dragStartX ?? cursor.left!)), + height: drawHeight, + }, + false + ); + } + + keysLastHandledAt = time; + window.requestAnimationFrame(handlePressedKeys); + }; + + vizLayoutViz.addEventListener('keydown', (e) => { + if (e.key === 'Tab') { + // Hide the cursor if the user tabs away + u.setCursor({ left: -5, top: -5 }); + return; + } + + if (!KNOWN_KEYS.has(e.key)) { + return; + } + + e.preventDefault(); + e.stopPropagation(); + + const newKey = !pressedKeys.has(e.key); + if (newKey) { + const initiateAnimationLoop = pressedKeys.size === 0; + pressedKeys.add(e.key); + + dragStartX = e.key === ' ' && dragStartX === null ? u.cursor.left! : dragStartX; + + if (initiateAnimationLoop) { + window.requestAnimationFrame(handlePressedKeys); + } + } + }); + + vizLayoutViz.addEventListener('keyup', (e) => { + if (!KNOWN_KEYS.has(e.key)) { + return; + } + + pressedKeys.delete(e.key); + + if (e.key === ' ') { + e.preventDefault(); + e.stopPropagation(); + + // We do this so setSelect hooks get fired, zooming the plot + u.setSelect(u.select); + dragStartX = null; + } + }); + + vizLayoutViz.addEventListener('focus', (e) => { + // We only want to initialize the cursor if the user is using keyboard controls + if (!vizLayoutViz?.matches(':focus-visible')) { + return; + } + + // Is there a more idiomatic way to do this? + const drawWidth = parseFloat(u.over.style.width); + const drawHeight = parseFloat(u.over.style.height); + u.setCursor({ left: drawWidth / 2, top: drawHeight / 2 }); + }); + + vizLayoutViz.addEventListener('blur', (e) => { + keysLastHandledAt = null; + dragStartX = null; + pressedKeys.clear(); + u.setSelect({ left: 0, top: 0, width: 0, height: 0 }, false); + }); +}; + +/** + * @alpha + */ +export const KeyboardPlugin: React.FC = ({ config }) => { + useLayoutEffect(() => config.addHook('init', initHook), [config]); + + return null; +}; diff --git a/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx b/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx index 75ebb4e8d8f..19053fc5f52 100644 --- a/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx +++ b/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx @@ -26,7 +26,7 @@ interface TooltipPluginProps { config: UPlotConfigBuilder; mode?: TooltipDisplayMode; sortOrder?: SortOrder; - sync?: DashboardCursorSync; + sync?: () => DashboardCursorSync; // Allows custom tooltip content rendering. Exposes aligned data frame with relevant indexes for data inspection // Use field.state.origin indexes from alignedData frame field to get access to original data frame and field index. renderTooltip?: (alignedFrame: DataFrame, seriesIdx: number | null, datapointIdx: number | null) => React.ReactNode; @@ -66,7 +66,15 @@ export const TooltipPlugin: React.FC = ({ useLayoutEffect(() => { let bbox: DOMRect | undefined = undefined; - const plotMouseLeave = () => { + const plotEnter = () => { + if (!isMounted()) { + return; + } + setIsActive(true); + plotInstance.current?.root.classList.add('plot-active'); + }; + + const plotLeave = () => { if (!isMounted()) { return; } @@ -75,24 +83,19 @@ export const TooltipPlugin: React.FC = ({ plotInstance.current?.root.classList.remove('plot-active'); }; - const plotMouseEnter = () => { - if (!isMounted()) { - return; - } - setIsActive(true); - plotInstance.current?.root.classList.add('plot-active'); - }; - // cache uPlot plotting area bounding box config.addHook('syncRect', (u, rect) => (bbox = rect)); config.addHook('init', (u) => { plotInstance.current = u; - u.over.addEventListener('mouseleave', plotMouseLeave); - u.over.addEventListener('mouseenter', plotMouseEnter); + u.root.parentElement?.addEventListener('focus', plotEnter); + u.over.addEventListener('mouseenter', plotEnter); - if (sync === DashboardCursorSync.Crosshair) { + u.root.parentElement?.addEventListener('blur', plotLeave); + u.over.addEventListener('mouseleave', plotLeave); + + if (sync && sync() === DashboardCursorSync.Crosshair) { u.root.classList.add('shared-crosshair'); } }); @@ -157,13 +160,15 @@ export const TooltipPlugin: React.FC = ({ return () => { setCoords(null); if (plotInstance.current) { - plotInstance.current.over.removeEventListener('mouseleave', plotMouseLeave); - plotInstance.current.over.removeEventListener('mouseenter', plotMouseEnter); + plotInstance.current.over.removeEventListener('mouseleave', plotLeave); + plotInstance.current.over.removeEventListener('mouseenter', plotEnter); + plotInstance.current.root.parentElement?.removeEventListener('focus', plotEnter); + plotInstance.current.root.parentElement?.removeEventListener('blur', plotLeave); } }; }, [config, setCoords, setIsActive, setFocusedPointIdx, setFocusedPointIdxs]); - if (focusedPointIdx === null || (!isActive && sync === DashboardCursorSync.Crosshair)) { + if (focusedPointIdx === null || (!isActive && sync && sync() === DashboardCursorSync.Crosshair)) { return null; } diff --git a/packages/grafana-ui/src/components/uPlot/plugins/index.ts b/packages/grafana-ui/src/components/uPlot/plugins/index.ts index 4d8cfbb9f9f..23c742bab8c 100644 --- a/packages/grafana-ui/src/components/uPlot/plugins/index.ts +++ b/packages/grafana-ui/src/components/uPlot/plugins/index.ts @@ -1,2 +1,3 @@ export { ZoomPlugin } from './ZoomPlugin'; export { TooltipPlugin } from './TooltipPlugin'; +export { KeyboardPlugin } from './KeyboardPlugin'; diff --git a/packages/grafana-ui/src/components/uPlot/utils.test.ts b/packages/grafana-ui/src/components/uPlot/utils.test.ts index 3897377bba5..47060f9ddc0 100644 --- a/packages/grafana-ui/src/components/uPlot/utils.test.ts +++ b/packages/grafana-ui/src/components/uPlot/utils.test.ts @@ -296,6 +296,132 @@ describe('preparePlotData', () => { `); }); + describe('ignores nullish-only stacks', () => { + test('single stacking group', () => { + const df = new MutableDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [9997, 9998, 9999] }, + { + name: 'a', + values: [-10, null, 10], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackA' } } }, + }, + { + name: 'b', + values: [10, null, null], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackA' } } }, + }, + { + name: 'c', + values: [20, undefined, 20], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackA' } } }, + }, + ], + }); + + expect(preparePlotData([df])).toMatchInlineSnapshot(` + Array [ + Array [ + 9997, + 9998, + 9999, + ], + Array [ + -10, + null, + 10, + ], + Array [ + 0, + null, + 10, + ], + Array [ + 20, + null, + 30, + ], + ] + `); + }); + test('multiple stacking groups', () => { + const df = new MutableDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [9997, 9998, 9999] }, + { + name: 'a', + values: [-10, undefined, 10], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackA' } } }, + }, + { + name: 'b', + values: [10, undefined, 10], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackA' } } }, + }, + { + name: 'c', + values: [20, undefined, 20], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackA' } } }, + }, + { + name: 'd', + values: [1, 2, null], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackB' } } }, + }, + { + name: 'e', + values: [1, 2, null], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackB' } } }, + }, + { + name: 'f', + values: [1, 2, null], + config: { custom: { stacking: { mode: StackingMode.Normal, group: 'stackB' } } }, + }, + ], + }); + + expect(preparePlotData([df])).toMatchInlineSnapshot(` + Array [ + Array [ + 9997, + 9998, + 9999, + ], + Array [ + -10, + null, + 10, + ], + Array [ + 0, + null, + 20, + ], + Array [ + 20, + null, + 40, + ], + Array [ + 1, + 2, + null, + ], + Array [ + 2, + 4, + null, + ], + Array [ + 3, + 6, + null, + ], + ] + `); + }); + }); describe('with legend sorted', () => { it('should affect when single group', () => { const df = new MutableDataFrame({ @@ -323,53 +449,53 @@ describe('preparePlotData', () => { }); expect(preparePlotData([df], undefined, { sortBy: 'Max', sortDesc: false } as any)).toMatchInlineSnapshot(` - Array [ - Array [ - 9997, - 9998, - 9999, - ], - Array [ - 0, - 30, - 20, - ], - Array [ - 10, - 10, - 10, - ], - Array [ - 20, - 50, - 40, - ], - ] - `); + Array [ + Array [ + 9997, + 9998, + 9999, + ], + Array [ + 0, + 30, + 20, + ], + Array [ + 10, + 10, + 10, + ], + Array [ + 20, + 50, + 40, + ], + ] + `); expect(preparePlotData([df], undefined, { sortBy: 'Max', sortDesc: true } as any)).toMatchInlineSnapshot(` - Array [ - Array [ - 9997, - 9998, - 9999, - ], - Array [ - -10, - 20, - 10, - ], - Array [ - 20, - 50, - 40, - ], - Array [ - 10, - 40, - 30, - ], - ] - `); + Array [ + Array [ + 9997, + 9998, + 9999, + ], + Array [ + -10, + 20, + 10, + ], + Array [ + 20, + 50, + 40, + ], + Array [ + 10, + 40, + 30, + ], + ] + `); }); }); }); diff --git a/packages/grafana-ui/src/components/uPlot/utils.ts b/packages/grafana-ui/src/components/uPlot/utils.ts index 426d14b024a..0a0288ddade 100755 --- a/packages/grafana-ui/src/components/uPlot/utils.ts +++ b/packages/grafana-ui/src/components/uPlot/utils.ts @@ -74,7 +74,7 @@ export function preparePlotData( // array or stacking groups for (const [_, seriesIds] of stackingGroups.entries()) { const seriesIdxs = orderIdsByCalcs({ ids: seriesIds, legend, frame }); - + const noValueStack = Array(dataLength).fill(true); const groupTotals = byPct ? Array(dataLength).fill(0) : null; if (byPct) { @@ -99,10 +99,13 @@ export function preparePlotData( for (let k = 0; k < dataLength; k++) { const v = currentlyStacking[k]; + if (v != null && noValueStack[k]) { + noValueStack[k] = false; + } acc[k] += v == null ? 0 : v / (byPct ? groupTotals![k] : 1); } - result[seriesIdx] = acc.slice(); + result[seriesIdx] = acc.slice().map((v, i) => (noValueStack[i] ? null : v)); } } diff --git a/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts b/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts index 3a807f5e695..e4ee63b4af2 100644 --- a/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts +++ b/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts @@ -10,6 +10,9 @@ export const darkThemeVarsTemplate = (theme: GrafanaTheme2) => $theme-name: dark; +$colors-action-hover: ${theme.colors.action.hover}; +$colors-action-selected: ${theme.colors.action.selected}; + // New Colors // ------------------------- $blue-light: ${theme.colors.primary.text}; diff --git a/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts b/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts index 57392812dce..196faabe1df 100644 --- a/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts +++ b/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts @@ -11,6 +11,9 @@ export const lightThemeVarsTemplate = (theme: GrafanaTheme2) => $theme-name: light; +$colors-action-hover: ${theme.colors.action.hover}; +$colors-action-selected: ${theme.colors.action.selected}; + // New Colors // ------------------------- $blue-light: ${theme.colors.primary.text}; diff --git a/packages/grafana-ui/src/types/icon.ts b/packages/grafana-ui/src/types/icon.ts index 627a0564744..3d55da853b3 100644 --- a/packages/grafana-ui/src/types/icon.ts +++ b/packages/grafana-ui/src/types/icon.ts @@ -114,6 +114,7 @@ export const getAvailableIcons = () => 'list-ui-alt', 'list-ul', 'lock', + 'map-marker', 'message', 'minus', 'minus-circle', @@ -187,6 +188,8 @@ export function getFieldTypeIcon(field?: Field): IconName { return 'toggle-on'; case FieldType.trace: return 'info-circle'; + case FieldType.geo: + return 'map-marker'; case FieldType.other: return 'brackets-curly'; } diff --git a/packages/grafana-ui/src/utils/standardEditors.tsx b/packages/grafana-ui/src/utils/standardEditors.tsx index b5570c4a587..8ddbb3cf528 100644 --- a/packages/grafana-ui/src/utils/standardEditors.tsx +++ b/packages/grafana-ui/src/utils/standardEditors.tsx @@ -10,13 +10,6 @@ import { StandardEditorsRegistryItem, StringFieldConfigSettings, stringOverrideProcessor, - ThresholdsConfig, - ThresholdsFieldConfigSettings, - thresholdsOverrideProcessor, - ValueMapping, - ValueMappingFieldConfigSettings, - valueMappingsOverrideProcessor, - ThresholdsMode, identityOverrideProcessor, TimeZone, FieldColor, @@ -37,8 +30,6 @@ import { MultiSelectValueEditor, TimeZonePicker, } from '../components'; -import { ValueMappingsValueEditor } from '../components/OptionsUI/mappings'; -import { ThresholdsValueEditor } from '../components/OptionsUI/thresholds'; import { UnitValueEditor } from '../components/OptionsUI/units'; import { DataLinksValueEditor } from '../components/OptionsUI/links'; import { ColorValueEditor } from '../components/OptionsUI/color'; @@ -140,42 +131,6 @@ export const getStandardFieldConfigs = () => { category, }; - const thresholds: FieldConfigPropertyItem = { - id: 'thresholds', - path: 'thresholds', - name: 'Thresholds', - editor: standardEditorsRegistry.get('thresholds').editor as any, - override: standardEditorsRegistry.get('thresholds').editor as any, - process: thresholdsOverrideProcessor, - settings: {}, - defaultValue: { - mode: ThresholdsMode.Absolute, - steps: [ - { value: -Infinity, color: 'green' }, - { value: 80, color: 'red' }, - ], - }, - shouldApply: () => true, - category: ['Thresholds'], - getItemsCount: (value) => (value ? value.steps.length : 0), - }; - - const mappings: FieldConfigPropertyItem = { - id: 'mappings', - path: 'mappings', - name: 'Value mappings', - description: 'Modify the display text based on input value', - - editor: standardEditorsRegistry.get('mappings').editor as any, - override: standardEditorsRegistry.get('mappings').editor as any, - process: valueMappingsOverrideProcessor, - settings: {}, - defaultValue: [], - shouldApply: (x) => x.type !== FieldType.time, - category: ['Value mappings'], - getItemsCount: (value?) => (value ? value.length : 0), - }; - const noValue: FieldConfigPropertyItem = { id: 'noValue', path: 'noValue', @@ -224,7 +179,7 @@ export const getStandardFieldConfigs = () => { category, }; - return [unit, min, max, decimals, displayName, color, noValue, thresholds, mappings, links]; + return [unit, min, max, decimals, displayName, color, noValue, links]; }; /** @@ -300,20 +255,6 @@ export const getStandardOptionEditors = () => { editor: UnitValueEditor as any, }; - const thresholds: StandardEditorsRegistryItem = { - id: 'thresholds', - name: 'Thresholds', - description: 'Allows defining thresholds', - editor: ThresholdsValueEditor as any, - }; - - const mappings: StandardEditorsRegistryItem = { - id: 'mappings', - name: 'Mappings', - description: 'Allows defining value mappings', - editor: ValueMappingsValueEditor as any, - }; - const color: StandardEditorsRegistryItem = { id: 'color', name: 'Color', @@ -366,8 +307,6 @@ export const getStandardOptionEditors = () => { radio, select, unit, - mappings, - thresholds, links, statsPicker, strings, diff --git a/packages/jaeger-ui-components/package.json b/packages/jaeger-ui-components/package.json index b69e41c3726..5de63be37d2 100644 --- a/packages/jaeger-ui-components/package.json +++ b/packages/jaeger-ui-components/package.json @@ -20,11 +20,10 @@ "@types/react-icons": "2.2.7", "@types/recompose": "0.30.10", "@types/reselect": "2.2.0", - "@types/slate-react": "0.22.5", "@types/tinycolor2": "1.4.3", "enzyme": "3.11.0", - "sinon": "12.0.1", - "typescript": "4.5.4" + "sinon": "13.0.0", + "typescript": "4.4.4" }, "dependencies": { "@emotion/css": "11.7.1", diff --git a/packages/jaeger-ui-components/src/TraceTimelineViewer/ListView/index.tsx b/packages/jaeger-ui-components/src/TraceTimelineViewer/ListView/index.tsx index 22bc0aeb421..810a6218784 100644 --- a/packages/jaeger-ui-components/src/TraceTimelineViewer/ListView/index.tsx +++ b/packages/jaeger-ui-components/src/TraceTimelineViewer/ListView/index.tsx @@ -246,6 +246,20 @@ export default class ListView extends React.Component { getRowPosition = (index: number): { height: number; y: number } => this._yPositions.getRowPosition(index, this._getHeight); + scrollToIndex = (index: number) => { + // calculate the position of the list view relative to the scroll parent + const { scrollElement } = this.props; + const scrollElementTop = scrollElement?.getBoundingClientRect().top || 0; + const listViewTop = (scrollElement?.scrollTop || 0) + (this._itemHolderElm?.getBoundingClientRect().top || 0); + const listViewOffset = listViewTop - scrollElementTop; + + const itemOffset = this.getRowPosition(index).y; + + // hard code a small offset to leave a little bit of space above the focused span, so it is visually clear + // that there is content above + this.props.scrollElement?.scrollTo({ top: itemOffset + listViewOffset - 80 }); + }; + /** * Scroll event listener that schedules a remeasuring of which items should be * rendered. diff --git a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanBarRow.tsx b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanBarRow.tsx index 19a517749b1..0a5caf1dce4 100644 --- a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanBarRow.tsx +++ b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanBarRow.tsx @@ -17,7 +17,7 @@ import IoAlert from 'react-icons/lib/io/alert'; import IoArrowRightA from 'react-icons/lib/io/arrow-right-a'; import IoNetwork from 'react-icons/lib/io/network'; import MdFileUpload from 'react-icons/lib/md/file-upload'; -import { css } from '@emotion/css'; +import { css, keyframes } from '@emotion/css'; import cx from 'classnames'; import { stylesFactory, withTheme2 } from '@grafana/ui'; import { GrafanaTheme2 } from '@grafana/data'; @@ -41,6 +41,18 @@ const viewClassName = 'jaegerView'; const nameColumnClassName = 'nameColumn'; const getStyles = stylesFactory((theme: GrafanaTheme2) => { + const animations = { + flash: keyframes` + label: flash; + from { + background-color: ${autoColor(theme, '#68b9ff')}; + } + to { + background-color: default; + } + `, + }; + return { nameWrapper: css` label: nameWrapper; @@ -168,6 +180,25 @@ const getStyles = stylesFactory((theme: GrafanaTheme2) => { outline: 1px solid ${autoColor(theme, '#ddd')}; } `, + rowFocused: css` + label: rowFocused; + background-color: ${autoColor(theme, '#cbe7ff')}; + animation: ${animations.flash} 1s cubic-bezier(0.12, 0, 0.39, 0); + & .${nameWrapperClassName}, .${viewClassName}, .${nameWrapperMatchingFilterClassName} { + background-color: ${autoColor(theme, '#cbe7ff')}; + animation: ${animations.flash} 1s cubic-bezier(0.12, 0, 0.39, 0); + } + & .${spanBarClassName} { + opacity: 1; + } + & .${spanBarLabelClassName} { + color: ${autoColor(theme, '#000')}; + } + &:hover .${nameWrapperClassName}, :hover .${viewClassName} { + background: ${autoColor(theme, '#d5ebff')}; + box-shadow: 0 1px 0 ${autoColor(theme, '#ddd')}; + } + `, rowExpandedAndMatchingFilter: css` label: rowExpandedAndMatchingFilter; @@ -268,6 +299,7 @@ type SpanBarRowProps = { isChildrenExpanded: boolean; isDetailExpanded: boolean; isMatchingFilter: boolean; + isFocused: boolean; onDetailToggled: (spanID: string) => void; onChildrenToggled: (spanID: string) => void; numTicks: number; @@ -330,6 +362,7 @@ export class UnthemedSpanBarRow extends React.PureComponent { isChildrenExpanded, isDetailExpanded, isMatchingFilter, + isFocused, numTicks, rpc, noInstrumentedServer, @@ -377,6 +410,7 @@ export class UnthemedSpanBarRow extends React.PureComponent { [styles.rowExpanded]: isDetailExpanded, [styles.rowMatchingFilter]: isMatchingFilter, [styles.rowExpandedAndMatchingFilter]: isMatchingFilter && isDetailExpanded, + [styles.rowFocused]: isFocused, [styles.rowClippingLeft]: clippingLeft, [styles.rowClippingRight]: clippingRight, }, diff --git a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.test.js b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.test.js index 273956ef4ae..b63b2bfcad5 100644 --- a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.test.js +++ b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.test.js @@ -22,7 +22,6 @@ import AccordianLogs from './AccordianLogs'; import DetailState from './DetailState'; import SpanDetail from './index'; import { formatDuration } from '../utils'; -import CopyIcon from '../../common/CopyIcon'; import LabeledList from '../../common/LabeledList'; import traceGenerator from '../../demo/trace-generators'; import transformTraceData from '../../model/transform-trace-data'; @@ -44,6 +43,7 @@ describe('', () => { tagsToggle: jest.fn(), warningsToggle: jest.fn(), referencesToggle: jest.fn(), + createFocusSpanLink: jest.fn(), }; span.logs = [ { @@ -180,7 +180,7 @@ describe('', () => { expect(props.referencesToggle).toHaveBeenLastCalledWith(span.spanID); }); - it('renders CopyIcon with deep link URL', () => { - expect(wrapper.find(CopyIcon).prop('copyText').includes(`?uiFind=${props.span.spanID}`)).toBe(true); + it('renders deep link URL', () => { + expect(wrapper.find('a').exists()).toBeTruthy(); }); }); diff --git a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.tsx b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.tsx index a1ea131a29d..9b3add1d7a6 100644 --- a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.tsx +++ b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetail/index.tsx @@ -16,21 +16,30 @@ import React from 'react'; import { css } from '@emotion/css'; import cx from 'classnames'; import { DataLinkButton, TextArea, useStyles2 } from '@grafana/ui'; -import { GrafanaTheme2 } from '@grafana/data'; +import { GrafanaTheme2, LinkModel } from '@grafana/data'; +import IoLink from 'react-icons/lib/io/link'; import AccordianKeyValues from './AccordianKeyValues'; import AccordianLogs from './AccordianLogs'; import AccordianText from './AccordianText'; import DetailState from './DetailState'; import { formatDuration } from '../utils'; -import CopyIcon from '../../common/CopyIcon'; import LabeledList from '../../common/LabeledList'; import { SpanLinkFunc, TNil } from '../../types'; import { TraceKeyValuePair, TraceLink, TraceLog, TraceSpan } from '../../types/trace'; import AccordianReferences from './AccordianReferences'; import { autoColor } from '../../Theme'; import { UIDivider } from '../../uiElementsContext'; -import { ubFlex, ubFlexAuto, ubItemsCenter, ubM0, ubMb1, ubMy1, ubTxRightAlign } from '../../uberUtilityStyles'; +import { + uAlignIcon, + ubFlex, + ubFlexAuto, + ubItemsCenter, + ubM0, + ubMb1, + ubMy1, + ubTxRightAlign, +} from '../../uberUtilityStyles'; const getStyles = (theme: GrafanaTheme2) => { return { @@ -99,6 +108,9 @@ const getStyles = (theme: GrafanaTheme2) => { word-break: break-all; white-space: pre; `, + LinkIcon: css` + font-size: 1.5em; + `, }; }; @@ -116,6 +128,8 @@ type SpanDetailProps = { referencesToggle: (spanID: string) => void; focusSpan: (uiFind: string) => void; createSpanLink?: SpanLinkFunc; + focusedSpanId?: string; + createFocusSpanLink: (traceId: string, spanId: string) => LinkModel; }; export default function SpanDetail(props: SpanDetailProps) { @@ -133,6 +147,7 @@ export default function SpanDetail(props: SpanDetailProps) { referencesToggle, focusSpan, createSpanLink, + createFocusSpanLink, } = props; const { isTagsOpen, @@ -147,6 +162,7 @@ export default function SpanDetail(props: SpanDetailProps) { process, duration, relativeStartTime, + traceID, spanID, logs, tags, @@ -171,9 +187,9 @@ export default function SpanDetail(props: SpanDetailProps) { value: formatDuration(relativeStartTime), }, ]; - const deepLinkCopyText = `${window.location.origin}${window.location.pathname}?uiFind=${spanID}`; const styles = useStyles2(getStyles); const link = createSpanLink?.(span); + const focusSpanLink = createFocusSpanLink(traceID, spanID); return (
diff --git a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetailRow.tsx b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetailRow.tsx index d6cdadac5f7..d254f34d554 100644 --- a/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetailRow.tsx +++ b/packages/jaeger-ui-components/src/TraceTimelineViewer/SpanDetailRow.tsx @@ -21,7 +21,7 @@ import SpanTreeOffset from './SpanTreeOffset'; import TimelineRow from './TimelineRow'; import { autoColor } from '../Theme'; import { stylesFactory, withTheme2 } from '@grafana/ui'; -import { GrafanaTheme2 } from '@grafana/data'; +import { GrafanaTheme2, LinkModel } from '@grafana/data'; import { TraceLog, TraceSpan, TraceKeyValuePair, TraceLink } from '../types/trace'; import { SpanLinkFunc } from '../types'; @@ -89,6 +89,8 @@ type SpanDetailRowProps = { removeHoverIndentGuideId: (spanID: string) => void; theme: GrafanaTheme2; createSpanLink?: SpanLinkFunc; + focusedSpanId?: string; + createFocusSpanLink: (traceId: string, spanId: string) => LinkModel; }; export class UnthemedSpanDetailRow extends React.PureComponent { @@ -121,6 +123,8 @@ export class UnthemedSpanDetailRow extends React.PureComponent diff --git a/packages/jaeger-ui-components/src/TraceTimelineViewer/VirtualizedTraceView.tsx b/packages/jaeger-ui-components/src/TraceTimelineViewer/VirtualizedTraceView.tsx index 01eed9b8b6f..bf34107f896 100644 --- a/packages/jaeger-ui-components/src/TraceTimelineViewer/VirtualizedTraceView.tsx +++ b/packages/jaeger-ui-components/src/TraceTimelineViewer/VirtualizedTraceView.tsx @@ -18,7 +18,7 @@ import { css } from '@emotion/css'; import { isEqual } from 'lodash'; import memoizeOne from 'memoize-one'; import { stylesFactory, withTheme2 } from '@grafana/ui'; -import { GrafanaTheme2 } from '@grafana/data'; +import { GrafanaTheme2, LinkModel } from '@grafana/data'; import ListView from './ListView'; import SpanBarRow from './SpanBarRow'; @@ -86,6 +86,8 @@ type TVirtualizedTraceViewOwnProps = { theme: GrafanaTheme2; createSpanLink?: SpanLinkFunc; scrollElement?: Element; + focusedSpanId?: string; + createFocusSpanLink: (traceId: string, spanId: string) => LinkModel; }; type VirtualizedTraceViewProps = TVirtualizedTraceViewOwnProps & TExtractUiFindFromStateReturn & TTraceTimeline; @@ -172,6 +174,10 @@ export class UnthemedVirtualizedTraceView extends React.Component; @@ -200,6 +206,7 @@ export class UnthemedVirtualizedTraceView extends React.Component { + if (spanID == null) { + return; + } + const i = this.getRowStates().findIndex((row) => row.span.spanID === spanID); + if (i >= 0) { + this.listView?.scrollToIndex(i); + } + }; + renderSpanBarRow(span: TraceSpan, spanIndex: number, key: string, style: React.CSSProperties, attrs: {}) { const { spanID } = span; const { serviceName } = span.process; @@ -340,6 +361,7 @@ export class UnthemedVirtualizedTraceView extends React.Component ); diff --git a/packages/jaeger-ui-components/src/TraceTimelineViewer/index.tsx b/packages/jaeger-ui-components/src/TraceTimelineViewer/index.tsx index b8e560a40d5..4d09b2ba79e 100644 --- a/packages/jaeger-ui-components/src/TraceTimelineViewer/index.tsx +++ b/packages/jaeger-ui-components/src/TraceTimelineViewer/index.tsx @@ -14,7 +14,7 @@ import React from 'react'; import { css } from '@emotion/css'; -import { GrafanaTheme2 } from '@grafana/data'; +import { GrafanaTheme2, LinkModel } from '@grafana/data'; import { stylesFactory, withTheme2 } from '@grafana/ui'; import TimelineHeaderRow from './TimelineHeaderRow'; @@ -103,6 +103,8 @@ type TProps = TExtractUiFindFromStateReturn & { theme: GrafanaTheme2; createSpanLink?: SpanLinkFunc; scrollElement?: Element; + focusedSpanId?: string; + createFocusSpanLink: (traceId: string, spanId: string) => LinkModel; }; type State = { diff --git a/packaging/docker/README.md b/packaging/docker/README.md index fcce71778f9..ea592592fce 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -28,7 +28,7 @@ Further documentation can be found at http://docs.grafana.org/installation/docke ### v7.3.0-beta1 -- OpenShift compatability. [#27813](https://github.com/grafana/grafana/pull/27813), [@xlson](https://github.com/grafana/grafana/pull/27813) +- OpenShift compatibility. [#27813](https://github.com/grafana/grafana/pull/27813), [@xlson](https://github.com/grafana/grafana/pull/27813) ### v7.0.0-beta3 diff --git a/pkg/api/admin_users.go b/pkg/api/admin_users.go index 01f595faa72..631a7abe6b2 100644 --- a/pkg/api/admin_users.go +++ b/pkg/api/admin_users.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" @@ -67,7 +68,11 @@ func AdminUpdateUserPassword(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &form); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - userID := c.ParamsInt64(":id") + + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } if len(form.Password) < 4 { return response.Error(400, "New password too short", nil) @@ -102,9 +107,12 @@ func (hs *HTTPServer) AdminUpdateUserPermissions(c *models.ReqContext) response. if err := web.Bind(c.Req, &form); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - userID := c.ParamsInt64(":id") + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } - err := updateUserPermissions(hs.SQLStore, userID, form.IsGrafanaAdmin) + err = updateUserPermissions(hs.SQLStore, userID, form.IsGrafanaAdmin) if err != nil { if errors.Is(err, models.ErrLastGrafanaAdmin) { return response.Error(400, models.ErrLastGrafanaAdmin.Error(), nil) @@ -117,7 +125,10 @@ func (hs *HTTPServer) AdminUpdateUserPermissions(c *models.ReqContext) response. } func AdminDeleteUser(c *models.ReqContext) response.Response { - userID := c.ParamsInt64(":id") + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } cmd := models.DeleteUserCommand{UserId: userID} @@ -133,7 +144,10 @@ func AdminDeleteUser(c *models.ReqContext) response.Response { // POST /api/admin/users/:id/disable func (hs *HTTPServer) AdminDisableUser(c *models.ReqContext) response.Response { - userID := c.ParamsInt64(":id") + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } // External users shouldn't be disabled from API authInfoQuery := &models.GetAuthInfoQuery{UserId: userID} @@ -149,7 +163,7 @@ func (hs *HTTPServer) AdminDisableUser(c *models.ReqContext) response.Response { return response.Error(500, "Failed to disable user", err) } - err := hs.AuthTokenService.RevokeAllUserTokens(c.Req.Context(), userID) + err = hs.AuthTokenService.RevokeAllUserTokens(c.Req.Context(), userID) if err != nil { return response.Error(500, "Failed to disable user", err) } @@ -159,7 +173,10 @@ func (hs *HTTPServer) AdminDisableUser(c *models.ReqContext) response.Response { // POST /api/admin/users/:id/enable func AdminEnableUser(c *models.ReqContext) response.Response { - userID := c.ParamsInt64(":id") + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } // External users shouldn't be disabled from API authInfoQuery := &models.GetAuthInfoQuery{UserId: userID} @@ -180,7 +197,10 @@ func AdminEnableUser(c *models.ReqContext) response.Response { // POST /api/admin/users/:id/logout func (hs *HTTPServer) AdminLogoutUser(c *models.ReqContext) response.Response { - userID := c.ParamsInt64(":id") + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } if c.UserId == userID { return response.Error(400, "You cannot logout yourself", nil) @@ -191,7 +211,10 @@ func (hs *HTTPServer) AdminLogoutUser(c *models.ReqContext) response.Response { // GET /api/admin/users/:id/auth-tokens func (hs *HTTPServer) AdminGetUserAuthTokens(c *models.ReqContext) response.Response { - userID := c.ParamsInt64(":id") + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } return hs.getUserAuthTokensInternal(c, userID) } @@ -201,7 +224,10 @@ func (hs *HTTPServer) AdminRevokeUserAuthToken(c *models.ReqContext) response.Re if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - userID := c.ParamsInt64(":id") + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } return hs.revokeUserAuthTokenInternal(c, userID, cmd) } diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 739cde47e5b..9162f30c72d 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -21,7 +21,11 @@ import ( ) func ValidateOrgAlert(c *models.ReqContext) { - id := c.ParamsInt64(":alertId") + id, err := strconv.ParseInt(web.Params(c.Req)[":alertId"], 10, 64) + if err != nil { + c.JsonApiErr(http.StatusBadRequest, "alertId is invalid", nil) + return + } query := models.GetAlertByIdQuery{Id: id} if err := bus.Dispatch(c.Req.Context(), &query); err != nil { @@ -178,7 +182,10 @@ func (hs *HTTPServer) AlertTest(c *models.ReqContext) response.Response { // GET /api/alerts/:id func GetAlert(c *models.ReqContext) response.Response { - id := c.ParamsInt64(":alertId") + id, err := strconv.ParseInt(web.Params(c.Req)[":alertId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "alertId is invalid", err) + } query := models.GetAlertByIdQuery{Id: id} if err := bus.Dispatch(c.Req.Context(), &query); err != nil { @@ -240,9 +247,13 @@ func getAlertNotificationsInternal(c *models.ReqContext) ([]*models.AlertNotific } func GetAlertNotificationByID(c *models.ReqContext) response.Response { + notificationId, err := strconv.ParseInt(web.Params(c.Req)[":notificationId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "notificationId is invalid", err) + } query := &models.GetAlertNotificationsQuery{ OrgId: c.OrgId, - Id: c.ParamsInt64(":notificationId"), + Id: notificationId, } if query.Id == 0 { @@ -426,9 +437,14 @@ func (hs *HTTPServer) fillWithSecureSettingsDataByUID(ctx context.Context, cmd * } func DeleteAlertNotification(c *models.ReqContext) response.Response { + notificationId, err := strconv.ParseInt(web.Params(c.Req)[":notificationId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "notificationId is invalid", err) + } + cmd := models.DeleteAlertNotificationCommand{ OrgId: c.OrgId, - Id: c.ParamsInt64(":notificationId"), + Id: notificationId, } if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { @@ -496,7 +512,10 @@ func PauseAlert(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &dto); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - alertID := c.ParamsInt64(":alertId") + alertID, err := strconv.ParseInt(web.Params(c.Req)[":alertId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "alertId is invalid", err) + } result := make(map[string]interface{}) result["alertId"] = alertID diff --git a/pkg/api/annotations.go b/pkg/api/annotations.go index 1d90aaba4dc..24e725f3654 100644 --- a/pkg/api/annotations.go +++ b/pkg/api/annotations.go @@ -3,6 +3,7 @@ package api import ( "errors" "net/http" + "strconv" "strings" "github.com/grafana/grafana/pkg/api/dtos" @@ -164,7 +165,11 @@ func UpdateAnnotation(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - annotationID := c.ParamsInt64(":annotationId") + + annotationID, err := strconv.ParseInt(web.Params(c.Req)[":annotationId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "annotationId is invalid", err) + } repo := annotations.GetRepository() @@ -194,7 +199,10 @@ func PatchAnnotation(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - annotationID := c.ParamsInt64(":annotationId") + annotationID, err := strconv.ParseInt(web.Params(c.Req)[":annotationId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "annotationId is invalid", err) + } repo := annotations.GetRepository() @@ -264,13 +272,16 @@ func DeleteAnnotations(c *models.ReqContext) response.Response { func DeleteAnnotationByID(c *models.ReqContext) response.Response { repo := annotations.GetRepository() - annotationID := c.ParamsInt64(":annotationId") + annotationID, err := strconv.ParseInt(web.Params(c.Req)[":annotationId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "annotationId is invalid", err) + } if resp := canSave(c, repo, annotationID); resp != nil { return resp } - err := repo.Delete(&annotations.DeleteParams{ + err = repo.Delete(&annotations.DeleteParams{ OrgId: c.OrgId, Id: annotationID, }) diff --git a/pkg/api/api.go b/pkg/api/api.go index 8fd4bb4cec9..ea6c3859c08 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/models" ac "github.com/grafana/grafana/pkg/services/accesscontrol" acmiddleware "github.com/grafana/grafana/pkg/services/accesscontrol/middleware" + "github.com/grafana/grafana/pkg/services/featuremgmt" sa "github.com/grafana/grafana/pkg/services/serviceaccounts/manager" ) @@ -54,11 +55,13 @@ func (hs *HTTPServer) registerRoutes() { r.Get("/datasources/", authorize(reqOrgAdmin, dataSourcesConfigurationAccessEvaluator), hs.Index) r.Get("/datasources/new", authorize(reqOrgAdmin, dataSourcesNewAccessEvaluator), hs.Index) r.Get("/datasources/edit/*", authorize(reqOrgAdmin, dataSourcesEditAccessEvaluator), hs.Index) - r.Get("/org/users", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead, ac.ScopeUsersAll)), hs.Index) + r.Get("/org/users", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead)), hs.Index) r.Get("/org/users/new", reqOrgAdmin, hs.Index) r.Get("/org/users/invite", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionUsersCreate)), hs.Index) r.Get("/org/teams", reqCanAccessTeams, hs.Index) r.Get("/org/teams/*", reqCanAccessTeams, hs.Index) + r.Get("/org/serviceaccounts", middleware.ReqOrgAdmin, hs.Index) + r.Get("/org/serviceaccounts/:serviceAccountId", middleware.ReqOrgAdmin, hs.Index) r.Get("/org/apikeys/", reqOrgAdmin, hs.Index) r.Get("/dashboard/import/", reqSignedIn, hs.Index) r.Get("/configuration", reqGrafanaAdmin, hs.Index) @@ -113,11 +116,11 @@ func (hs *HTTPServer) registerRoutes() { r.Get("/verify", hs.Index) r.Get("/signup", hs.Index) r.Get("/api/user/signup/options", routing.Wrap(GetSignUpOptions)) - r.Post("/api/user/signup", quota("user"), routing.Wrap(SignUp)) + r.Post("/api/user/signup", quota("user"), routing.Wrap(hs.SignUp)) r.Post("/api/user/signup/step2", routing.Wrap(hs.SignUpStep2)) // invited - r.Get("/api/user/invite/:code", routing.Wrap(GetInviteInfoByCode)) + r.Get("/api/user/invite/:code", routing.Wrap(hs.GetInviteInfoByCode)) r.Post("/api/user/invite/complete", routing.Wrap(hs.CompleteInvite)) // reset password @@ -142,19 +145,19 @@ func (hs *HTTPServer) registerRoutes() { // user (signed in) apiRoute.Group("/user", func(userRoute routing.RouteRegister) { userRoute.Get("/", routing.Wrap(hs.GetSignedInUser)) - userRoute.Put("/", routing.Wrap(UpdateSignedInUser)) - userRoute.Post("/using/:id", routing.Wrap(UserSetUsingOrg)) - userRoute.Get("/orgs", routing.Wrap(GetSignedInUserOrgList)) - userRoute.Get("/teams", routing.Wrap(GetSignedInUserTeamList)) + userRoute.Put("/", routing.Wrap(hs.UpdateSignedInUser)) + userRoute.Post("/using/:id", routing.Wrap(hs.UserSetUsingOrg)) + userRoute.Get("/orgs", routing.Wrap(hs.GetSignedInUserOrgList)) + userRoute.Get("/teams", routing.Wrap(hs.GetSignedInUserTeamList)) - userRoute.Post("/stars/dashboard/:id", routing.Wrap(StarDashboard)) - userRoute.Delete("/stars/dashboard/:id", routing.Wrap(UnstarDashboard)) + userRoute.Post("/stars/dashboard/:id", routing.Wrap(hs.StarDashboard)) + userRoute.Delete("/stars/dashboard/:id", routing.Wrap(hs.UnstarDashboard)) - userRoute.Put("/password", routing.Wrap(ChangeUserPassword)) + userRoute.Put("/password", routing.Wrap(hs.ChangeUserPassword)) userRoute.Get("/quotas", routing.Wrap(GetUserQuotas)) - userRoute.Put("/helpflags/:id", routing.Wrap(SetHelpFlag)) + userRoute.Put("/helpflags/:id", routing.Wrap(hs.SetHelpFlag)) // For dev purpose - userRoute.Get("/helpflags/clear", routing.Wrap(ClearHelpFlags)) + userRoute.Get("/helpflags/clear", routing.Wrap(hs.ClearHelpFlags)) userRoute.Get("/preferences", routing.Wrap(hs.GetUserPreferences)) userRoute.Put("/preferences", routing.Wrap(hs.UpdateUserPreferences)) @@ -168,25 +171,25 @@ func (hs *HTTPServer) registerRoutes() { usersRoute.Get("/", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersRead, ac.ScopeGlobalUsersAll)), routing.Wrap(hs.searchUsersService.SearchUsers)) usersRoute.Get("/search", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersRead, ac.ScopeGlobalUsersAll)), routing.Wrap(hs.searchUsersService.SearchUsersWithPaging)) usersRoute.Get("/:id", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersRead, userIDScope)), routing.Wrap(hs.GetUserByID)) - usersRoute.Get("/:id/teams", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersTeamRead, userIDScope)), routing.Wrap(GetUserTeams)) - usersRoute.Get("/:id/orgs", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersRead, userIDScope)), routing.Wrap(GetUserOrgList)) + usersRoute.Get("/:id/teams", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersTeamRead, userIDScope)), routing.Wrap(hs.GetUserTeams)) + usersRoute.Get("/:id/orgs", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersRead, userIDScope)), routing.Wrap(hs.GetUserOrgList)) // query parameters /users/lookup?loginOrEmail=admin@example.com - usersRoute.Get("/lookup", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersRead, ac.ScopeGlobalUsersAll)), routing.Wrap(GetUserByLoginOrEmail)) - usersRoute.Put("/:id", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersWrite, userIDScope)), routing.Wrap(UpdateUser)) - usersRoute.Post("/:id/using/:orgId", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersWrite, userIDScope)), routing.Wrap(UpdateUserActiveOrg)) + usersRoute.Get("/lookup", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersRead, ac.ScopeGlobalUsersAll)), routing.Wrap(hs.GetUserByLoginOrEmail)) + usersRoute.Put("/:id", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersWrite, userIDScope)), routing.Wrap(hs.UpdateUser)) + usersRoute.Post("/:id/using/:orgId", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionUsersWrite, userIDScope)), routing.Wrap(hs.UpdateUserActiveOrg)) }) // team (admin permission required) apiRoute.Group("/teams", func(teamsRoute routing.RouteRegister) { - teamsRoute.Post("/", authorize(reqCanAccessTeams, ac.EvalPermission(ActionTeamsCreate)), routing.Wrap(hs.CreateTeam)) - teamsRoute.Put("/:teamId", reqCanAccessTeams, routing.Wrap(hs.UpdateTeam)) - teamsRoute.Delete("/:teamId", reqCanAccessTeams, routing.Wrap(hs.DeleteTeamByID)) - teamsRoute.Get("/:teamId/members", reqCanAccessTeams, routing.Wrap(hs.GetTeamMembers)) - teamsRoute.Post("/:teamId/members", reqCanAccessTeams, routing.Wrap(hs.AddTeamMember)) - teamsRoute.Put("/:teamId/members/:userId", reqCanAccessTeams, routing.Wrap(hs.UpdateTeamMember)) - teamsRoute.Delete("/:teamId/members/:userId", reqCanAccessTeams, routing.Wrap(hs.RemoveTeamMember)) - teamsRoute.Get("/:teamId/preferences", reqCanAccessTeams, routing.Wrap(hs.GetTeamPreferences)) - teamsRoute.Put("/:teamId/preferences", reqCanAccessTeams, routing.Wrap(hs.UpdateTeamPreferences)) + teamsRoute.Post("/", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsCreate)), routing.Wrap(hs.CreateTeam)) + teamsRoute.Put("/:teamId", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.UpdateTeam)) + teamsRoute.Delete("/:teamId", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsDelete, ac.ScopeTeamsID)), routing.Wrap(hs.DeleteTeamByID)) + teamsRoute.Get("/:teamId/members", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsPermissionsRead, ac.ScopeTeamsID)), routing.Wrap(hs.GetTeamMembers)) + teamsRoute.Post("/:teamId/members", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsPermissionsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.AddTeamMember)) + teamsRoute.Put("/:teamId/members/:userId", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsPermissionsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.UpdateTeamMember)) + teamsRoute.Delete("/:teamId/members/:userId", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsPermissionsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.RemoveTeamMember)) + teamsRoute.Get("/:teamId/preferences", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsRead, ac.ScopeTeamsID)), routing.Wrap(hs.GetTeamPreferences)) + teamsRoute.Put("/:teamId/preferences", authorize(reqCanAccessTeams, ac.EvalPermission(ac.ActionTeamsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.UpdateTeamPreferences)) }) // team without requirement of user to be org admin @@ -204,18 +207,18 @@ func (hs *HTTPServer) registerRoutes() { // current org apiRoute.Group("/org", func(orgRoute routing.RouteRegister) { userIDScope := ac.Scope("users", "id", ac.Parameter(":userId")) - orgRoute.Put("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(UpdateCurrentOrg)) - orgRoute.Put("/address", authorize(reqOrgAdmin, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(UpdateCurrentOrgAddress)) - orgRoute.Get("/users", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead, ac.ScopeUsersAll)), routing.Wrap(hs.GetOrgUsersForCurrentOrg)) - orgRoute.Get("/users/search", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead, ac.ScopeUsersAll)), routing.Wrap(hs.SearchOrgUsersWithPaging)) + orgRoute.Put("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(hs.UpdateCurrentOrg)) + orgRoute.Put("/address", authorize(reqOrgAdmin, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(hs.UpdateCurrentOrgAddress)) + orgRoute.Get("/users", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead)), routing.Wrap(hs.GetOrgUsersForCurrentOrg)) + orgRoute.Get("/users/search", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead)), routing.Wrap(hs.SearchOrgUsersWithPaging)) orgRoute.Post("/users", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersAdd, ac.ScopeUsersAll)), quota("user"), routing.Wrap(hs.AddOrgUserToCurrentOrg)) orgRoute.Patch("/users/:userId", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRoleUpdate, userIDScope)), routing.Wrap(hs.UpdateOrgUserForCurrentOrg)) orgRoute.Delete("/users/:userId", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRemove, userIDScope)), routing.Wrap(hs.RemoveOrgUserForCurrentOrg)) // invites - orgRoute.Get("/invites", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionUsersCreate)), routing.Wrap(GetPendingOrgInvites)) - orgRoute.Post("/invites", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionUsersCreate)), quota("user"), routing.Wrap(AddOrgInvite)) - orgRoute.Patch("/invites/:code/revoke", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionUsersCreate)), routing.Wrap(RevokeInvite)) + orgRoute.Get("/invites", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionUsersCreate)), routing.Wrap(hs.GetPendingOrgInvites)) + orgRoute.Post("/invites", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionUsersCreate)), quota("user"), routing.Wrap(hs.AddOrgInvite)) + orgRoute.Patch("/invites/:code/revoke", authorize(reqOrgAdmin, ac.EvalPermission(ac.ActionUsersCreate)), routing.Wrap(hs.RevokeInvite)) // prefs orgRoute.Get("/preferences", authorize(reqOrgAdmin, ac.EvalPermission(ActionOrgsPreferencesRead)), routing.Wrap(hs.GetOrgPreferences)) @@ -224,7 +227,7 @@ func (hs *HTTPServer) registerRoutes() { // current org without requirement of user to be org admin apiRoute.Group("/org", func(orgRoute routing.RouteRegister) { - orgRoute.Get("/users/lookup", authorize(reqOrgAdminFolderAdminOrTeamAdmin, ac.EvalPermission(ac.ActionOrgUsersRead, ac.ScopeUsersAll)), routing.Wrap(hs.GetOrgUsersForCurrentOrgLookup)) + orgRoute.Get("/users/lookup", authorize(reqOrgAdminFolderAdminOrTeamAdmin, ac.EvalPermission(ac.ActionOrgUsersRead)), routing.Wrap(hs.GetOrgUsersForCurrentOrgLookup)) }) // create new org @@ -237,9 +240,9 @@ func (hs *HTTPServer) registerRoutes() { apiRoute.Group("/orgs/:orgId", func(orgsRoute routing.RouteRegister) { userIDScope := ac.Scope("users", "id", ac.Parameter(":userId")) orgsRoute.Get("/", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ActionOrgsRead)), routing.Wrap(GetOrgByID)) - orgsRoute.Put("/", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(UpdateOrg)) - orgsRoute.Put("/address", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(UpdateOrgAddress)) - orgsRoute.Delete("/", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ActionOrgsDelete)), routing.Wrap(DeleteOrgByID)) + orgsRoute.Put("/", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(hs.UpdateOrg)) + orgsRoute.Put("/address", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ActionOrgsWrite)), routing.Wrap(hs.UpdateOrgAddress)) + orgsRoute.Delete("/", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ActionOrgsDelete)), routing.Wrap(hs.DeleteOrgByID)) orgsRoute.Get("/users", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ac.ActionOrgUsersRead, ac.ScopeUsersAll)), routing.Wrap(hs.GetOrgUsers)) orgsRoute.Post("/users", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ac.ActionOrgUsersAdd, ac.ScopeUsersAll)), routing.Wrap(hs.AddOrgUser)) orgsRoute.Patch("/users/:userId", authorizeInOrg(reqGrafanaAdmin, acmiddleware.UseOrgFromContextParams, ac.EvalPermission(ac.ActionOrgUsersRoleUpdate, userIDScope)), routing.Wrap(hs.UpdateOrgUser)) @@ -266,15 +269,15 @@ func (hs *HTTPServer) registerRoutes() { // Data sources apiRoute.Group("/datasources", func(datasourceRoute routing.RouteRegister) { - datasourceRoute.Get("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead, ScopeDatasourcesAll)), routing.Wrap(hs.GetDataSources)) + datasourceRoute.Get("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead)), routing.Wrap(hs.GetDataSources)) datasourceRoute.Post("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesCreate)), quota("data_source"), routing.Wrap(AddDataSource)) datasourceRoute.Put("/:id", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesWrite, ScopeDatasourceID)), routing.Wrap(hs.UpdateDataSource)) datasourceRoute.Delete("/:id", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesDelete, ScopeDatasourceID)), routing.Wrap(hs.DeleteDataSourceById)) datasourceRoute.Delete("/uid/:uid", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesDelete, ScopeDatasourceUID)), routing.Wrap(hs.DeleteDataSourceByUID)) datasourceRoute.Delete("/name/:name", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesDelete, ScopeDatasourceName)), routing.Wrap(hs.DeleteDataSourceByName)) - datasourceRoute.Get("/:id", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead, ScopeDatasourceID)), routing.Wrap(hs.GetDataSourceById)) - datasourceRoute.Get("/uid/:uid", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead, ScopeDatasourceUID)), routing.Wrap(hs.GetDataSourceByUID)) - datasourceRoute.Get("/name/:name", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead, ScopeDatasourceName)), routing.Wrap(GetDataSourceByName)) + datasourceRoute.Get("/:id", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead)), routing.Wrap(hs.GetDataSourceById)) + datasourceRoute.Get("/uid/:uid", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead)), routing.Wrap(hs.GetDataSourceByUID)) + datasourceRoute.Get("/name/:name", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead)), routing.Wrap(GetDataSourceByName)) }) apiRoute.Get("/datasources/id/:name", authorize(reqSignedIn, ac.EvalPermission(ActionDatasourcesIDRead, ScopeDatasourceName)), routing.Wrap(GetDataSourceIdByName)) @@ -339,7 +342,6 @@ func (hs *HTTPServer) registerRoutes() { dashboardRoute.Post("/db", routing.Wrap(hs.PostDashboard)) dashboardRoute.Get("/home", routing.Wrap(hs.GetHomeDashboard)) dashboardRoute.Get("/tags", GetDashboardTags) - dashboardRoute.Post("/import", routing.Wrap(hs.ImportDashboard)) dashboardRoute.Group("/id/:dashboardId", func(dashIdRoute routing.RouteRegister) { dashIdRoute.Get("/versions", routing.Wrap(GetDashboardVersions)) @@ -360,13 +362,13 @@ func (hs *HTTPServer) registerRoutes() { // Playlist apiRoute.Group("/playlists", func(playlistRoute routing.RouteRegister) { - playlistRoute.Get("/", routing.Wrap(SearchPlaylists)) - playlistRoute.Get("/:id", ValidateOrgPlaylist, routing.Wrap(GetPlaylist)) - playlistRoute.Get("/:id/items", ValidateOrgPlaylist, routing.Wrap(GetPlaylistItems)) - playlistRoute.Get("/:id/dashboards", ValidateOrgPlaylist, routing.Wrap(GetPlaylistDashboards)) - playlistRoute.Delete("/:id", reqEditorRole, ValidateOrgPlaylist, routing.Wrap(DeletePlaylist)) - playlistRoute.Put("/:id", reqEditorRole, ValidateOrgPlaylist, routing.Wrap(UpdatePlaylist)) - playlistRoute.Post("/", reqEditorRole, routing.Wrap(CreatePlaylist)) + playlistRoute.Get("/", routing.Wrap(hs.SearchPlaylists)) + playlistRoute.Get("/:id", hs.ValidateOrgPlaylist, routing.Wrap(hs.GetPlaylist)) + playlistRoute.Get("/:id/items", hs.ValidateOrgPlaylist, routing.Wrap(hs.GetPlaylistItems)) + playlistRoute.Get("/:id/dashboards", hs.ValidateOrgPlaylist, routing.Wrap(hs.GetPlaylistDashboards)) + playlistRoute.Delete("/:id", reqEditorRole, hs.ValidateOrgPlaylist, routing.Wrap(hs.DeletePlaylist)) + playlistRoute.Put("/:id", reqEditorRole, hs.ValidateOrgPlaylist, routing.Wrap(hs.UpdatePlaylist)) + playlistRoute.Post("/", reqEditorRole, routing.Wrap(hs.CreatePlaylist)) }) // Search @@ -435,7 +437,7 @@ func (hs *HTTPServer) registerRoutes() { // Some channels may have info liveRoute.Get("/info/*", routing.Wrap(hs.Live.HandleInfoHTTP)) - if hs.Cfg.FeatureToggles["live-pipeline"] { + if hs.Features.IsEnabled(featuremgmt.FlagLivePipeline) { // POST Live data to be processed according to channel rules. liveRoute.Post("/pipeline/push/*", hs.LivePushGateway.HandlePipelinePush) liveRoute.Post("/pipeline-convert-test", routing.Wrap(hs.Live.HandlePipelineConvertTestHTTP), reqOrgAdmin) @@ -458,6 +460,9 @@ func (hs *HTTPServer) registerRoutes() { // admin api r.Group("/api/admin", func(adminRoute routing.RouteRegister) { adminRoute.Get("/settings", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionSettingsRead)), routing.Wrap(hs.AdminGetSettings)) + if hs.Features.IsEnabled(featuremgmt.FlagShowFeatureFlagsInUI) { + adminRoute.Get("/settings/features", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionSettingsRead)), hs.Features.HandleGetSettings) + } adminRoute.Get("/stats", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionServerStatsRead)), routing.Wrap(AdminGetStats)) adminRoute.Post("/pause-all-alerts", reqGrafanaAdmin, routing.Wrap(PauseAllAlerts)) diff --git a/pkg/api/apierrors/dashboard.go b/pkg/api/apierrors/dashboard.go new file mode 100644 index 00000000000..a4003646143 --- /dev/null +++ b/pkg/api/apierrors/dashboard.go @@ -0,0 +1,49 @@ +package apierrors + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/alerting" + "github.com/grafana/grafana/pkg/util" +) + +// ToDashboardErrorResponse returns a different response status according to the dashboard error type +func ToDashboardErrorResponse(ctx context.Context, pluginStore plugins.Store, err error) response.Response { + var dashboardErr models.DashboardErr + if ok := errors.As(err, &dashboardErr); ok { + if body := dashboardErr.Body(); body != nil { + return response.JSON(dashboardErr.StatusCode, body) + } + if dashboardErr.StatusCode != http.StatusBadRequest { + return response.Error(dashboardErr.StatusCode, dashboardErr.Error(), err) + } + return response.Error(dashboardErr.StatusCode, dashboardErr.Error(), nil) + } + + if errors.Is(err, models.ErrFolderNotFound) { + return response.Error(http.StatusBadRequest, err.Error(), nil) + } + + var validationErr alerting.ValidationError + if ok := errors.As(err, &validationErr); ok { + return response.Error(http.StatusUnprocessableEntity, validationErr.Error(), err) + } + + var pluginErr models.UpdatePluginDashboardError + if ok := errors.As(err, &pluginErr); ok { + message := fmt.Sprintf("The dashboard belongs to plugin %s.", pluginErr.PluginId) + // look up plugin name + if plugin, exists := pluginStore.Plugin(ctx, pluginErr.PluginId); exists { + message = fmt.Sprintf("The dashboard belongs to plugin %s.", plugin.Name) + } + return response.JSON(http.StatusPreconditionFailed, util.DynMap{"status": "plugin-dashboard", "message": message}) + } + + return response.Error(http.StatusInternalServerError, "Failed to save dashboard", err) +} diff --git a/pkg/api/apikey.go b/pkg/api/apikey.go index 1d470955c96..beece581a68 100644 --- a/pkg/api/apikey.go +++ b/pkg/api/apikey.go @@ -3,6 +3,7 @@ package api import ( "errors" "net/http" + "strconv" "time" "github.com/grafana/grafana/pkg/api/dtos" @@ -10,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/apikeygen" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/web" ) @@ -41,11 +43,14 @@ func GetAPIKeys(c *models.ReqContext) response.Response { // DeleteAPIKey deletes an API key func DeleteAPIKey(c *models.ReqContext) response.Response { - id := c.ParamsInt64(":id") + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } cmd := &models.DeleteApiKeyCommand{Id: id, OrgId: c.OrgId} - err := bus.Dispatch(c.Req.Context(), cmd) + err = bus.Dispatch(c.Req.Context(), cmd) if err != nil { var status int if errors.Is(err, models.ErrApiKeyNotFound) { @@ -79,7 +84,7 @@ func (hs *HTTPServer) AddAPIKey(c *models.ReqContext) response.Response { } cmd.OrgId = c.OrgId var err error - if hs.Cfg.FeatureToggles["service-accounts"] { + if hs.Features.IsEnabled(featuremgmt.FlagServiceAccounts) { // Api keys should now be created with addadditionalapikey endpoint return response.Error(400, "API keys should now be added via the AdditionalAPIKey endpoint.", err) } @@ -116,7 +121,7 @@ func (hs *HTTPServer) AdditionalAPIKey(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - if !hs.Cfg.FeatureToggles["service-accounts"] { + if !hs.Features.IsEnabled(featuremgmt.FlagServiceAccounts) { return response.Error(500, "Requires services-accounts feature", errors.New("feature missing")) } diff --git a/pkg/api/common_test.go b/pkg/api/common_test.go index eb6f4d6b16b..ed2e956e096 100644 --- a/pkg/api/common_test.go +++ b/pkg/api/common_test.go @@ -11,21 +11,25 @@ import ( "path/filepath" "testing" - "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/remotecache" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/database" + acmiddleware "github.com/grafana/grafana/pkg/services/accesscontrol/middleware" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/accesscontrol/ossaccesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourceservices" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/contexthandler" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/ldap" "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/services/searchusers" @@ -33,10 +37,11 @@ import ( "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" + "github.com/stretchr/testify/require" ) -func loggedInUserScenario(t *testing.T, desc string, url string, fn scenarioFunc) { - loggedInUserScenarioWithRole(t, desc, "GET", url, url, models.ROLE_EDITOR, fn) +func loggedInUserScenario(t *testing.T, desc string, url string, routePattern string, fn scenarioFunc) { + loggedInUserScenarioWithRole(t, desc, "GET", url, routePattern, models.ROLE_EDITOR, fn) } func loggedInUserScenarioWithRole(t *testing.T, desc string, method string, url string, routePattern string, role models.RoleType, fn scenarioFunc) { @@ -63,7 +68,6 @@ func loggedInUserScenarioWithRole(t *testing.T, desc string, method string, url case "DELETE": sc.m.Delete(routePattern, sc.defaultHandler) } - fn(sc) }) } @@ -176,7 +180,9 @@ func getContextHandler(t *testing.T, cfg *setting.Cfg) *contexthandler.ContextHa userAuthTokenSvc := auth.NewFakeUserAuthTokenService() renderSvc := &fakeRenderService{} authJWTSvc := models.NewFakeJWTService() - ctxHdlr := contexthandler.ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + ctxHdlr := contexthandler.ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore, tracer) return ctxHdlr } @@ -210,8 +216,8 @@ func (s *fakeRenderService) Init() error { } func setupAccessControlScenarioContext(t *testing.T, cfg *setting.Cfg, url string, permissions []*accesscontrol.Permission) (*scenarioContext, *HTTPServer) { - cfg.FeatureToggles = make(map[string]bool) - cfg.FeatureToggles["accesscontrol"] = true + features := featuremgmt.WithFeatures(featuremgmt.FlagAccesscontrol) + cfg.IsFeatureToggleEnabled = features.IsEnabled cfg.Quota.Enabled = false bus := bus.GetBus() @@ -219,10 +225,12 @@ func setupAccessControlScenarioContext(t *testing.T, cfg *setting.Cfg, url strin Cfg: cfg, Bus: bus, Live: newTestLive(t), + Features: features, QuotaService: "a.QuotaService{Cfg: cfg}, RouteRegister: routing.NewRouteRegister(), AccessControl: accesscontrolmock.New().WithPermissions(permissions), searchUsersService: searchusers.ProvideUsersService(bus, filters.ProvideOSSSearchUserFilter()), + ldapGroups: ldap.ProvideGroupsService(), } sc := setupScenarioContext(t, url) @@ -293,13 +301,25 @@ func setInitCtxSignedInOrgAdmin(initCtx *models.ReqContext) { initCtx.SignedInUser = &models.SignedInUser{UserId: testUserID, OrgId: 1, OrgRole: models.ROLE_ADMIN, Login: testUserLogin} } +func setupSimpleHTTPServer(features *featuremgmt.FeatureManager) *HTTPServer { + if features == nil { + features = featuremgmt.WithFeatures() + } + cfg := setting.NewCfg() + cfg.IsFeatureToggleEnabled = features.IsEnabled + + return &HTTPServer{ + Cfg: cfg, + Features: features, + Bus: bus.GetBus(), + } +} + func setupHTTPServer(t *testing.T, useFakeAccessControl bool, enableAccessControl bool) accessControlScenarioContext { // Use a new conf + features := featuremgmt.WithFeatures("accesscontrol", enableAccessControl) cfg := setting.NewCfg() - cfg.FeatureToggles = make(map[string]bool) - if enableAccessControl { - cfg.FeatureToggles["accesscontrol"] = enableAccessControl - } + cfg.IsFeatureToggleEnabled = features.IsEnabled return setupHTTPServerWithCfg(t, useFakeAccessControl, enableAccessControl, cfg) } @@ -307,8 +327,10 @@ func setupHTTPServer(t *testing.T, useFakeAccessControl bool, enableAccessContro func setupHTTPServerWithCfg(t *testing.T, useFakeAccessControl, enableAccessControl bool, cfg *setting.Cfg) accessControlScenarioContext { t.Helper() + features := featuremgmt.WithFeatures("accesscontrol", enableAccessControl) + cfg.IsFeatureToggleEnabled = features.IsEnabled + var acmock *accesscontrolmock.Mock - var ac *ossaccesscontrol.OSSAccessControlService // Use a test DB db := sqlstore.InitTestDB(t) @@ -316,13 +338,15 @@ func setupHTTPServerWithCfg(t *testing.T, useFakeAccessControl, enableAccessCont bus := bus.GetBus() + routeRegister := routing.NewRouteRegister() // Create minimal HTTP Server hs := &HTTPServer{ Cfg: cfg, + Features: features, Bus: bus, Live: newTestLive(t), QuotaService: "a.QuotaService{Cfg: cfg}, - RouteRegister: routing.NewRouteRegister(), + RouteRegister: routeRegister, SQLStore: db, searchUsersService: searchusers.ProvideUsersService(bus, filters.ProvideOSSSearchUserFilter()), } @@ -334,14 +358,20 @@ func setupHTTPServerWithCfg(t *testing.T, useFakeAccessControl, enableAccessCont acmock = acmock.WithDisabled() } hs.AccessControl = acmock + teamPermissionService, err := resourceservices.ProvideTeamPermissions(routeRegister, db, acmock, database.ProvideService(db)) + require.NoError(t, err) + hs.TeamPermissionsService = teamPermissionService } else { - ac = ossaccesscontrol.ProvideService(cfg, &usagestats.UsageStatsMock{T: t}) + ac := ossaccesscontrol.ProvideService(hs.Features, &usagestats.UsageStatsMock{T: t}, database.ProvideService(db)) hs.AccessControl = ac // Perform role registration err := hs.declareFixedRoles() require.NoError(t, err) err = ac.RegisterFixedRoles() require.NoError(t, err) + teamPermissionService, err := resourceservices.ProvideTeamPermissions(routeRegister, db, ac, database.ProvideService(db)) + require.NoError(t, err) + hs.TeamPermissionsService = teamPermissionService } // Instantiate a new Server @@ -355,6 +385,8 @@ func setupHTTPServerWithCfg(t *testing.T, useFakeAccessControl, enableAccessCont c.Map(initCtx) }) + m.Use(acmiddleware.LoadPermissionsMiddleware(hs.AccessControl)) + // Register all routes hs.registerRoutes() hs.RouteRegister.Register(m.Router) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 915a345e237..32763f62a6c 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" + "github.com/grafana/grafana/pkg/api/apierrors" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/bus" @@ -381,7 +382,7 @@ func (hs *HTTPServer) postDashboard(c *models.ReqContext, cmd models.SaveDashboa } if err != nil { - return hs.dashboardSaveErrorToApiResponse(ctx, err) + return apierrors.ToDashboardErrorResponse(ctx, hs.pluginStore, err) } if hs.Cfg.EditorsCanAdmin && newDashboard { @@ -409,40 +410,6 @@ func (hs *HTTPServer) postDashboard(c *models.ReqContext, cmd models.SaveDashboa }) } -func (hs *HTTPServer) dashboardSaveErrorToApiResponse(ctx context.Context, err error) response.Response { - var dashboardErr models.DashboardErr - if ok := errors.As(err, &dashboardErr); ok { - if body := dashboardErr.Body(); body != nil { - return response.JSON(dashboardErr.StatusCode, body) - } - if dashboardErr.StatusCode != 400 { - return response.Error(dashboardErr.StatusCode, dashboardErr.Error(), err) - } - return response.Error(dashboardErr.StatusCode, dashboardErr.Error(), nil) - } - - if errors.Is(err, models.ErrFolderNotFound) { - return response.Error(400, err.Error(), nil) - } - - var validationErr alerting.ValidationError - if ok := errors.As(err, &validationErr); ok { - return response.Error(422, validationErr.Error(), err) - } - - var pluginErr models.UpdatePluginDashboardError - if ok := errors.As(err, &pluginErr); ok { - message := fmt.Sprintf("The dashboard belongs to plugin %s.", pluginErr.PluginId) - // look up plugin name - if plugin, exists := hs.pluginStore.Plugin(ctx, pluginErr.PluginId); exists { - message = fmt.Sprintf("The dashboard belongs to plugin %s.", plugin.Name) - } - return response.JSON(412, util.DynMap{"status": "plugin-dashboard", "message": message}) - } - - return response.Error(500, "Failed to save dashboard", err) -} - // GetHomeDashboard returns the home dashboard. func (hs *HTTPServer) GetHomeDashboard(c *models.ReqContext) response.Response { prefsQuery := models.GetPreferencesWithDefaultsQuery{User: c.SignedInUser} @@ -529,7 +496,10 @@ func (hs *HTTPServer) addGettingStartedPanelToHomeDashboard(c *models.ReqContext // GetDashboardVersions returns all dashboard versions as JSON func GetDashboardVersions(c *models.ReqContext) response.Response { - dashID := c.ParamsInt64(":dashboardId") + dashID, err := strconv.ParseInt(web.Params(c.Req)[":dashboardId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "dashboardId is invalid", err) + } guardian := guardian.New(c.Req.Context(), dashID, c.OrgId, c.SignedInUser) if canSave, err := guardian.CanSave(); err != nil || !canSave { @@ -568,7 +538,10 @@ func GetDashboardVersions(c *models.ReqContext) response.Response { // GetDashboardVersion returns the dashboard version with the given ID. func GetDashboardVersion(c *models.ReqContext) response.Response { - dashID := c.ParamsInt64(":dashboardId") + dashID, err := strconv.ParseInt(web.Params(c.Req)[":dashboardId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "dashboardId is invalid", err) + } guardian := guardian.New(c.Req.Context(), dashID, c.OrgId, c.SignedInUser) if canSave, err := guardian.CanSave(); err != nil || !canSave { @@ -660,7 +633,12 @@ func (hs *HTTPServer) RestoreDashboardVersion(c *models.ReqContext) response.Res if err := web.Bind(c.Req, &apiCmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - dash, rsp := getDashboardHelper(c.Req.Context(), c.OrgId, c.ParamsInt64(":dashboardId"), "") + dashboardId, err := strconv.ParseInt(web.Params(c.Req)[":dashboardId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "dashboardId is invalid", err) + } + + dash, rsp := getDashboardHelper(c.Req.Context(), c.OrgId, dashboardId, "") if rsp != nil { return rsp } diff --git a/pkg/api/dashboard_permission.go b/pkg/api/dashboard_permission.go index d51c8cf1c7e..9ef7f688aba 100644 --- a/pkg/api/dashboard_permission.go +++ b/pkg/api/dashboard_permission.go @@ -3,6 +3,7 @@ package api import ( "errors" "net/http" + "strconv" "time" "github.com/grafana/grafana/pkg/api/dtos" @@ -13,7 +14,10 @@ import ( ) func (hs *HTTPServer) GetDashboardPermissionList(c *models.ReqContext) response.Response { - dashID := c.ParamsInt64(":dashboardId") + dashID, err := strconv.ParseInt(web.Params(c.Req)[":dashboardId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "dashboardId is invalid", err) + } _, rsp := getDashboardHelper(c.Req.Context(), c.OrgId, dashID, "") if rsp != nil { @@ -61,7 +65,10 @@ func (hs *HTTPServer) UpdateDashboardPermissions(c *models.ReqContext) response. return response.Error(400, err.Error(), err) } - dashID := c.ParamsInt64(":dashboardId") + dashID, err := strconv.ParseInt(web.Params(c.Req)[":dashboardId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "dashboardId is invalid", err) + } _, rsp := getDashboardHelper(c.Req.Context(), c.OrgId, dashID, "") if rsp != nil { diff --git a/pkg/api/dashboard_permission_test.go b/pkg/api/dashboard_permission_test.go index 8a8fb92591d..a79733baba1 100644 --- a/pkg/api/dashboard_permission_test.go +++ b/pkg/api/dashboard_permission_test.go @@ -32,7 +32,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { } loggedInUserScenarioWithRole(t, "When calling GET on", "GET", "/api/dashboards/id/1/permissions", - "/api/dashboards/id/:id/permissions", models.ROLE_EDITOR, func(sc *scenarioContext) { + "/api/dashboards/id/:dashboardId/permissions", models.ROLE_EDITOR, func(sc *scenarioContext) { setUp() callGetDashboardPermissions(sc, hs) assert.Equal(t, 404, sc.resp.Code) @@ -47,7 +47,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { setUp() @@ -75,7 +75,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { } loggedInUserScenarioWithRole(t, "When calling GET on", "GET", "/api/dashboards/id/1/permissions", - "/api/dashboards/id/:id/permissions", models.ROLE_EDITOR, func(sc *scenarioContext) { + "/api/dashboards/id/:dashboardId/permissions", models.ROLE_EDITOR, func(sc *scenarioContext) { setUp() callGetDashboardPermissions(sc, hs) assert.Equal(t, 403, sc.resp.Code) @@ -90,7 +90,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { setUp() @@ -127,7 +127,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { } loggedInUserScenarioWithRole(t, "When calling GET on", "GET", "/api/dashboards/id/1/permissions", - "/api/dashboards/id/:id/permissions", models.ROLE_ADMIN, func(sc *scenarioContext) { + "/api/dashboards/id/:dashboardId/permissions", models.ROLE_ADMIN, func(sc *scenarioContext) { setUp() callGetDashboardPermissions(sc, hs) assert.Equal(t, 200, sc.resp.Code) @@ -150,7 +150,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { setUp() @@ -188,7 +188,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { setUp() @@ -229,7 +229,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { setUp() @@ -258,7 +258,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { callUpdateDashboardPermissions(t, sc) @@ -300,7 +300,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { setUp() @@ -344,7 +344,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { var resp []*models.DashboardAclInfoDTO loggedInUserScenarioWithRole(t, "When calling GET on", "GET", "/api/dashboards/id/1/permissions", - "/api/dashboards/id/:id/permissions", models.ROLE_ADMIN, func(sc *scenarioContext) { + "/api/dashboards/id/:dashboardId/permissions", models.ROLE_ADMIN, func(sc *scenarioContext) { setUp() callGetDashboardPermissions(sc, hs) assert.Equal(t, 200, sc.resp.Code) @@ -375,7 +375,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { updateDashboardPermissionScenario(t, updatePermissionContext{ desc: "When calling POST on", url: "/api/dashboards/id/1/permissions", - routePattern: "/api/dashboards/id/:id/permissions", + routePattern: "/api/dashboards/id/:dashboardId/permissions", cmd: cmd, fn: func(sc *scenarioContext) { setUp() diff --git a/pkg/api/dashboard_test.go b/pkg/api/dashboard_test.go index 8d09914daf6..c44ef24a034 100644 --- a/pkg/api/dashboard_test.go +++ b/pkg/api/dashboard_test.go @@ -20,6 +20,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/libraryelements" "github.com/grafana/grafana/pkg/services/live" "github.com/grafana/grafana/pkg/services/provisioning" @@ -88,8 +89,17 @@ type testState struct { } func newTestLive(t *testing.T) *live.GrafanaLive { + features := featuremgmt.WithFeatures() cfg := &setting.Cfg{AppURL: "http://localhost:3000/"} - gLive, err := live.ProvideService(nil, cfg, routing.NewRouteRegister(), nil, nil, nil, nil, sqlstore.InitTestDB(t), nil, &usagestats.UsageStatsMock{T: t}, nil) + cfg.IsFeatureToggleEnabled = features.IsEnabled + gLive, err := live.ProvideService(nil, cfg, + routing.NewRouteRegister(), + nil, nil, nil, + sqlstore.InitTestDB(t), + nil, + &usagestats.UsageStatsMock{T: t}, + nil, + features) require.NoError(t, err) return gLive } diff --git a/pkg/api/datasource/validation.go b/pkg/api/datasource/validation.go index a8420814d7e..c9dd491dab6 100644 --- a/pkg/api/datasource/validation.go +++ b/pkg/api/datasource/validation.go @@ -1,17 +1,37 @@ package datasource import ( + "errors" "fmt" "net/url" "regexp" "strings" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb/mssql" ) var logger = log.New("datasource") +// requiredURL contains the set of data sources that require a URL. +var requiredURL = map[string]bool{ + models.DS_GRAPHITE: true, + models.DS_INFLUXDB: true, + models.DS_INFLUXDB_08: true, + models.DS_ES: true, + models.DS_PROMETHEUS: true, + models.DS_ALERTMANAGER: true, + models.DS_JAEGER: true, + models.DS_LOKI: true, + models.DS_OPENTSDB: true, + models.DS_TEMPO: true, + models.DS_ZIPKIN: true, + models.DS_MYSQL: true, + models.DS_POSTGRES: true, + models.DS_MSSQL: true, +} + // URLValidationError represents an error from validating a data source URL. type URLValidationError struct { Err error @@ -40,6 +60,11 @@ var reURL = regexp.MustCompile("^[^:]*://") // The data source's type and URL must be provided. If successful, the valid URL object is returned, otherwise an // error is returned. func ValidateURL(typeName, urlStr string) (*url.URL, error) { + // Check for empty URLs + if _, exists := requiredURL[typeName]; exists && strings.TrimSpace(urlStr) == "" { + return nil, URLValidationError{Err: errors.New("empty URL string"), URL: ""} + } + var u *url.URL var err error switch strings.ToLower(typeName) { diff --git a/pkg/api/datasources.go b/pkg/api/datasources.go index d0334bd3413..b24612f763f 100644 --- a/pkg/api/datasources.go +++ b/pkg/api/datasources.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "sort" + "strconv" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana/pkg/api/datasource" @@ -30,8 +31,13 @@ func (hs *HTTPServer) GetDataSources(c *models.ReqContext) response.Response { return response.Error(500, "Failed to query datasources", err) } + filtered, err := filterDatasourcesByQueryPermission(c.Req.Context(), c.SignedInUser, query.Result) + if err != nil { + return response.Error(500, "Failed to query datasources", err) + } + result := make(dtos.DataSourceList, 0) - for _, ds := range query.Result { + for _, ds := range filtered { dsItem := dtos.DataSourceListItemDTO{ OrgId: ds.OrgId, Id: ds.Id, @@ -81,9 +87,14 @@ func (hs *HTTPServer) getDataSourceAccessControlMetadata(c *models.ReqContext, d return accesscontrol.GetResourcesMetadata(c.Req.Context(), userPermissions, "datasources", dsIDs)[key], nil } +// GET /api/datasources/:id func (hs *HTTPServer) GetDataSourceById(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } query := models.GetDataSourceQuery{ - Id: c.ParamsInt64(":id"), + Id: id, OrgId: c.OrgId, } @@ -97,21 +108,28 @@ func (hs *HTTPServer) GetDataSourceById(c *models.ReqContext) response.Response return response.Error(500, "Failed to query datasources", err) } - ds := query.Result - dtos := convertModelToDtos(ds) + filtered, err := filterDatasourcesByQueryPermission(c.Req.Context(), c.SignedInUser, []*models.DataSource{query.Result}) + if err != nil || len(filtered) != 1 { + return response.Error(404, "Data source not found", err) + } + dto := convertModelToDtos(filtered[0]) // Add accesscontrol metadata - metadata, err := hs.getDataSourceAccessControlMetadata(c, ds.Id) + metadata, err := hs.getDataSourceAccessControlMetadata(c, dto.Id) if err != nil { return response.Error(http.StatusInternalServerError, "Failed to query metadata", err) } - dtos.AccessControl = metadata + dto.AccessControl = metadata - return response.JSON(200, &dtos) + return response.JSON(200, &dto) } +// DELETE /api/datasources/:id func (hs *HTTPServer) DeleteDataSourceById(c *models.ReqContext) response.Response { - id := c.ParamsInt64(":id") + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } if id <= 0 { return response.Error(400, "Missing valid datasource id", nil) @@ -152,16 +170,21 @@ func (hs *HTTPServer) GetDataSourceByUID(c *models.ReqContext) response.Response return response.Error(http.StatusInternalServerError, "Failed to query datasource", err) } - dtos := convertModelToDtos(ds) + filtered, err := filterDatasourcesByQueryPermission(c.Req.Context(), c.SignedInUser, []*models.DataSource{ds}) + if err != nil || len(filtered) != 1 { + return response.Error(404, "Data source not found", err) + } + + dto := convertModelToDtos(filtered[0]) // Add accesscontrol metadata - metadata, err := hs.getDataSourceAccessControlMetadata(c, ds.Id) + metadata, err := hs.getDataSourceAccessControlMetadata(c, dto.Id) if err != nil { return response.Error(http.StatusInternalServerError, "Failed to query metadata", err) } - dtos.AccessControl = metadata + dto.AccessControl = metadata - return response.JSON(200, &dtos) + return response.JSON(200, &dto) } // DELETE /api/datasources/uid/:uid @@ -199,6 +222,7 @@ func (hs *HTTPServer) DeleteDataSourceByUID(c *models.ReqContext) response.Respo }) } +// DELETE /api/datasources/name/:name func (hs *HTTPServer) DeleteDataSourceByName(c *models.ReqContext) response.Response { name := web.Params(c.Req)[":name"] @@ -232,27 +256,27 @@ func (hs *HTTPServer) DeleteDataSourceByName(c *models.ReqContext) response.Resp }) } -func validateURL(tp string, u string) response.Response { - if u != "" { - if _, err := datasource.ValidateURL(tp, u); err != nil { - datasourcesLogger.Error("Received invalid data source URL as part of data source command", - "url", u) - return response.Error(400, fmt.Sprintf("Validation error, invalid URL: %q", u), err) - } +func validateURL(cmdType string, url string) response.Response { + if _, err := datasource.ValidateURL(cmdType, url); err != nil { + return response.Error(400, fmt.Sprintf("Validation error, invalid URL: %q", url), err) } return nil } +// POST /api/datasources/ func AddDataSource(c *models.ReqContext) response.Response { cmd := models.AddDataSourceCommand{} if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } + datasourcesLogger.Debug("Received command to add data source", "url", cmd.Url) cmd.OrgId = c.OrgId - if resp := validateURL(cmd.Type, cmd.Url); resp != nil { - return resp + if cmd.Url != "" { + if resp := validateURL(cmd.Type, cmd.Url); resp != nil { + return resp + } } if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { @@ -272,6 +296,7 @@ func AddDataSource(c *models.ReqContext) response.Response { }) } +// PUT /api/datasources/:id func (hs *HTTPServer) UpdateDataSource(c *models.ReqContext) response.Response { cmd := models.UpdateDataSourceCommand{} if err := web.Bind(c.Req, &cmd); err != nil { @@ -279,12 +304,27 @@ func (hs *HTTPServer) UpdateDataSource(c *models.ReqContext) response.Response { } datasourcesLogger.Debug("Received command to update data source", "url", cmd.Url) cmd.OrgId = c.OrgId - cmd.Id = c.ParamsInt64(":id") + var err error + if cmd.Id, err = strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64); err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } if resp := validateURL(cmd.Type, cmd.Url); resp != nil { return resp } - err := hs.fillWithSecureJSONData(c.Req.Context(), &cmd) + ds, err := getRawDataSourceById(c.Req.Context(), cmd.Id, cmd.OrgId) + if err != nil { + if errors.Is(err, models.ErrDataSourceNotFound) { + return response.Error(404, "Data source not found", nil) + } + return response.Error(500, "Failed to update datasource", err) + } + + if ds.ReadOnly { + return response.Error(403, "Cannot update read-only data source", nil) + } + + err = hs.fillWithSecureJSONData(c.Req.Context(), &cmd) if err != nil { return response.Error(500, "Failed to update datasource", err) } @@ -385,8 +425,13 @@ func GetDataSourceByName(c *models.ReqContext) response.Response { return response.Error(500, "Failed to query datasources", err) } - dtos := convertModelToDtos(query.Result) - return response.JSON(200, &dtos) + filtered, err := filterDatasourcesByQueryPermission(c.Req.Context(), c.SignedInUser, []*models.DataSource{query.Result}) + if err != nil || len(filtered) != 1 { + return response.Error(404, "Data source not found", err) + } + + dto := convertModelToDtos(filtered[0]) + return response.JSON(200, &dto) } // Get /api/datasources/id/:name @@ -410,7 +455,11 @@ func GetDataSourceIdByName(c *models.ReqContext) response.Response { // /api/datasources/:id/resources/* func (hs *HTTPServer) CallDatasourceResource(c *models.ReqContext) { - datasourceID := c.ParamsInt64(":id") + datasourceID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + c.JsonApiErr(http.StatusBadRequest, "id is invalid", err) + return + } ds, err := hs.DataSourceCache.GetDatasource(c.Req.Context(), datasourceID, c.SignedInUser, c.SkipCache) if err != nil { if errors.Is(err, models.ErrDataSourceAccessDenied) { @@ -465,24 +514,27 @@ func convertModelToDtos(ds *models.DataSource) dtos.DataSource { // CheckDatasourceHealth sends a health check request to the plugin datasource // /api/datasource/:id/health func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Response { - datasourceID := c.ParamsInt64(":id") + datasourceID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } ds, err := hs.DataSourceCache.GetDatasource(c.Req.Context(), datasourceID, c.SignedInUser, c.SkipCache) if err != nil { if errors.Is(err, models.ErrDataSourceAccessDenied) { - return response.Error(403, "Access denied to datasource", err) + return response.Error(http.StatusForbidden, "Access denied to datasource", err) } - return response.Error(500, "Unable to load datasource metadata", err) + return response.Error(http.StatusInternalServerError, "Unable to load datasource metadata", err) } plugin, exists := hs.pluginStore.Plugin(c.Req.Context(), ds.Type) if !exists { - return response.Error(500, "Unable to find datasource plugin", err) + return response.Error(http.StatusInternalServerError, "Unable to find datasource plugin", err) } dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn()) if err != nil { - return response.Error(500, "Unable to get datasource model", err) + return response.Error(http.StatusInternalServerError, "Unable to get datasource model", err) } req := &backend.CheckHealthRequest{ PluginContext: backend.PluginContext{ @@ -493,6 +545,16 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo }, } + var dsURL string + if req.PluginContext.DataSourceInstanceSettings != nil { + dsURL = req.PluginContext.DataSourceInstanceSettings.URL + } + + err = hs.PluginRequestValidator.Validate(dsURL, c.Req) + if err != nil { + return response.Error(http.StatusForbidden, "Access denied", err) + } + resp, err := hs.pluginClient.CheckHealth(c.Req.Context(), req) if err != nil { return translatePluginRequestErrorToAPIError(err) @@ -508,17 +570,17 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo var jsonDetails map[string]interface{} err = json.Unmarshal(resp.JSONDetails, &jsonDetails) if err != nil { - return response.Error(500, "Failed to unmarshal detailed response from backend plugin", err) + return response.Error(http.StatusInternalServerError, "Failed to unmarshal detailed response from backend plugin", err) } payload["details"] = jsonDetails } if resp.Status != backend.HealthStatusOk { - return response.JSON(400, payload) + return response.JSON(http.StatusBadRequest, payload) } - return response.JSON(200, payload) + return response.JSON(http.StatusOK, payload) } func (hs *HTTPServer) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string { @@ -530,3 +592,19 @@ func (hs *HTTPServer) decryptSecureJsonDataFn() func(map[string][]byte) map[stri return decryptedJsonData } } + +func filterDatasourcesByQueryPermission(ctx context.Context, user *models.SignedInUser, datasources []*models.DataSource) ([]*models.DataSource, error) { + query := models.DatasourcesPermissionFilterQuery{ + User: user, + Datasources: datasources, + } + + if err := bus.Dispatch(ctx, &query); err != nil { + if !errors.Is(err, bus.ErrHandlerNotFound) { + return nil, err + } + return datasources, nil + } + + return query.Result, nil +} diff --git a/pkg/api/datasources_test.go b/pkg/api/datasources_test.go index c95bf866cea..ac43d1cd016 100644 --- a/pkg/api/datasources_test.go +++ b/pkg/api/datasources_test.go @@ -12,13 +12,12 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/bus" ) const ( @@ -28,7 +27,7 @@ const ( ) func TestDataSourcesProxy_userLoggedIn(t *testing.T) { - loggedInUserScenario(t, "When calling GET on", "/api/datasources/", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "/api/datasources/", "/api/datasources/", func(sc *scenarioContext) { // Stubs the database query bus.AddHandler("test", func(ctx context.Context, query *models.GetDataSourcesQuery) error { assert.Equal(t, testOrgID, query.OrgId) @@ -61,7 +60,7 @@ func TestDataSourcesProxy_userLoggedIn(t *testing.T) { }) loggedInUserScenario(t, "Should be able to save a data source when calling DELETE on non-existing", - "/api/datasources/name/12345", func(sc *scenarioContext) { + "/api/datasources/name/12345", "/api/datasources/name/:name", func(sc *scenarioContext) { // handler func being tested hs := &HTTPServer{ Bus: bus.GetBus(), @@ -192,6 +191,16 @@ func TestAPI_Datasources_AccessControl(t *testing.T) { Type: "postgresql", Access: "Proxy", } + testDatasourceReadOnly := models.DataSource{ + Id: 4, + Uid: "testUID", + OrgId: testOrgID, + Name: "test", + Url: "http://localhost:5432", + Type: "postgresql", + Access: "Proxy", + ReadOnly: true, + } getDatasourceStub := func(ctx context.Context, query *models.GetDataSourceQuery) error { result := testDatasource result.Id = query.Id @@ -211,6 +220,21 @@ func TestAPI_Datasources_AccessControl(t *testing.T) { cmd.Result = &testDatasource return nil } + updateDatasourceReadOnlyStub := func(ctx context.Context, cmd *models.UpdateDataSourceCommand) error { + cmd.Result = &testDatasourceReadOnly + return nil + } + + getDatasourceNotFoundStub := func(ctx context.Context, cmd *models.GetDataSourceQuery) error { + cmd.Result = nil + return models.ErrDataSourceNotFound + } + + getDatasourceReadOnlyStub := func(ctx context.Context, query *models.GetDataSourceQuery) error { + query.Result = &testDatasourceReadOnly + return nil + } + deleteDatasourceStub := func(ctx context.Context, cmd *models.DeleteDataSourceCommand) error { cmd.DeletedDatasourcesCount = 1 return nil @@ -233,13 +257,28 @@ func TestAPI_Datasources_AccessControl(t *testing.T) { }) return bytes.NewReader(s) } - type acTestCaseWithHandler struct { busStubs []bus.HandlerFunc body func() io.Reader accessControlTestCase } tests := []acTestCaseWithHandler{ + { + busStubs: []bus.HandlerFunc{getDatasourceNotFoundStub, updateDatasourceStub}, + body: updateDatasourceBody, + accessControlTestCase: accessControlTestCase{ + expectedCode: http.StatusNotFound, + desc: "DatasourcesPut should return 404 if datasource not found", + url: fmt.Sprintf("/api/datasources/%v", "12345678"), + method: http.MethodPut, + permissions: []*accesscontrol.Permission{ + { + Action: ActionDatasourcesWrite, + Scope: ScopeDatasourcesAll, + }, + }, + }, + }, { busStubs: []bus.HandlerFunc{getDatasourcesStub}, accessControlTestCase: accessControlTestCase{ @@ -304,6 +343,22 @@ func TestAPI_Datasources_AccessControl(t *testing.T) { permissions: []*accesscontrol.Permission{{Action: "wrong"}}, }, }, + { + busStubs: []bus.HandlerFunc{getDatasourceReadOnlyStub, updateDatasourceReadOnlyStub}, + body: updateDatasourceBody, + accessControlTestCase: accessControlTestCase{ + expectedCode: http.StatusForbidden, + desc: "DatasourcesPut should return 403 for read only datasource", + url: fmt.Sprintf("/api/datasources/%v", testDatasourceReadOnly.Id), + method: http.MethodPut, + permissions: []*accesscontrol.Permission{ + { + Action: ActionDatasourcesWrite, + Scope: fmt.Sprintf("datasources:id:%v", testDatasourceReadOnly.Id), + }, + }, + }, + }, { busStubs: []bus.HandlerFunc{getDatasourceStub, deleteDatasourceStub}, accessControlTestCase: accessControlTestCase{ @@ -505,6 +560,7 @@ func TestAPI_Datasources_AccessControl(t *testing.T) { } else { sc.req, err = http.NewRequest(test.method, test.url, nil) } + assert.NoError(t, err) sc.exec() diff --git a/pkg/api/dtos/index.go b/pkg/api/dtos/index.go index 59388d6c293..5afb1e62064 100644 --- a/pkg/api/dtos/index.go +++ b/pkg/api/dtos/index.go @@ -56,20 +56,21 @@ const ( ) type NavLink struct { - Id string `json:"id,omitempty"` - Text string `json:"text,omitempty"` - Description string `json:"description,omitempty"` - Section string `json:"section,omitempty"` - SubTitle string `json:"subTitle,omitempty"` - Icon string `json:"icon,omitempty"` - Img string `json:"img,omitempty"` - Url string `json:"url,omitempty"` - Target string `json:"target,omitempty"` - SortWeight int64 `json:"sortWeight,omitempty"` - Divider bool `json:"divider,omitempty"` - HideFromMenu bool `json:"hideFromMenu,omitempty"` - HideFromTabs bool `json:"hideFromTabs,omitempty"` - Children []*NavLink `json:"children,omitempty"` + Id string `json:"id,omitempty"` + Text string `json:"text,omitempty"` + Description string `json:"description,omitempty"` + Section string `json:"section,omitempty"` + SubTitle string `json:"subTitle,omitempty"` + Icon string `json:"icon,omitempty"` + Img string `json:"img,omitempty"` + Url string `json:"url,omitempty"` + Target string `json:"target,omitempty"` + SortWeight int64 `json:"sortWeight,omitempty"` + Divider bool `json:"divider,omitempty"` + HideFromMenu bool `json:"hideFromMenu,omitempty"` + HideFromTabs bool `json:"hideFromTabs,omitempty"` + Children []*NavLink `json:"children,omitempty"` + HighlightText string `json:"highlightText,omitempty"` } // NavIDCfg is the id for org configuration navigation node diff --git a/pkg/api/dtos/plugins.go b/pkg/api/dtos/plugins.go index dcef9df59aa..aea91af657d 100644 --- a/pkg/api/dtos/plugins.go +++ b/pkg/api/dtos/plugins.go @@ -1,7 +1,6 @@ package dtos import ( - "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/plugins" ) @@ -59,16 +58,6 @@ func (slice PluginList) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } -type ImportDashboardCommand struct { - PluginId string `json:"pluginId"` - Path string `json:"path"` - Overwrite bool `json:"overwrite"` - Dashboard *simplejson.Json `json:"dashboard"` - Inputs []plugins.ImportDashboardInput `json:"inputs"` - FolderId int64 `json:"folderId"` - FolderUid string `json:"folderUid"` -} - type InstallPluginCommand struct { Version string `json:"version"` } diff --git a/pkg/api/folder.go b/pkg/api/folder.go index a59c3cb483b..a2f2ad16375 100644 --- a/pkg/api/folder.go +++ b/pkg/api/folder.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/apierrors" "github.com/grafana/grafana/pkg/api/dtos" @@ -51,7 +52,13 @@ func (hs *HTTPServer) GetFolderByUID(c *models.ReqContext) response.Response { func (hs *HTTPServer) GetFolderByID(c *models.ReqContext) response.Response { s := dashboards.NewFolderService(c.OrgId, c.SignedInUser, hs.SQLStore) - folder, err := s.GetFolderByID(c.Req.Context(), c.ParamsInt64(":id")) + + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + + folder, err := s.GetFolderByID(c.Req.Context(), id) if err != nil { return apierrors.ToFolderErrorResponse(err) } diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go index 49932626ddb..0e45d41c620 100644 --- a/pkg/api/frontendsettings.go +++ b/pkg/api/frontendsettings.go @@ -2,7 +2,6 @@ package api import ( "context" - "errors" "strconv" "github.com/grafana/grafana/pkg/bus" @@ -25,20 +24,12 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab return nil, err } - dsFilterQuery := models.DatasourcesPermissionFilterQuery{ - User: c.SignedInUser, - Datasources: query.Result, + filtered, err := filterDatasourcesByQueryPermission(c.Req.Context(), c.SignedInUser, query.Result) + if err != nil { + return nil, err } - if err := bus.Dispatch(c.Req.Context(), &dsFilterQuery); err != nil { - if !errors.Is(err, bus.ErrHandlerNotFound) { - return nil, err - } - - orgDataSources = query.Result - } else { - orgDataSources = dsFilterQuery.Result - } + orgDataSources = filtered } dataSources := make(map[string]plugins.DataSourceDTO) @@ -223,6 +214,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i "verifyEmailEnabled": setting.VerifyEmailEnabled, "sigV4AuthEnabled": setting.SigV4AuthEnabled, "exploreEnabled": setting.ExploreEnabled, + "queryHistoryEnabled": hs.Cfg.QueryHistoryEnabled, "googleAnalyticsId": setting.GoogleAnalyticsId, "rudderstackWriteKey": setting.RudderstackWriteKey, "rudderstackDataPlaneUrl": setting.RudderstackDataPlaneUrl, @@ -247,8 +239,8 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i "commit": commit, "buildstamp": buildstamp, "edition": hs.License.Edition(), - "latestVersion": hs.updateChecker.LatestGrafanaVersion(), - "hasUpdate": hs.updateChecker.GrafanaUpdateAvailable(), + "latestVersion": hs.grafanaUpdateChecker.LatestVersion(), + "hasUpdate": hs.grafanaUpdateChecker.UpdateAvailable(), "env": setting.Env, }, "licenseInfo": map[string]interface{}{ @@ -258,7 +250,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i "edition": hs.License.Edition(), "enabledFeatures": hs.License.EnabledFeatures(), }, - "featureToggles": hs.Cfg.FeatureToggles, + "featureToggles": hs.Features.GetEnabled(c.Req.Context()), "rendererAvailable": hs.RenderService.IsAvailable(), "rendererVersion": hs.RenderService.Version(), "http2Enabled": hs.Cfg.Protocol == setting.HTTP2Scheme, @@ -278,7 +270,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i "enabled": hs.Cfg.SectionWithEnvOverrides("caching").Key("enabled").MustBool(true), }, "recordedQueries": map[string]bool{ - "enabled": hs.Cfg.SectionWithEnvOverrides("recorded_queries").Key("enabled").MustBool(false), + "enabled": hs.Cfg.SectionWithEnvOverrides("recorded_queries").Key("enabled").MustBool(true), }, "unifiedAlertingEnabled": hs.Cfg.UnifiedAlerting.Enabled, "featureHighlights": map[string]bool{ diff --git a/pkg/api/frontendsettings_test.go b/pkg/api/frontendsettings_test.go index f873f8ce1b5..11a5a950434 100644 --- a/pkg/api/frontendsettings_test.go +++ b/pkg/api/frontendsettings_test.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/bus" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/services/sqlstore" @@ -19,9 +20,10 @@ import ( "github.com/stretchr/testify/require" ) -func setupTestEnvironment(t *testing.T, cfg *setting.Cfg) (*web.Mux, *HTTPServer) { +func setupTestEnvironment(t *testing.T, cfg *setting.Cfg, features *featuremgmt.FeatureManager) (*web.Mux, *HTTPServer) { t.Helper() sqlstore.InitTestDB(t) + cfg.IsFeatureToggleEnabled = features.IsEnabled { oldVersion := setting.BuildVersion @@ -37,18 +39,19 @@ func setupTestEnvironment(t *testing.T, cfg *setting.Cfg) (*web.Mux, *HTTPServer sqlStore := sqlstore.InitTestDB(t) hs := &HTTPServer{ - Cfg: cfg, - Bus: bus.GetBus(), - License: &licensing.OSSLicensingService{Cfg: cfg}, + Cfg: cfg, + Features: features, + Bus: bus.GetBus(), + License: &licensing.OSSLicensingService{Cfg: cfg}, RenderService: &rendering.RenderingService{ Cfg: cfg, RendererPluginManager: &fakeRendererManager{}, }, - SQLStore: sqlStore, - SettingsProvider: setting.ProvideProvider(cfg), - pluginStore: &fakePluginStore{}, - updateChecker: &updatechecker.Service{}, - AccessControl: accesscontrolmock.New().WithDisabled(), + SQLStore: sqlStore, + SettingsProvider: setting.ProvideProvider(cfg), + pluginStore: &fakePluginStore{}, + grafanaUpdateChecker: &updatechecker.GrafanaService{}, + AccessControl: accesscontrolmock.New().WithDisabled(), } m := web.New() @@ -73,7 +76,8 @@ func TestHTTPServer_GetFrontendSettings_hideVersionAnonymous(t *testing.T) { cfg.Env = "testing" cfg.BuildVersion = "7.8.9" cfg.BuildCommit = "01234567" - m, hs := setupTestEnvironment(t, cfg) + + m, hs := setupTestEnvironment(t, cfg, featuremgmt.WithFeatures()) req := httptest.NewRequest(http.MethodGet, "/api/frontend/settings", nil) diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 4b106b8db56..5a33efa7706 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -13,10 +13,6 @@ import ( "strings" "sync" - "github.com/grafana/grafana/pkg/services/query" - "github.com/grafana/grafana/pkg/services/serviceaccounts" - "github.com/grafana/grafana/pkg/services/thumbs" - "github.com/grafana/grafana/pkg/api/routing" httpstatic "github.com/grafana/grafana/pkg/api/static" "github.com/grafana/grafana/pkg/bus" @@ -32,29 +28,38 @@ import ( "github.com/grafana/grafana/pkg/plugins/plugincontext" "github.com/grafana/grafana/pkg/services/accesscontrol" acmiddleware "github.com/grafana/grafana/pkg/services/accesscontrol/middleware" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourceservices" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/cleanup" "github.com/grafana/grafana/pkg/services/contexthandler" "github.com/grafana/grafana/pkg/services/datasourceproxy" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/encryption" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/hooks" + "github.com/grafana/grafana/pkg/services/ldap" "github.com/grafana/grafana/pkg/services/libraryelements" "github.com/grafana/grafana/pkg/services/librarypanels" "github.com/grafana/grafana/pkg/services/live" "github.com/grafana/grafana/pkg/services/live/pushhttp" "github.com/grafana/grafana/pkg/services/login" + "github.com/grafana/grafana/pkg/services/login/authinfoservice" "github.com/grafana/grafana/pkg/services/ngalert" "github.com/grafana/grafana/pkg/services/provisioning" + "github.com/grafana/grafana/pkg/services/query" + "github.com/grafana/grafana/pkg/services/queryhistory" "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/services/schemaloader" "github.com/grafana/grafana/pkg/services/search" "github.com/grafana/grafana/pkg/services/searchusers" "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/shorturls" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/teamguardian" + "github.com/grafana/grafana/pkg/services/thumbs" "github.com/grafana/grafana/pkg/services/updatechecker" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util/errutil" @@ -64,17 +69,19 @@ import ( ) type HTTPServer struct { - log log.Logger - web *web.Mux - context context.Context - httpSrv *http.Server - middlewares []web.Handler + log log.Logger + web *web.Mux + context context.Context + httpSrv *http.Server + middlewares []web.Handler + namedMiddlewares []routing.RegisterNamedMiddleware PluginContextProvider *plugincontext.Provider RouteRegister routing.RouteRegister Bus bus.Bus RenderService rendering.Service Cfg *setting.Cfg + Features *featuremgmt.FeatureManager SettingsProvider setting.Provider HooksService *hooks.HooksService CacheService *localcache.CacheService @@ -95,6 +102,7 @@ type HTTPServer struct { pluginErrorResolver plugins.ErrorResolver SearchService *search.SearchService ShortURLService shorturls.Service + QueryHistoryService queryhistory.Service Live *live.GrafanaLive LivePushGateway *pushhttp.Gateway ThumbService thumbs.Service @@ -111,12 +119,16 @@ type HTTPServer struct { SecretsService secrets.Service DataSourcesService *datasources.Service cleanUpService *cleanup.CleanUpService - tracingService tracing.Tracer - updateChecker *updatechecker.Service + tracer tracing.Tracer + grafanaUpdateChecker *updatechecker.GrafanaService + pluginsUpdateChecker *updatechecker.PluginsService searchUsersService searchusers.Service + ldapGroups ldap.Groups teamGuardian teamguardian.TeamGuardian queryDataService *query.Service serviceAccountsService serviceaccounts.Service + authInfoService authinfoservice.Service + TeamPermissionsService *resourcepermissions.Service } type ServerOptions struct { @@ -130,18 +142,20 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi pluginDashboardManager plugins.PluginDashboardManager, pluginStore plugins.Store, pluginClient plugins.Client, pluginErrorResolver plugins.ErrorResolver, settingsProvider setting.Provider, dataSourceCache datasources.CacheService, userTokenService models.UserTokenService, - cleanUpService *cleanup.CleanUpService, shortURLService shorturls.Service, thumbService thumbs.Service, - remoteCache *remotecache.RemoteCache, provisioningService provisioning.ProvisioningService, + cleanUpService *cleanup.CleanUpService, shortURLService shorturls.Service, queryHistoryService queryhistory.Service, + thumbService thumbs.Service, remoteCache *remotecache.RemoteCache, provisioningService provisioning.ProvisioningService, loginService login.Service, accessControl accesscontrol.AccessControl, dataSourceProxy *datasourceproxy.DataSourceProxyService, searchService *search.SearchService, live *live.GrafanaLive, livePushGateway *pushhttp.Gateway, plugCtxProvider *plugincontext.Provider, - contextHandler *contexthandler.ContextHandler, + contextHandler *contexthandler.ContextHandler, features *featuremgmt.FeatureManager, schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG, libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service, - quotaService *quota.QuotaService, socialService social.Service, tracingService tracing.Tracer, - encryptionService encryption.Internal, updateChecker *updatechecker.Service, searchUsersService searchusers.Service, + quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer, + encryptionService encryption.Internal, grafanaUpdateChecker *updatechecker.GrafanaService, + pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service, dataSourcesService *datasources.Service, secretsService secrets.Service, queryDataService *query.Service, - teamGuardian teamguardian.TeamGuardian, serviceaccountsService serviceaccounts.Service) (*HTTPServer, error) { + ldapGroups ldap.Groups, teamGuardian teamguardian.TeamGuardian, serviceaccountsService serviceaccounts.Service, + authInfoService authinfoservice.Service, resourcePermissionServices *resourceservices.ResourceServices) (*HTTPServer, error) { web.Env = cfg.Env m := web.New() @@ -161,12 +175,15 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi pluginStaticRouteResolver: pluginStaticRouteResolver, pluginDashboardManager: pluginDashboardManager, pluginErrorResolver: pluginErrorResolver, - updateChecker: updateChecker, + grafanaUpdateChecker: grafanaUpdateChecker, + pluginsUpdateChecker: pluginsUpdateChecker, SettingsProvider: settingsProvider, DataSourceCache: dataSourceCache, AuthTokenService: userTokenService, cleanUpService: cleanUpService, ShortURLService: shortURLService, + QueryHistoryService: queryHistoryService, + Features: features, ThumbService: thumbService, RemoteCacheService: remoteCache, ProvisioningService: provisioningService, @@ -183,7 +200,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi LibraryPanelService: libraryPanelService, LibraryElementService: libraryElementService, QuotaService: quotaService, - tracingService: tracingService, + tracer: tracer, log: log.New("http.server"), web: m, Listener: opts.Listener, @@ -192,9 +209,12 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi SecretsService: secretsService, DataSourcesService: dataSourcesService, searchUsersService: searchUsersService, + ldapGroups: ldapGroups, teamGuardian: teamGuardian, queryDataService: queryDataService, serviceAccountsService: serviceaccountsService, + authInfoService: authInfoService, + TeamPermissionsService: resourcePermissionServices.GetTeamService(), } if hs.Listener != nil { hs.log.Debug("Using provided listener") @@ -211,6 +231,10 @@ func (hs *HTTPServer) AddMiddleware(middleware web.Handler) { hs.middlewares = append(hs.middlewares, middleware) } +func (hs *HTTPServer) AddNamedMiddleware(middleware routing.RegisterNamedMiddleware) { + hs.namedMiddlewares = append(hs.namedMiddlewares, middleware) +} + func (hs *HTTPServer) Run(ctx context.Context) error { hs.context = ctx @@ -397,7 +421,7 @@ func (hs *HTTPServer) applyRoutes() { // start with middlewares & static routes hs.addMiddlewaresAndStaticRoutes() // then add view routes & api routes - hs.RouteRegister.Register(hs.web) + hs.RouteRegister.Register(hs.web, hs.namedMiddlewares...) // then custom app proxy routes hs.initAppPluginRoutes(hs.web) // lastly not found route @@ -407,7 +431,7 @@ func (hs *HTTPServer) applyRoutes() { func (hs *HTTPServer) addMiddlewaresAndStaticRoutes() { m := hs.web - m.Use(middleware.RequestTracing()) + m.Use(middleware.RequestTracing(hs.tracer)) m.Use(middleware.Logger(hs.Cfg)) diff --git a/pkg/api/index.go b/pkg/api/index.go index 9de80ede62b..f07b9ba375a 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" ac "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" ) @@ -85,7 +86,7 @@ func (hs *HTTPServer) getAppLinks(c *models.ReqContext) ([]*dtos.NavLink, error) SortWeight: dtos.WeightPlugin, } - if hs.Cfg.IsNewNavigationEnabled() { + if hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { appLink.Section = dtos.NavSectionPlugin } else { appLink.Section = dtos.NavSectionCore @@ -143,7 +144,9 @@ func (hs *HTTPServer) getAppLinks(c *models.ReqContext) ([]*dtos.NavLink, error) } func enableServiceAccount(hs *HTTPServer, c *models.ReqContext) bool { - return c.OrgRole == models.ROLE_ADMIN && hs.Cfg.IsServiceAccountEnabled() && hs.serviceAccountsService.Migrated(c.Req.Context(), c.OrgId) + return c.OrgRole == models.ROLE_ADMIN && + hs.Features.IsEnabled(featuremgmt.FlagServiceAccounts) && + hs.serviceAccountsService.Migrated(c.Req.Context(), c.OrgId) } func enableTeams(hs *HTTPServer, c *models.ReqContext) bool { @@ -154,7 +157,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto hasAccess := ac.HasAccess(hs.AccessControl, c) navTree := []*dtos.NavLink{} - if hs.Cfg.IsNewNavigationEnabled() { + if hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { navTree = append(navTree, &dtos.NavLink{ Text: "Home", Id: "home", @@ -165,7 +168,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto }) } - if hasEditPerm && !hs.Cfg.IsNewNavigationEnabled() { + if hasEditPerm && !hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { children := hs.buildCreateNavLinks(c) navTree = append(navTree, &dtos.NavLink{ Text: "Create", @@ -181,7 +184,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto dashboardChildLinks := hs.buildDashboardNavLinks(c, hasEditPerm) dashboardsUrl := "/" - if hs.Cfg.IsNewNavigationEnabled() { + if hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { dashboardsUrl = "/dashboards" } @@ -251,7 +254,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto }) } - if hasAccess(ac.ReqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead, ac.ScopeUsersAll)) { + if hasAccess(ac.ReqOrgAdmin, ac.EvalPermission(ac.ActionOrgUsersRead)) { configNodes = append(configNodes, &dtos.NavLink{ Text: "Users", Id: "users", @@ -312,7 +315,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto }) } - if hs.Cfg.FeatureToggles["live-pipeline"] { + if hs.Features.IsEnabled(featuremgmt.FlagLivePipeline) { liveNavLinks := []*dtos.NavLink{} liveNavLinks = append(liveNavLinks, &dtos.NavLink{ @@ -327,7 +330,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto navTree = append(navTree, &dtos.NavLink{ Id: "live", Text: "Live", - SubTitle: "Event Streaming", + SubTitle: "Event streaming", Icon: "exchange-alt", Url: hs.Cfg.AppSubURL + "/live", Children: liveNavLinks, @@ -346,7 +349,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto SortWeight: dtos.WeightConfig, Children: configNodes, } - if hs.Cfg.IsNewNavigationEnabled() { + if hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { configNode.Section = dtos.NavSectionConfig } else { configNode.Section = dtos.NavSectionCore @@ -358,7 +361,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto if len(adminNavLinks) > 0 { navSection := dtos.NavSectionCore - if hs.Cfg.IsNewNavigationEnabled() { + if hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { navSection = dtos.NavSectionConfig } serverAdminNode := navlinks.GetServerAdminNode(adminNavLinks, navSection) @@ -386,7 +389,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto func (hs *HTTPServer) buildDashboardNavLinks(c *models.ReqContext, hasEditPerm bool) []*dtos.NavLink { dashboardChildNavs := []*dtos.NavLink{} - if !hs.Cfg.IsNewNavigationEnabled() { + if !hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { dashboardChildNavs = append(dashboardChildNavs, &dtos.NavLink{ Text: "Home", Id: "home", Url: hs.Cfg.AppSubURL + "/", Icon: "home-alt", HideFromTabs: true, }) @@ -417,16 +420,16 @@ func (hs *HTTPServer) buildDashboardNavLinks(c *models.ReqContext, hasEditPerm b }) } - if hasEditPerm && hs.Cfg.IsNewNavigationEnabled() { + if hasEditPerm && hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { dashboardChildNavs = append(dashboardChildNavs, &dtos.NavLink{ Text: "Divider", Divider: true, Id: "divider", HideFromTabs: true, }) dashboardChildNavs = append(dashboardChildNavs, &dtos.NavLink{ - Text: "New dashboard", Icon: "plus", Url: hs.Cfg.AppSubURL + "/dashboard/new", HideFromTabs: true, + Text: "New dashboard", Icon: "plus", Url: hs.Cfg.AppSubURL + "/dashboard/new", HideFromTabs: true, Id: "new-dashboard", }) if c.OrgRole == models.ROLE_ADMIN || c.OrgRole == models.ROLE_EDITOR { dashboardChildNavs = append(dashboardChildNavs, &dtos.NavLink{ - Text: "New folder", SubTitle: "Create a new folder to organize your dashboards", Id: "folder", + Text: "New folder", SubTitle: "Create a new folder to organize your dashboards", Id: "new-folder", Icon: "plus", Url: hs.Cfg.AppSubURL + "/dashboards/folder/new", HideFromTabs: true, }) } @@ -472,7 +475,7 @@ func (hs *HTTPServer) buildAlertNavLinks(c *models.ReqContext, uaVisibleForOrg b func (hs *HTTPServer) buildCreateNavLinks(c *models.ReqContext) []*dtos.NavLink { children := []*dtos.NavLink{ - {Text: "Dashboard", Icon: "apps", Url: hs.Cfg.AppSubURL + "/dashboard/new"}, + {Text: "Dashboard", Icon: "apps", Url: hs.Cfg.AppSubURL + "/dashboard/new", Id: "create-dashboard"}, } if c.OrgRole == models.ROLE_ADMIN || c.OrgRole == models.ROLE_EDITOR { children = append(children, &dtos.NavLink{ @@ -608,8 +611,8 @@ func (hs *HTTPServer) setIndexViewData(c *models.ReqContext) (*dtos.IndexViewDat GoogleTagManagerId: setting.GoogleTagManagerId, BuildVersion: setting.BuildVersion, BuildCommit: setting.BuildCommit, - NewGrafanaVersion: hs.updateChecker.LatestGrafanaVersion(), - NewGrafanaVersionExists: hs.updateChecker.GrafanaUpdateAvailable(), + NewGrafanaVersion: hs.grafanaUpdateChecker.LatestVersion(), + NewGrafanaVersionExists: hs.grafanaUpdateChecker.UpdateAvailable(), AppName: setting.ApplicationName, AppNameBodyClass: "app-grafana", FavIcon: "public/img/fav32.png", @@ -622,7 +625,7 @@ func (hs *HTTPServer) setIndexViewData(c *models.ReqContext) (*dtos.IndexViewDat LoadingLogo: "public/img/grafana_icon.svg", } - if hs.Cfg.FeatureToggles["accesscontrol"] { + if hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { userPermissions, err := hs.AccessControl.GetUserPermissions(c.Req.Context(), c.SignedInUser) if err != nil { return nil, err diff --git a/pkg/api/ldap_debug.go b/pkg/api/ldap_debug.go index 838820445c8..ad223cdb66f 100644 --- a/pkg/api/ldap_debug.go +++ b/pkg/api/ldap_debug.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/bus" @@ -163,7 +164,10 @@ func (hs *HTTPServer) PostSyncUserWithLDAP(c *models.ReqContext) response.Respon return response.Error(http.StatusBadRequest, "Failed to obtain the LDAP configuration. Please verify the configuration and try again", err) } - userId := c.ParamsInt64(":id") + userId, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } query := models.GetUserByIdQuery{Id: userId} @@ -307,14 +311,11 @@ func (hs *HTTPServer) GetUserFromLDAP(c *models.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "An organization was not found - Please verify your LDAP configuration", err) } - cmd := &models.GetTeamsForLDAPGroupCommand{Groups: user.Groups} - err = bus.Dispatch(c.Req.Context(), cmd) - if err != nil && !errors.Is(err, bus.ErrHandlerNotFound) { + u.Teams, err = hs.ldapGroups.GetTeams(user.Groups) + if err != nil { return response.Error(http.StatusBadRequest, "Unable to find the teams for this user", err) } - u.Teams = cmd.Result - return response.JSON(200, u) } diff --git a/pkg/api/ldap_debug_test.go b/pkg/api/ldap_debug_test.go index 503005103d6..9cb6e339723 100644 --- a/pkg/api/ldap_debug_test.go +++ b/pkg/api/ldap_debug_test.go @@ -62,7 +62,7 @@ func getUserFromLDAPContext(t *testing.T, requestURL string) *scenarioContext { setting.LDAPEnabled = true t.Cleanup(func() { setting.LDAPEnabled = origLDAP }) - hs := &HTTPServer{Cfg: setting.NewCfg()} + hs := &HTTPServer{Cfg: setting.NewCfg(), ldapGroups: ldap.ProvideGroupsService()} sc.defaultHandler = routing.Wrap(func(c *models.ReqContext) response.Response { sc.context = c @@ -274,11 +274,6 @@ func TestGetUserFromLDAPAPIEndpoint_WithTeamHandler(t *testing.T) { return nil }) - bus.AddHandler("test", func(ctx context.Context, cmd *models.GetTeamsForLDAPGroupCommand) error { - cmd.Result = []models.TeamOrgGroupDTO{} - return nil - }) - getLDAPConfig = func(*setting.Cfg) (*ldap.Config, error) { return &ldap.Config{}, nil } @@ -310,7 +305,7 @@ func TestGetUserFromLDAPAPIEndpoint_WithTeamHandler(t *testing.T) { "roles": [ { "orgId": 1, "orgRole": "Admin", "orgName": "Main Org.", "groupDN": "cn=admins,ou=groups,dc=grafana,dc=org" } ], - "teams": [] + "teams": null } ` @@ -639,7 +634,7 @@ func TestLDAP_AccessControl(t *testing.T) { }, }, { - url: "/api/admin/ldap/sync/test", + url: "/api/admin/ldap/sync/1", method: http.MethodPost, desc: "PostSyncUserWithLDAP should return 200 for user without required permissions", expectedCode: http.StatusOK, @@ -648,7 +643,7 @@ func TestLDAP_AccessControl(t *testing.T) { }, }, { - url: "/api/admin/ldap/sync/test", + url: "/api/admin/ldap/sync/1", method: http.MethodPost, desc: "PostSyncUserWithLDAP should return 200 for user without required permissions", expectedCode: http.StatusForbidden, diff --git a/pkg/api/org.go b/pkg/api/org.go index 68e88d25d4c..7aeea8e5c81 100644 --- a/pkg/api/org.go +++ b/pkg/api/org.go @@ -4,11 +4,13 @@ import ( "context" "errors" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -22,7 +24,11 @@ func GetCurrentOrg(c *models.ReqContext) response.Response { // GET /api/orgs/:orgId func GetOrgByID(c *models.ReqContext) response.Response { - return getOrgHelper(c.Req.Context(), c.ParamsInt64(":orgId")) + orgId, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } + return getOrgHelper(c.Req.Context(), orgId) } // GET /api/orgs/name/:name @@ -84,7 +90,7 @@ func (hs *HTTPServer) CreateOrg(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - acEnabled := hs.Cfg.FeatureToggles["accesscontrol"] + acEnabled := hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) if !acEnabled && !(setting.AllowUserOrgCreate || c.IsGrafanaAdmin) { return response.Error(403, "Access denied", nil) } @@ -106,26 +112,30 @@ func (hs *HTTPServer) CreateOrg(c *models.ReqContext) response.Response { } // PUT /api/org -func UpdateCurrentOrg(c *models.ReqContext) response.Response { +func (hs *HTTPServer) UpdateCurrentOrg(c *models.ReqContext) response.Response { form := dtos.UpdateOrgForm{} if err := web.Bind(c.Req, &form); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - return updateOrgHelper(c.Req.Context(), form, c.OrgId) + return hs.updateOrgHelper(c.Req.Context(), form, c.OrgId) } // PUT /api/orgs/:orgId -func UpdateOrg(c *models.ReqContext) response.Response { +func (hs *HTTPServer) UpdateOrg(c *models.ReqContext) response.Response { form := dtos.UpdateOrgForm{} if err := web.Bind(c.Req, &form); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - return updateOrgHelper(c.Req.Context(), form, c.ParamsInt64(":orgId")) + orgId, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } + return hs.updateOrgHelper(c.Req.Context(), form, orgId) } -func updateOrgHelper(ctx context.Context, form dtos.UpdateOrgForm, orgID int64) response.Response { +func (hs *HTTPServer) updateOrgHelper(ctx context.Context, form dtos.UpdateOrgForm, orgID int64) response.Response { cmd := models.UpdateOrgCommand{Name: form.Name, OrgId: orgID} - if err := sqlstore.UpdateOrg(ctx, &cmd); err != nil { + if err := hs.SQLStore.UpdateOrg(ctx, &cmd); err != nil { if errors.Is(err, models.ErrOrgNameTaken) { return response.Error(400, "Organization name taken", err) } @@ -136,24 +146,28 @@ func updateOrgHelper(ctx context.Context, form dtos.UpdateOrgForm, orgID int64) } // PUT /api/org/address -func UpdateCurrentOrgAddress(c *models.ReqContext) response.Response { +func (hs *HTTPServer) UpdateCurrentOrgAddress(c *models.ReqContext) response.Response { form := dtos.UpdateOrgAddressForm{} if err := web.Bind(c.Req, &form); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - return updateOrgAddressHelper(c.Req.Context(), form, c.OrgId) + return hs.updateOrgAddressHelper(c.Req.Context(), form, c.OrgId) } // PUT /api/orgs/:orgId/address -func UpdateOrgAddress(c *models.ReqContext) response.Response { +func (hs *HTTPServer) UpdateOrgAddress(c *models.ReqContext) response.Response { form := dtos.UpdateOrgAddressForm{} if err := web.Bind(c.Req, &form); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - return updateOrgAddressHelper(c.Req.Context(), form, c.ParamsInt64(":orgId")) + orgId, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } + return hs.updateOrgAddressHelper(c.Req.Context(), form, orgId) } -func updateOrgAddressHelper(ctx context.Context, form dtos.UpdateOrgAddressForm, orgID int64) response.Response { +func (hs *HTTPServer) updateOrgAddressHelper(ctx context.Context, form dtos.UpdateOrgAddressForm, orgID int64) response.Response { cmd := models.UpdateOrgAddressCommand{ OrgId: orgID, Address: models.Address{ @@ -166,7 +180,7 @@ func updateOrgAddressHelper(ctx context.Context, form dtos.UpdateOrgAddressForm, }, } - if err := sqlstore.UpdateOrgAddress(ctx, &cmd); err != nil { + if err := hs.SQLStore.UpdateOrgAddress(ctx, &cmd); err != nil { return response.Error(500, "Failed to update org address", err) } @@ -174,14 +188,17 @@ func updateOrgAddressHelper(ctx context.Context, form dtos.UpdateOrgAddressForm, } // DELETE /api/orgs/:orgId -func DeleteOrgByID(c *models.ReqContext) response.Response { - orgID := c.ParamsInt64(":orgId") +func (hs *HTTPServer) DeleteOrgByID(c *models.ReqContext) response.Response { + orgID, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } // before deleting an org, check if user does not belong to the current org if c.OrgId == orgID { return response.Error(400, "Can not delete org for current user", nil) } - if err := sqlstore.DeleteOrg(c.Req.Context(), &models.DeleteOrgCommand{Id: orgID}); err != nil { + if err := hs.SQLStore.DeleteOrg(c.Req.Context(), &models.DeleteOrgCommand{Id: orgID}); err != nil { if errors.Is(err, models.ErrOrgNotFound) { return response.Error(404, "Failed to delete organization. ID not found", nil) } diff --git a/pkg/api/org_invite.go b/pkg/api/org_invite.go index 651af77e244..957053455dd 100644 --- a/pkg/api/org_invite.go +++ b/pkg/api/org_invite.go @@ -17,10 +17,10 @@ import ( "github.com/grafana/grafana/pkg/web" ) -func GetPendingOrgInvites(c *models.ReqContext) response.Response { +func (hs *HTTPServer) GetPendingOrgInvites(c *models.ReqContext) response.Response { query := models.GetTempUsersQuery{OrgId: c.OrgId, Status: models.TmpUserInvitePending} - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + if err := hs.SQLStore.GetTempUsersQuery(c.Req.Context(), &query); err != nil { return response.Error(500, "Failed to get invites from db", err) } @@ -31,7 +31,7 @@ func GetPendingOrgInvites(c *models.ReqContext) response.Response { return response.JSON(200, query.Result) } -func AddOrgInvite(c *models.ReqContext) response.Response { +func (hs *HTTPServer) AddOrgInvite(c *models.ReqContext) response.Response { inviteDto := dtos.AddInviteForm{} if err := web.Bind(c.Req, &inviteDto); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) @@ -42,12 +42,12 @@ func AddOrgInvite(c *models.ReqContext) response.Response { // first try get existing user userQuery := models.GetUserByLoginQuery{LoginOrEmail: inviteDto.LoginOrEmail} - if err := bus.Dispatch(c.Req.Context(), &userQuery); err != nil { + if err := hs.SQLStore.GetUserByLogin(c.Req.Context(), &userQuery); err != nil { if !errors.Is(err, models.ErrUserNotFound) { return response.Error(500, "Failed to query db for existing user check", err) } } else { - return inviteExistingUserToOrg(c, userQuery.Result, &inviteDto) + return hs.inviteExistingUserToOrg(c, userQuery.Result, &inviteDto) } if setting.DisableLoginForm { @@ -68,7 +68,7 @@ func AddOrgInvite(c *models.ReqContext) response.Response { cmd.Role = inviteDto.Role cmd.RemoteAddr = c.Req.RemoteAddr - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.CreateTempUser(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to save invite to database", err) } @@ -86,7 +86,7 @@ func AddOrgInvite(c *models.ReqContext) response.Response { }, } - if err := bus.Dispatch(c.Req.Context(), &emailCmd); err != nil { + if err := hs.AlertNG.NotificationService.SendEmailCommandHandler(c.Req.Context(), &emailCmd); err != nil { if errors.Is(err, models.ErrSmtpNotEnabled) { return response.Error(412, err.Error(), err) } @@ -95,7 +95,7 @@ func AddOrgInvite(c *models.ReqContext) response.Response { } emailSentCmd := models.UpdateTempUserWithEmailSentCommand{Code: cmd.Result.Code} - if err := bus.Dispatch(c.Req.Context(), &emailSentCmd); err != nil { + if err := hs.SQLStore.UpdateTempUserWithEmailSent(c.Req.Context(), &emailSentCmd); err != nil { return response.Error(500, "Failed to update invite with email sent info", err) } @@ -105,10 +105,10 @@ func AddOrgInvite(c *models.ReqContext) response.Response { return response.Success(fmt.Sprintf("Created invite for %s", inviteDto.LoginOrEmail)) } -func inviteExistingUserToOrg(c *models.ReqContext, user *models.User, inviteDto *dtos.AddInviteForm) response.Response { +func (hs *HTTPServer) inviteExistingUserToOrg(c *models.ReqContext, user *models.User, inviteDto *dtos.AddInviteForm) response.Response { // user exists, add org role createOrgUserCmd := models.AddOrgUserCommand{OrgId: c.OrgId, UserId: user.Id, Role: inviteDto.Role} - if err := bus.Dispatch(c.Req.Context(), &createOrgUserCmd); err != nil { + if err := hs.SQLStore.AddOrgUser(c.Req.Context(), &createOrgUserCmd); err != nil { if errors.Is(err, models.ErrOrgUserAlreadyAdded) { return response.Error(412, fmt.Sprintf("User %s is already added to organization", inviteDto.LoginOrEmail), err) } @@ -126,7 +126,7 @@ func inviteExistingUserToOrg(c *models.ReqContext, user *models.User, inviteDto }, } - if err := bus.Dispatch(c.Req.Context(), &emailCmd); err != nil { + if err := hs.AlertNG.NotificationService.SendEmailCommandHandler(c.Req.Context(), &emailCmd); err != nil { return response.Error(500, "Failed to send email invited_to_org", err) } } @@ -137,8 +137,8 @@ func inviteExistingUserToOrg(c *models.ReqContext, user *models.User, inviteDto }) } -func RevokeInvite(c *models.ReqContext) response.Response { - if ok, rsp := updateTempUserStatus(c.Req.Context(), web.Params(c.Req)[":code"], models.TmpUserRevoked); !ok { +func (hs *HTTPServer) RevokeInvite(c *models.ReqContext) response.Response { + if ok, rsp := hs.updateTempUserStatus(c.Req.Context(), web.Params(c.Req)[":code"], models.TmpUserRevoked); !ok { return rsp } @@ -148,9 +148,9 @@ func RevokeInvite(c *models.ReqContext) response.Response { // GetInviteInfoByCode gets a pending user invite corresponding to a certain code. // A response containing an InviteInfo object is returned if the invite is found. // If a (pending) invite is not found, 404 is returned. -func GetInviteInfoByCode(c *models.ReqContext) response.Response { +func (hs *HTTPServer) GetInviteInfoByCode(c *models.ReqContext) response.Response { query := models.GetTempUserByCodeQuery{Code: web.Params(c.Req)[":code"]} - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + if err := hs.SQLStore.GetTempUserByCode(c.Req.Context(), &query); err != nil { if errors.Is(err, models.ErrTempUserNotFound) { return response.Error(404, "Invite not found", nil) } @@ -177,7 +177,7 @@ func (hs *HTTPServer) CompleteInvite(c *models.ReqContext) response.Response { } query := models.GetTempUserByCodeQuery{Code: completeInvite.InviteCode} - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + if err := hs.SQLStore.GetTempUserByCode(c.Req.Context(), &query); err != nil { if errors.Is(err, models.ErrTempUserNotFound) { return response.Error(404, "Invite not found", nil) } @@ -213,7 +213,7 @@ func (hs *HTTPServer) CompleteInvite(c *models.ReqContext) response.Response { return response.Error(500, "failed to publish event", err) } - if ok, rsp := applyUserInvite(c.Req.Context(), user, invite, true); !ok { + if ok, rsp := hs.applyUserInvite(c.Req.Context(), user, invite, true); !ok { return rsp } @@ -231,33 +231,33 @@ func (hs *HTTPServer) CompleteInvite(c *models.ReqContext) response.Response { }) } -func updateTempUserStatus(ctx context.Context, code string, status models.TempUserStatus) (bool, response.Response) { +func (hs *HTTPServer) updateTempUserStatus(ctx context.Context, code string, status models.TempUserStatus) (bool, response.Response) { // update temp user status updateTmpUserCmd := models.UpdateTempUserStatusCommand{Code: code, Status: status} - if err := bus.Dispatch(ctx, &updateTmpUserCmd); err != nil { + if err := hs.SQLStore.UpdateTempUserStatus(ctx, &updateTmpUserCmd); err != nil { return false, response.Error(500, "Failed to update invite status", err) } return true, nil } -func applyUserInvite(ctx context.Context, user *models.User, invite *models.TempUserDTO, setActive bool) (bool, response.Response) { +func (hs *HTTPServer) applyUserInvite(ctx context.Context, user *models.User, invite *models.TempUserDTO, setActive bool) (bool, response.Response) { // add to org addOrgUserCmd := models.AddOrgUserCommand{OrgId: invite.OrgId, UserId: user.Id, Role: invite.Role} - if err := bus.Dispatch(ctx, &addOrgUserCmd); err != nil { + if err := hs.SQLStore.AddOrgUser(ctx, &addOrgUserCmd); err != nil { if !errors.Is(err, models.ErrOrgUserAlreadyAdded) { return false, response.Error(500, "Error while trying to create org user", err) } } // update temp user status - if ok, rsp := updateTempUserStatus(ctx, invite.Code, models.TmpUserCompleted); !ok { + if ok, rsp := hs.updateTempUserStatus(ctx, invite.Code, models.TmpUserCompleted); !ok { return false, rsp } if setActive { // set org to active - if err := bus.Dispatch(ctx, &models.SetUsingOrgCommand{OrgId: invite.OrgId, UserId: user.Id}); err != nil { + if err := hs.SQLStore.SetUsingOrg(ctx, &models.SetUsingOrgCommand{OrgId: invite.OrgId, UserId: user.Id}); err != nil { return false, response.Error(500, "Failed to set org as active", err) } } diff --git a/pkg/api/org_users.go b/pkg/api/org_users.go index 7cd32b7bba5..8bbd2a1eba4 100644 --- a/pkg/api/org_users.go +++ b/pkg/api/org_users.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" @@ -30,7 +31,12 @@ func (hs *HTTPServer) AddOrgUser(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - cmd.OrgId = c.ParamsInt64(":orgId") + + var err error + cmd.OrgId, err = strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } return hs.addOrgUserHelper(c.Req.Context(), cmd) } @@ -71,6 +77,7 @@ func (hs *HTTPServer) GetOrgUsersForCurrentOrg(c *models.ReqContext) response.Re OrgId: c.OrgId, Query: c.Query("query"), Limit: c.QueryInt("limit"), + User: c.SignedInUser, }, c.SignedInUser) if err != nil { @@ -86,6 +93,7 @@ func (hs *HTTPServer) GetOrgUsersForCurrentOrgLookup(c *models.ReqContext) respo OrgId: c.OrgId, Query: c.Query("query"), Limit: c.QueryInt("limit"), + User: c.SignedInUser, }, c.SignedInUser) if err != nil { @@ -120,10 +128,16 @@ func (hs *HTTPServer) getUserAccessControlMetadata(c *models.ReqContext, resourc // GET /api/orgs/:orgId/users func (hs *HTTPServer) GetOrgUsers(c *models.ReqContext) response.Response { + orgId, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } + result, err := hs.getOrgUsersHelper(c, &models.GetOrgUsersQuery{ - OrgId: c.ParamsInt64(":orgId"), + OrgId: orgId, Query: "", Limit: 0, + User: c.SignedInUser, }, c.SignedInUser) if err != nil { @@ -183,8 +197,9 @@ func (hs *HTTPServer) SearchOrgUsersWithPaging(c *models.ReqContext) response.Re query := &models.SearchOrgUsersQuery{ OrgID: c.OrgId, Query: c.Query("query"), - Limit: perPage, Page: page, + Limit: perPage, + User: c.SignedInUser, } if err := hs.SQLStore.SearchOrgUsers(ctx, query); err != nil { @@ -215,18 +230,29 @@ func (hs *HTTPServer) UpdateOrgUserForCurrentOrg(c *models.ReqContext) response. return response.Error(http.StatusBadRequest, "bad request data", err) } cmd.OrgId = c.OrgId - cmd.UserId = c.ParamsInt64(":userId") + var err error + cmd.UserId, err = strconv.ParseInt(web.Params(c.Req)[":userId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "userId is invalid", err) + } return hs.updateOrgUserHelper(c.Req.Context(), cmd) } // PATCH /api/orgs/:orgId/users/:userId func (hs *HTTPServer) UpdateOrgUser(c *models.ReqContext) response.Response { cmd := models.UpdateOrgUserCommand{} + var err error if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - cmd.OrgId = c.ParamsInt64(":orgId") - cmd.UserId = c.ParamsInt64(":userId") + cmd.OrgId, err = strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } + cmd.UserId, err = strconv.ParseInt(web.Params(c.Req)[":userId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "userId is invalid", err) + } return hs.updateOrgUserHelper(c.Req.Context(), cmd) } @@ -246,8 +272,13 @@ func (hs *HTTPServer) updateOrgUserHelper(ctx context.Context, cmd models.Update // DELETE /api/org/users/:userId func (hs *HTTPServer) RemoveOrgUserForCurrentOrg(c *models.ReqContext) response.Response { + userId, err := strconv.ParseInt(web.Params(c.Req)[":userId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "userId is invalid", err) + } + return hs.removeOrgUserHelper(c.Req.Context(), &models.RemoveOrgUserCommand{ - UserId: c.ParamsInt64(":userId"), + UserId: userId, OrgId: c.OrgId, ShouldDeleteOrphanedUser: true, }) @@ -255,9 +286,17 @@ func (hs *HTTPServer) RemoveOrgUserForCurrentOrg(c *models.ReqContext) response. // DELETE /api/orgs/:orgId/users/:userId func (hs *HTTPServer) RemoveOrgUser(c *models.ReqContext) response.Response { + userId, err := strconv.ParseInt(web.Params(c.Req)[":userId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "userId is invalid", err) + } + orgId, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } return hs.removeOrgUserHelper(c.Req.Context(), &models.RemoveOrgUserCommand{ - UserId: c.ParamsInt64(":userId"), - OrgId: c.ParamsInt64(":orgId"), + UserId: userId, + OrgId: orgId, }) } diff --git a/pkg/api/org_users_test.go b/pkg/api/org_users_test.go index af135129a78..009e00d18a4 100644 --- a/pkg/api/org_users_test.go +++ b/pkg/api/org_users_test.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -34,13 +35,14 @@ func setUpGetOrgUsersDB(t *testing.T, sqlStore *sqlstore.SQLStore) { } func TestOrgUsersAPIEndpoint_userLoggedIn(t *testing.T) { - settings := setting.NewCfg() - hs := &HTTPServer{Cfg: settings} + hs := setupSimpleHTTPServer(featuremgmt.WithFeatures()) + settings := hs.Cfg sqlStore := sqlstore.InitTestDB(t) + sqlStore.Cfg = settings hs.SQLStore = sqlStore - loggedInUserScenario(t, "When calling GET on", "api/org/users", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "api/org/users", "api/org/users", func(sc *scenarioContext) { setUpGetOrgUsersDB(t, sqlStore) sc.handlerFunc = hs.GetOrgUsersForCurrentOrg @@ -54,7 +56,7 @@ func TestOrgUsersAPIEndpoint_userLoggedIn(t *testing.T) { assert.Len(t, resp, 3) }) - loggedInUserScenario(t, "When calling GET on", "api/org/users/search", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "api/org/users/search", "api/org/users/search", func(sc *scenarioContext) { setUpGetOrgUsersDB(t, sqlStore) sc.handlerFunc = hs.SearchOrgUsersWithPaging @@ -72,7 +74,7 @@ func TestOrgUsersAPIEndpoint_userLoggedIn(t *testing.T) { assert.Equal(t, 1, resp.Page) }) - loggedInUserScenario(t, "When calling GET with page and limit query parameters on", "api/org/users/search", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET with page and limit query parameters on", "api/org/users/search", "api/org/users/search", func(sc *scenarioContext) { setUpGetOrgUsersDB(t, sqlStore) sc.handlerFunc = hs.SearchOrgUsersWithPaging @@ -97,7 +99,7 @@ func TestOrgUsersAPIEndpoint_userLoggedIn(t *testing.T) { } t.Cleanup(func() { settings.HiddenUsers = make(map[string]struct{}) }) - loggedInUserScenario(t, "When calling GET on", "api/org/users", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "api/org/users", "api/org/users", func(sc *scenarioContext) { setUpGetOrgUsersDB(t, sqlStore) sc.handlerFunc = hs.GetOrgUsersForCurrentOrg @@ -556,7 +558,10 @@ func TestPostOrgUsersAPIEndpoint_AccessControl(t *testing.T) { require.NoError(t, err) assert.EqualValuesf(t, tc.expectedMessage, message, "server did not answer expected message") - getUsersQuery := models.GetOrgUsersQuery{OrgId: tc.targetOrg} + getUsersQuery := models.GetOrgUsersQuery{OrgId: tc.targetOrg, User: &models.SignedInUser{ + OrgId: tc.targetOrg, + Permissions: map[int64]map[string][]string{tc.targetOrg: {"org.users:read": {"users:*"}}}, + }} err = sc.db.GetOrgUsers(context.Background(), &getUsersQuery) require.NoError(t, err) assert.Len(t, getUsersQuery.Result, tc.expectedUserCount) @@ -686,7 +691,7 @@ func TestPatchOrgUsersAPIEndpoint_AccessControl(t *testing.T) { UserId: tc.targetUserId, OrgId: tc.targetOrg, } - err = sqlstore.GetSignedInUser(context.Background(), &getUserQuery) + err = sc.db.GetSignedInUser(context.Background(), &getUserQuery) require.NoError(t, err) assert.Equal(t, tc.expectedUserRole, getUserQuery.Result.OrgRole) } @@ -799,7 +804,13 @@ func TestDeleteOrgUsersAPIEndpoint_AccessControl(t *testing.T) { require.NoError(t, err) assert.Equal(t, tc.expectedMessage, message) - getUsersQuery := models.GetOrgUsersQuery{OrgId: tc.targetOrg} + getUsersQuery := models.GetOrgUsersQuery{ + OrgId: tc.targetOrg, + User: &models.SignedInUser{ + OrgId: tc.targetOrg, + Permissions: map[int64]map[string][]string{tc.targetOrg: {accesscontrol.ActionOrgUsersRead: {accesscontrol.ScopeUsersAll}}}, + }, + } err = sc.db.GetOrgUsers(context.Background(), &getUsersQuery) require.NoError(t, err) assert.Len(t, getUsersQuery.Result, tc.expectedUserCount) diff --git a/pkg/api/playlist.go b/pkg/api/playlist.go index 4557ac2d159..5e496c0fd0e 100644 --- a/pkg/api/playlist.go +++ b/pkg/api/playlist.go @@ -3,17 +3,21 @@ package api import ( "context" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/web" ) -func ValidateOrgPlaylist(c *models.ReqContext) { - id := c.ParamsInt64(":id") +func (hs *HTTPServer) ValidateOrgPlaylist(c *models.ReqContext) { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + c.JsonApiErr(http.StatusBadRequest, "id is invalid", nil) + return + } query := models.GetPlaylistByIdQuery{Id: id} - err := bus.Dispatch(c.Req.Context(), &query) + err = hs.SQLStore.GetPlaylist(c.Req.Context(), &query) if err != nil { c.JsonApiErr(404, "Playlist not found", err) @@ -31,7 +35,7 @@ func ValidateOrgPlaylist(c *models.ReqContext) { } } -func SearchPlaylists(c *models.ReqContext) response.Response { +func (hs *HTTPServer) SearchPlaylists(c *models.ReqContext) response.Response { query := c.Query("query") limit := c.QueryInt("limit") @@ -45,7 +49,7 @@ func SearchPlaylists(c *models.ReqContext) response.Response { OrgId: c.OrgId, } - err := bus.Dispatch(c.Req.Context(), &searchQuery) + err := hs.SQLStore.SearchPlaylists(c.Req.Context(), &searchQuery) if err != nil { return response.Error(500, "Search failed", err) } @@ -53,15 +57,18 @@ func SearchPlaylists(c *models.ReqContext) response.Response { return response.JSON(200, searchQuery.Result) } -func GetPlaylist(c *models.ReqContext) response.Response { - id := c.ParamsInt64(":id") +func (hs *HTTPServer) GetPlaylist(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } cmd := models.GetPlaylistByIdQuery{Id: id} - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.GetPlaylist(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Playlist not found", err) } - playlistDTOs, _ := LoadPlaylistItemDTOs(c.Req.Context(), id) + playlistDTOs, _ := hs.LoadPlaylistItemDTOs(c.Req.Context(), id) dto := &models.PlaylistDTO{ Id: cmd.Result.Id, @@ -74,8 +81,8 @@ func GetPlaylist(c *models.ReqContext) response.Response { return response.JSON(200, dto) } -func LoadPlaylistItemDTOs(ctx context.Context, id int64) ([]models.PlaylistItemDTO, error) { - playlistitems, err := LoadPlaylistItems(ctx, id) +func (hs *HTTPServer) LoadPlaylistItemDTOs(ctx context.Context, id int64) ([]models.PlaylistItemDTO, error) { + playlistitems, err := hs.LoadPlaylistItems(ctx, id) if err != nil { return nil, err @@ -97,19 +104,22 @@ func LoadPlaylistItemDTOs(ctx context.Context, id int64) ([]models.PlaylistItemD return playlistDTOs, nil } -func LoadPlaylistItems(ctx context.Context, id int64) ([]models.PlaylistItem, error) { +func (hs *HTTPServer) LoadPlaylistItems(ctx context.Context, id int64) ([]models.PlaylistItem, error) { itemQuery := models.GetPlaylistItemsByIdQuery{PlaylistId: id} - if err := bus.Dispatch(ctx, &itemQuery); err != nil { + if err := hs.SQLStore.GetPlaylistItem(ctx, &itemQuery); err != nil { return nil, err } return *itemQuery.Result, nil } -func GetPlaylistItems(c *models.ReqContext) response.Response { - id := c.ParamsInt64(":id") +func (hs *HTTPServer) GetPlaylistItems(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } - playlistDTOs, err := LoadPlaylistItemDTOs(c.Req.Context(), id) + playlistDTOs, err := hs.LoadPlaylistItemDTOs(c.Req.Context(), id) if err != nil { return response.Error(500, "Could not load playlist items", err) @@ -118,10 +128,13 @@ func GetPlaylistItems(c *models.ReqContext) response.Response { return response.JSON(200, playlistDTOs) } -func GetPlaylistDashboards(c *models.ReqContext) response.Response { - playlistID := c.ParamsInt64(":id") +func (hs *HTTPServer) GetPlaylistDashboards(c *models.ReqContext) response.Response { + playlistID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } - playlists, err := LoadPlaylistDashboards(c.Req.Context(), c.OrgId, c.SignedInUser, playlistID) + playlists, err := hs.LoadPlaylistDashboards(c.Req.Context(), c.OrgId, c.SignedInUser, playlistID) if err != nil { return response.Error(500, "Could not load dashboards", err) } @@ -129,44 +142,51 @@ func GetPlaylistDashboards(c *models.ReqContext) response.Response { return response.JSON(200, playlists) } -func DeletePlaylist(c *models.ReqContext) response.Response { - id := c.ParamsInt64(":id") +func (hs *HTTPServer) DeletePlaylist(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } cmd := models.DeletePlaylistCommand{Id: id, OrgId: c.OrgId} - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.DeletePlaylist(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to delete playlist", err) } return response.JSON(200, "") } -func CreatePlaylist(c *models.ReqContext) response.Response { +func (hs *HTTPServer) CreatePlaylist(c *models.ReqContext) response.Response { cmd := models.CreatePlaylistCommand{} if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } cmd.OrgId = c.OrgId - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.CreatePlaylist(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to create playlist", err) } return response.JSON(200, cmd.Result) } -func UpdatePlaylist(c *models.ReqContext) response.Response { +func (hs *HTTPServer) UpdatePlaylist(c *models.ReqContext) response.Response { cmd := models.UpdatePlaylistCommand{} if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } cmd.OrgId = c.OrgId - cmd.Id = c.ParamsInt64(":id") + var err error + cmd.Id, err = strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.UpdatePlaylist(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to save playlist", err) } - playlistDTOs, err := LoadPlaylistItemDTOs(c.Req.Context(), cmd.Id) + playlistDTOs, err := hs.LoadPlaylistItemDTOs(c.Req.Context(), cmd.Id) if err != nil { return response.Error(500, "Failed to save playlist", err) } diff --git a/pkg/api/playlist_play.go b/pkg/api/playlist_play.go index c7ea1737444..efd07dfab6e 100644 --- a/pkg/api/playlist_play.go +++ b/pkg/api/playlist_play.go @@ -6,18 +6,17 @@ import ( "strconv" "github.com/grafana/grafana/pkg/api/dtos" - "github.com/grafana/grafana/pkg/bus" _ "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/search" ) -func populateDashboardsByID(ctx context.Context, dashboardByIDs []int64, dashboardIDOrder map[int64]int) (dtos.PlaylistDashboardsSlice, error) { +func (hs *HTTPServer) populateDashboardsByID(ctx context.Context, dashboardByIDs []int64, dashboardIDOrder map[int64]int) (dtos.PlaylistDashboardsSlice, error) { result := make(dtos.PlaylistDashboardsSlice, 0) if len(dashboardByIDs) > 0 { dashboardQuery := models.GetDashboardsQuery{DashboardIds: dashboardByIDs} - if err := bus.Dispatch(ctx, &dashboardQuery); err != nil { + if err := hs.SQLStore.GetDashboards(ctx, &dashboardQuery); err != nil { return result, err } @@ -36,7 +35,7 @@ func populateDashboardsByID(ctx context.Context, dashboardByIDs []int64, dashboa return result, nil } -func populateDashboardsByTag(ctx context.Context, orgID int64, signedInUser *models.SignedInUser, dashboardByTag []string, dashboardTagOrder map[string]int) dtos.PlaylistDashboardsSlice { +func (hs *HTTPServer) populateDashboardsByTag(ctx context.Context, orgID int64, signedInUser *models.SignedInUser, dashboardByTag []string, dashboardTagOrder map[string]int) dtos.PlaylistDashboardsSlice { result := make(dtos.PlaylistDashboardsSlice, 0) for _, tag := range dashboardByTag { @@ -49,7 +48,7 @@ func populateDashboardsByTag(ctx context.Context, orgID int64, signedInUser *mod OrgId: orgID, } - if err := bus.Dispatch(ctx, &searchQuery); err == nil { + if err := hs.SearchService.SearchHandler(ctx, &searchQuery); err == nil { for _, item := range searchQuery.Result { result = append(result, dtos.PlaylistDashboard{ Id: item.ID, @@ -66,8 +65,8 @@ func populateDashboardsByTag(ctx context.Context, orgID int64, signedInUser *mod return result } -func LoadPlaylistDashboards(ctx context.Context, orgID int64, signedInUser *models.SignedInUser, playlistID int64) (dtos.PlaylistDashboardsSlice, error) { - playlistItems, _ := LoadPlaylistItems(ctx, playlistID) +func (hs *HTTPServer) LoadPlaylistDashboards(ctx context.Context, orgID int64, signedInUser *models.SignedInUser, playlistID int64) (dtos.PlaylistDashboardsSlice, error) { + playlistItems, _ := hs.LoadPlaylistItems(ctx, playlistID) dashboardByIDs := make([]int64, 0) dashboardByTag := make([]string, 0) @@ -89,9 +88,9 @@ func LoadPlaylistDashboards(ctx context.Context, orgID int64, signedInUser *mode result := make(dtos.PlaylistDashboardsSlice, 0) - var k, _ = populateDashboardsByID(ctx, dashboardByIDs, dashboardIDOrder) + var k, _ = hs.populateDashboardsByID(ctx, dashboardByIDs, dashboardIDOrder) result = append(result, k...) - result = append(result, populateDashboardsByTag(ctx, orgID, signedInUser, dashboardByTag, dashboardTagOrder)...) + result = append(result, hs.populateDashboardsByTag(ctx, orgID, signedInUser, dashboardByTag, dashboardTagOrder)...) sort.Sort(result) return result, nil diff --git a/pkg/api/pluginproxy/ds_proxy.go b/pkg/api/pluginproxy/ds_proxy.go index deb1409ef36..cb9564fa183 100644 --- a/pkg/api/pluginproxy/ds_proxy.go +++ b/pkg/api/pluginproxy/ds_proxy.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/api/datasource" "github.com/grafana/grafana/pkg/infra/httpclient" glog "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/datasources" @@ -23,7 +24,7 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util/proxyutil" - "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" ) var ( @@ -42,6 +43,7 @@ type DataSourceProxy struct { clientProvider httpclient.Provider oAuthTokenService oauthtoken.OAuthTokenService dataSourcesService *datasources.Service + tracer tracing.Tracer } type handleResponseTransport struct { @@ -75,7 +77,8 @@ func (lw *logWrapper) Write(p []byte) (n int, err error) { // NewDataSourceProxy creates a new Datasource proxy func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext, proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider, - oAuthTokenService oauthtoken.OAuthTokenService, dsService *datasources.Service) (*DataSourceProxy, error) { + oAuthTokenService oauthtoken.OAuthTokenService, dsService *datasources.Service, + tracer tracing.Tracer) (*DataSourceProxy, error) { targetURL, err := datasource.ValidateURL(ds.Type, ds.Url) if err != nil { return nil, err @@ -91,6 +94,7 @@ func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ct clientProvider: clientProvider, oAuthTokenService: oAuthTokenService, dataSourcesService: dsService, + tracer: tracer, }, nil } @@ -147,35 +151,29 @@ func (proxy *DataSourceProxy) HandleRequest() { } proxy.logRequest() - - span, ctx := opentracing.StartSpanFromContext(proxy.ctx.Req.Context(), "datasource reverse proxy") - defer span.Finish() + ctx, span := proxy.tracer.Start(proxy.ctx.Req.Context(), "datasource reverse proxy") + defer span.End() proxy.ctx.Req = proxy.ctx.Req.WithContext(ctx) - span.SetTag("datasource_name", proxy.ds.Name) - span.SetTag("datasource_type", proxy.ds.Type) - span.SetTag("user", proxy.ctx.SignedInUser.Login) - span.SetTag("org_id", proxy.ctx.SignedInUser.OrgId) + span.SetAttributes("datasource_name", proxy.ds.Name, attribute.Key("datasource_name").String(proxy.ds.Name)) + span.SetAttributes("datasource_type", proxy.ds.Type, attribute.Key("datasource_type").String(proxy.ds.Type)) + span.SetAttributes("user", proxy.ctx.SignedInUser.Login, attribute.Key("user").String(proxy.ctx.SignedInUser.Login)) + span.SetAttributes("org_id", proxy.ctx.SignedInUser.OrgId, attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgId)) proxy.addTraceFromHeaderValue(span, "X-Panel-Id", "panel_id") proxy.addTraceFromHeaderValue(span, "X-Dashboard-Id", "dashboard_id") - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(proxy.ctx.Req.Header)); err != nil { - logger.Error("Failed to inject span context instance", "err", err) - } + proxy.tracer.Inject(ctx, proxy.ctx.Req.Header, span) reverseProxy.ServeHTTP(proxy.ctx.Resp, proxy.ctx.Req) } -func (proxy *DataSourceProxy) addTraceFromHeaderValue(span opentracing.Span, headerName string, tagName string) { +func (proxy *DataSourceProxy) addTraceFromHeaderValue(span tracing.Span, headerName string, tagName string) { panelId := proxy.ctx.Req.Header.Get(headerName) dashId, err := strconv.Atoi(panelId) if err == nil { - span.SetTag(tagName, dashId) + span.SetAttributes(tagName, dashId, attribute.Key(tagName).Int(dashId)) } } diff --git a/pkg/api/pluginproxy/ds_proxy_test.go b/pkg/api/pluginproxy/ds_proxy_test.go index 8a99da692f0..729aa365306 100644 --- a/pkg/api/pluginproxy/ds_proxy_test.go +++ b/pkg/api/pluginproxy/ds_proxy_test.go @@ -16,8 +16,10 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/secrets" @@ -32,6 +34,8 @@ import ( func TestDataSourceProxy_routeRule(t *testing.T) { httpClientProvider := httpclient.NewProvider() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) t.Run("Plugin with routes", func(t *testing.T) { routes := []*plugins.Route{ @@ -125,9 +129,9 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path", func(t *testing.T) { ctx, req := setUp() - dsService := datasources.ProvideService(bus.New(), nil, secretsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, - &oauthtoken.Service{}, dsService) + &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[0] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -138,8 +142,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic url", func(t *testing.T) { ctx, req := setUp() - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[3] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -150,8 +154,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path with no url", func(t *testing.T) { ctx, req := setUp() - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[4] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -161,8 +165,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic body", func(t *testing.T) { ctx, req := setUp() - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[5] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -175,8 +179,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("Validating request", func(t *testing.T) { t.Run("plugin route with valid role", func(t *testing.T) { ctx, _ := setUp() - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -184,8 +188,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is editor", func(t *testing.T) { ctx, _ := setUp() - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.Error(t, err) @@ -194,8 +198,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is admin", func(t *testing.T) { ctx, _ := setUp() ctx.SignedInUser.OrgRole = models.ROLE_ADMIN - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -285,8 +289,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }, } - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) @@ -301,8 +305,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { req, err := http.NewRequest("GET", "http://localhost/asd", nil) require.NoError(t, err) client = newFakeHTTPClient(t, json2) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[1], dsInfo, cfg) @@ -318,8 +322,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { require.NoError(t, err) client = newFakeHTTPClient(t, []byte{}) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) @@ -340,8 +344,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -366,8 +370,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -390,8 +394,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -418,8 +422,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var pluginRoutes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -441,8 +445,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req.Header.Set("Origin", "grafana.com") @@ -505,8 +509,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer) require.NoError(t, err) req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -560,18 +564,18 @@ func TestDataSourceProxy_routeRule(t *testing.T) { secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) tests := []*testCase{ - createAuthTest(t, secretsService, models.DS_INFLUXDB_08, authTypePassword, authCheckQuery, false), - createAuthTest(t, secretsService, models.DS_INFLUXDB_08, authTypePassword, authCheckQuery, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, authTypePassword, authCheckHeader, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, authTypePassword, authCheckHeader, false), - createAuthTest(t, secretsService, models.DS_INFLUXDB, authTypeBasic, authCheckHeader, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, authTypeBasic, authCheckHeader, false), + createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false), + createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true), + createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true), + createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false), + createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true), + createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false), // These two should be enough for any other datasource at the moment. Proxy has special handling // only for Influx, others have the same path and only BasicAuth. Non BasicAuth datasources // do not go through proxy but through TSDB API which is not tested here. - createAuthTest(t, secretsService, models.DS_ES, authTypeBasic, authCheckHeader, false), - createAuthTest(t, secretsService, models.DS_ES, authTypeBasic, authCheckHeader, true), + createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false), + createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true), } for _, test := range tests { runDatasourceAuthTest(t, secretsService, test) @@ -631,12 +635,15 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }, ds } + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) { ctx, ds := setUp(t) var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -653,8 +660,8 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }) var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -675,8 +682,8 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }) var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -700,8 +707,8 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -710,6 +717,29 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { require.NotNil(t, req) require.Equal(t, "/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", req.RequestURI) }) + t.Run("Data source should handle proxy path url encoding correctly with opentelemetry", func(t *testing.T) { + var req *http.Request + ctx, ds := setUp(t, setUpCfg{ + writeCb: func(w http.ResponseWriter, r *http.Request) { + req = r + w.WriteHeader(200) + _, err := w.Write([]byte("OK")) + require.NoError(t, err) + }, + }) + + ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) + var routes []*plugins.Route + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) + require.NoError(t, err) + + proxy.HandleRequest() + require.NoError(t, writeErr) + require.NotNil(t, req) + require.Equal(t, "/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", req.RequestURI) + }) } func TestNewDataSourceProxy_InvalidURL(t *testing.T) { @@ -722,10 +752,12 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) { Url: "://host/root", } cfg := setting.Cfg{} + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - _, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", &cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", &cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.Error(t, err) assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`)) } @@ -740,11 +772,13 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) { Url: "127.0.01:5432", } cfg := setting.Cfg{} + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - _, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", &cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", &cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) } @@ -755,6 +789,9 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) { Context: &web.Context{}, SignedInUser: &models.SignedInUser{OrgRole: models.ROLE_EDITOR}, } + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + tcs := []struct { description string url string @@ -783,8 +820,8 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) { var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", &cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", &cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) if tc.err == nil { require.NoError(t, err) assert.Equal(t, &url.URL{ @@ -805,11 +842,13 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett Type: "custom", Url: "http://host/root/", } + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) var routes []*plugins.Route secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -861,7 +900,7 @@ const ( authCheckHeader = "header" ) -func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, authType string, authCheck string, useSecureJsonData bool) *testCase { +func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase { ctx := context.Background() // Basic user:password @@ -872,6 +911,7 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, Id: 1, Type: dsType, JsonData: simplejson.New(), + Url: url, }, } var message string @@ -927,9 +967,12 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, test *testCase) { ctx := &models.ReqContext{} + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + var routes []*plugins.Route - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -956,6 +999,8 @@ func Test_PathCheck(t *testing.T) { Method: http.MethodGet, }, } + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) setUp := func() (*models.ReqContext, *http.Request) { req, err := http.NewRequest("GET", "http://localhost/asd", nil) @@ -968,8 +1013,8 @@ func Test_PathCheck(t *testing.T) { } ctx, _ := setUp() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) - proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) + proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) require.Nil(t, proxy.validateRequest()) diff --git a/pkg/api/plugins.go b/pkg/api/plugins.go index ef2bc8827f4..431edfd16c5 100644 --- a/pkg/api/plugins.go +++ b/pkg/api/plugins.go @@ -74,8 +74,6 @@ func (hs *HTTPServer) GetPluginList(c *models.ReqContext) response.Response { Category: pluginDef.Category, Info: pluginDef.Info, Dependencies: pluginDef.Dependencies, - LatestVersion: pluginDef.GrafanaComVersion, - HasUpdate: pluginDef.GrafanaComHasUpdate, DefaultNavUrl: pluginDef.DefaultNavURL, State: pluginDef.State, Signature: pluginDef.Signature, @@ -83,6 +81,12 @@ func (hs *HTTPServer) GetPluginList(c *models.ReqContext) response.Response { SignatureOrg: pluginDef.SignatureOrg, } + update, exists := hs.pluginsUpdateChecker.HasUpdate(c.Req.Context(), pluginDef.ID) + if exists { + listItem.LatestVersion = update + listItem.HasUpdate = true + } + if pluginSetting, exists := pluginSettingsMap[pluginDef.ID]; exists { listItem.Enabled = pluginSetting.Enabled listItem.Pinned = pluginSetting.Pinned @@ -127,8 +131,6 @@ func (hs *HTTPServer) GetPluginSettingByID(c *models.ReqContext) response.Respon BaseUrl: plugin.BaseURL, Module: plugin.Module, DefaultNavUrl: plugin.DefaultNavURL, - LatestVersion: plugin.GrafanaComVersion, - HasUpdate: plugin.GrafanaComHasUpdate, State: plugin.State, Signature: plugin.Signature, SignatureType: plugin.SignatureType, @@ -151,6 +153,12 @@ func (hs *HTTPServer) GetPluginSettingByID(c *models.ReqContext) response.Respon dto.JsonData = query.Result.JsonData } + update, exists := hs.pluginsUpdateChecker.HasUpdate(c.Req.Context(), plugin.ID) + if exists { + dto.LatestVersion = update + dto.HasUpdate = true + } + return response.JSON(200, dto) } @@ -217,51 +225,6 @@ func (hs *HTTPServer) GetPluginMarkdown(c *models.ReqContext) response.Response return resp } -func (hs *HTTPServer) ImportDashboard(c *models.ReqContext) response.Response { - apiCmd := dtos.ImportDashboardCommand{} - if err := web.Bind(c.Req, &apiCmd); err != nil { - return response.Error(http.StatusBadRequest, "bad request data", err) - } - var err error - if apiCmd.PluginId == "" && apiCmd.Dashboard == nil { - return response.Error(422, "Dashboard must be set", nil) - } - - limitReached, err := hs.QuotaService.QuotaReached(c, "dashboard") - if err != nil { - return response.Error(500, "failed to get quota", err) - } - if limitReached { - return response.Error(403, "Quota reached", nil) - } - - trimDefaults := c.QueryBoolWithDefault("trimdefaults", true) - if trimDefaults && !hs.LoadSchemaService.IsDisabled() { - apiCmd.Dashboard, err = hs.LoadSchemaService.DashboardApplyDefaults(apiCmd.Dashboard) - if err != nil { - return response.Error(500, "Error while applying default value to the dashboard json", err) - } - } - - dashInfo, dash, err := hs.pluginDashboardManager.ImportDashboard(c.Req.Context(), apiCmd.PluginId, apiCmd.Path, c.OrgId, apiCmd.FolderId, - apiCmd.Dashboard, apiCmd.Overwrite, apiCmd.Inputs, c.SignedInUser) - if err != nil { - return hs.dashboardSaveErrorToApiResponse(c.Req.Context(), err) - } - - err = hs.LibraryPanelService.ImportLibraryPanelsForDashboard(c.Req.Context(), c.SignedInUser, dash, apiCmd.FolderId) - if err != nil { - return response.Error(500, "Error while importing library panels", err) - } - - err = hs.LibraryPanelService.ConnectLibraryPanelsForDashboard(c.Req.Context(), c.SignedInUser, dash) - if err != nil { - return response.Error(500, "Error while connecting library panels", err) - } - - return response.JSON(200, dashInfo) -} - // CollectPluginMetrics collect metrics from a plugin. // // /api/plugins/:pluginId/metrics diff --git a/pkg/api/quota.go b/pkg/api/quota.go index 485a72dbf44..fee44d6ed46 100644 --- a/pkg/api/quota.go +++ b/pkg/api/quota.go @@ -2,6 +2,7 @@ package api import ( "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/bus" @@ -15,7 +16,11 @@ func (hs *HTTPServer) GetCurrentOrgQuotas(c *models.ReqContext) response.Respons } func (hs *HTTPServer) GetOrgQuotas(c *models.ReqContext) response.Response { - return hs.getOrgQuotasHelper(c, c.ParamsInt64(":orgId")) + orgId, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } + return hs.getOrgQuotasHelper(c, orgId) } func (hs *HTTPServer) getOrgQuotasHelper(c *models.ReqContext, orgID int64) response.Response { @@ -33,13 +38,17 @@ func (hs *HTTPServer) getOrgQuotasHelper(c *models.ReqContext, orgID int64) resp func (hs *HTTPServer) UpdateOrgQuota(c *models.ReqContext) response.Response { cmd := models.UpdateOrgQuotaCmd{} + var err error if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } if !hs.Cfg.Quota.Enabled { return response.Error(404, "Quotas not enabled", nil) } - cmd.OrgId = c.ParamsInt64(":orgId") + cmd.OrgId, err = strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } cmd.Target = web.Params(c.Req)[":target"] if _, ok := hs.Cfg.Quota.Org.ToMap()[cmd.Target]; !ok { @@ -56,7 +65,13 @@ func GetUserQuotas(c *models.ReqContext) response.Response { if !setting.Quota.Enabled { return response.Error(404, "Quotas not enabled", nil) } - query := models.GetUserQuotasQuery{UserId: c.ParamsInt64(":id")} + + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + + query := models.GetUserQuotasQuery{UserId: id} if err := bus.Dispatch(c.Req.Context(), &query); err != nil { return response.Error(500, "Failed to get org quotas", err) @@ -67,13 +82,17 @@ func GetUserQuotas(c *models.ReqContext) response.Response { func UpdateUserQuota(c *models.ReqContext) response.Response { cmd := models.UpdateUserQuotaCmd{} + var err error if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } if !setting.Quota.Enabled { return response.Error(404, "Quotas not enabled", nil) } - cmd.UserId = c.ParamsInt64(":id") + cmd.UserId, err = strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } cmd.Target = web.Params(c.Req)[":target"] if _, ok := setting.Quota.User.ToMap()[cmd.Target]; !ok { diff --git a/pkg/api/render.go b/pkg/api/render.go index 0293c2a4147..157c789713b 100644 --- a/pkg/api/render.go +++ b/pkg/api/render.go @@ -53,12 +53,16 @@ func (hs *HTTPServer) RenderToPng(c *models.ReqContext) { } result, err := hs.RenderService.Render(c.Req.Context(), rendering.Opts{ + TimeoutOpts: rendering.TimeoutOpts{ + Timeout: time.Duration(timeout) * time.Second, + }, + AuthOpts: rendering.AuthOpts{ + OrgID: c.OrgId, + UserID: c.UserId, + OrgRole: c.OrgRole, + }, Width: width, Height: height, - Timeout: time.Duration(timeout) * time.Second, - OrgID: c.OrgId, - UserID: c.UserId, - OrgRole: c.OrgRole, Path: web.Params(c.Req)["*"] + queryParams, Timezone: queryReader.Get("tz", ""), Encoding: queryReader.Get("encoding", ""), @@ -66,7 +70,7 @@ func (hs *HTTPServer) RenderToPng(c *models.ReqContext) { DeviceScaleFactor: scale, Headers: headers, Theme: rendering.ThemeDark, - }) + }, nil) if err != nil { if errors.Is(err, rendering.ErrTimeout) { c.Handle(hs.Cfg, 500, err.Error(), err) diff --git a/pkg/api/roles.go b/pkg/api/roles.go index 16f0d7c6cd6..943649a41d3 100644 --- a/pkg/api/roles.go +++ b/pkg/api/roles.go @@ -3,6 +3,7 @@ package api import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/setting" ) // API related actions @@ -24,8 +25,6 @@ const ( ActionOrgsQuotasWrite = "orgs.quotas:write" ActionOrgsDelete = "orgs:delete" ActionOrgsCreate = "orgs:create" - - ActionTeamsCreate = "teams:create" ) // API related scopes @@ -63,6 +62,26 @@ func (hs *HTTPServer) declareFixedRoles() error { Grants: []string{accesscontrol.RoleGrafanaAdmin}, } + datasourcesExplorerRole := accesscontrol.RoleRegistration{ + Role: accesscontrol.RoleDTO{ + Version: 4, + Name: "fixed:datasources:explorer", + DisplayName: "Data source explorer", + Description: "Enable the Explore feature. Data source permissions still apply; you can only query data sources for which you have query permissions.", + Group: "Data sources", + Permissions: []accesscontrol.Permission{ + { + Action: accesscontrol.ActionDatasourcesExplore, + }, + }, + }, + Grants: []string{string(models.ROLE_EDITOR)}, + } + + if setting.ViewersCanEdit { + datasourcesExplorerRole.Grants = append(datasourcesExplorerRole.Grants, string(models.ROLE_VIEWER)) + } + datasourcesReaderRole := accesscontrol.RoleRegistration{ Role: accesscontrol.RoleDTO{ Version: 3, @@ -134,6 +153,7 @@ func (hs *HTTPServer) declareFixedRoles() error { Group: "Infrequently used", Permissions: []accesscontrol.Permission{ {Action: ActionDatasourcesQuery}, + {Action: ActionDatasourcesRead}, }, }, Grants: []string{string(models.ROLE_VIEWER)}, @@ -187,29 +207,47 @@ func (hs *HTTPServer) declareFixedRoles() error { Grants: []string{string(accesscontrol.RoleGrafanaAdmin)}, } - teamWriterGrants := []string{string(models.ROLE_ADMIN)} + teamCreatorGrants := []string{string(models.ROLE_ADMIN)} if hs.Cfg.EditorsCanAdmin { - teamWriterGrants = append(teamWriterGrants, string(models.ROLE_EDITOR)) + teamCreatorGrants = append(teamCreatorGrants, string(models.ROLE_EDITOR)) } - teamsWriterRole := accesscontrol.RoleRegistration{ + teamsCreatorRole := accesscontrol.RoleRegistration{ Role: accesscontrol.RoleDTO{ - Name: "fixed:teams:writer", - DisplayName: "Team writer", + Name: "fixed:teams:creator", + DisplayName: "Team creator", Description: "Create teams.", Group: "Teams", Version: 1, Permissions: []accesscontrol.Permission{ - { - Action: ActionTeamsCreate, - }, + {Action: accesscontrol.ActionTeamsCreate}, }, }, - Grants: teamWriterGrants, + Grants: teamCreatorGrants, + } + + teamsWriterRole := accesscontrol.RoleRegistration{ + Role: accesscontrol.RoleDTO{ + Name: "fixed:teams:writer", + DisplayName: "Team writer", + Description: "Create, read, write, or delete a team as well as controlling team memberships.", + Group: "Teams", + Version: 1, + Permissions: []accesscontrol.Permission{ + {Action: accesscontrol.ActionTeamsCreate}, + {Action: accesscontrol.ActionTeamsDelete, Scope: accesscontrol.ScopeTeamsAll}, + {Action: accesscontrol.ActionTeamsPermissionsRead, Scope: accesscontrol.ScopeTeamsAll}, + {Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: accesscontrol.ScopeTeamsAll}, + {Action: accesscontrol.ActionTeamsRead, Scope: accesscontrol.ScopeTeamsAll}, + {Action: accesscontrol.ActionTeamsWrite, Scope: accesscontrol.ScopeTeamsAll}, + }, + }, + Grants: []string{string(models.ROLE_ADMIN)}, } return hs.AccessControl.DeclareFixedRoles( provisioningWriterRole, datasourcesReaderRole, datasourcesWriterRole, datasourcesIdReaderRole, - datasourcesCompatibilityReaderRole, orgReaderRole, orgWriterRole, orgMaintainerRole, teamsWriterRole, + datasourcesCompatibilityReaderRole, orgReaderRole, orgWriterRole, + orgMaintainerRole, teamsCreatorRole, teamsWriterRole, datasourcesExplorerRole, ) } @@ -218,7 +256,7 @@ func (hs *HTTPServer) declareFixedRoles() error { // dataSourcesConfigurationAccessEvaluator is used to protect the "Configure > Data sources" tab access var dataSourcesConfigurationAccessEvaluator = accesscontrol.EvalAll( - accesscontrol.EvalPermission(ActionDatasourcesRead, ScopeDatasourcesAll), + accesscontrol.EvalPermission(ActionDatasourcesRead), accesscontrol.EvalAny( accesscontrol.EvalPermission(ActionDatasourcesCreate), accesscontrol.EvalPermission(ActionDatasourcesDelete), @@ -228,14 +266,14 @@ var dataSourcesConfigurationAccessEvaluator = accesscontrol.EvalAll( // dataSourcesNewAccessEvaluator is used to protect the "Configure > Data sources > New" page access var dataSourcesNewAccessEvaluator = accesscontrol.EvalAll( - accesscontrol.EvalPermission(ActionDatasourcesRead, ScopeDatasourcesAll), + accesscontrol.EvalPermission(ActionDatasourcesRead), accesscontrol.EvalPermission(ActionDatasourcesCreate), accesscontrol.EvalPermission(ActionDatasourcesWrite), ) // dataSourcesEditAccessEvaluator is used to protect the "Configure > Data sources > Edit" page access var dataSourcesEditAccessEvaluator = accesscontrol.EvalAll( - accesscontrol.EvalPermission(ActionDatasourcesRead, ScopeDatasourcesAll), + accesscontrol.EvalPermission(ActionDatasourcesRead), accesscontrol.EvalPermission(ActionDatasourcesWrite), ) diff --git a/pkg/api/routing/route_register.go b/pkg/api/routing/route_register.go index 7ae07933250..4c2a59eee25 100644 --- a/pkg/api/routing/route_register.go +++ b/pkg/api/routing/route_register.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/grafana/grafana/pkg/middleware" - "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/web" ) @@ -44,7 +44,7 @@ type RouteRegister interface { // Register iterates over all routes added to the RouteRegister // and add them to the `Router` pass as an parameter. - Register(Router) + Register(Router, ...RegisterNamedMiddleware) // Reset resets the route register. Reset() @@ -52,8 +52,8 @@ type RouteRegister interface { type RegisterNamedMiddleware func(name string) web.Handler -func ProvideRegister(cfg *setting.Cfg) *RouteRegisterImpl { - return NewRouteRegister(middleware.ProvideRouteOperationName, middleware.RequestMetrics(cfg)) +func ProvideRegister(features featuremgmt.FeatureToggles) *RouteRegisterImpl { + return NewRouteRegister(middleware.ProvideRouteOperationName, middleware.RequestMetrics(features)) } // NewRouteRegister creates a new RouteRegister with all middlewares sent as params @@ -118,8 +118,13 @@ func (rr *RouteRegisterImpl) Group(pattern string, fn func(rr RouteRegister), ha rr.groups = append(rr.groups, group) } -func (rr *RouteRegisterImpl) Register(router Router) { +func (rr *RouteRegisterImpl) Register(router Router, namedMiddlewares ...RegisterNamedMiddleware) { for _, r := range rr.routes { + // Add global named middlewares + for i, m := range namedMiddlewares { + r.handlers = insertHandler(r.handlers, len(rr.namedMiddlewares)+i, m(r.pattern)) + } + // GET requests have to be added to macaron routing using Get() // Otherwise HEAD requests will not be allowed. // https://github.com/go-macaron/macaron/blob/a325110f8b392bce3e5cdeb8c44bf98078ada3be/router.go#L198 @@ -131,7 +136,7 @@ func (rr *RouteRegisterImpl) Register(router Router) { } for _, g := range rr.groups { - g.Register(router) + g.Register(router, namedMiddlewares...) } } @@ -159,6 +164,15 @@ func (rr *RouteRegisterImpl) route(pattern, method string, handlers ...web.Handl }) } +func insertHandler(a []web.Handler, index int, value web.Handler) []web.Handler { + if len(a) == index { + return append(a, value) + } + a = append(a[:index+1], a[index:]...) + a[index] = value + return a +} + func (rr *RouteRegisterImpl) Get(pattern string, handlers ...web.Handler) { rr.route(pattern, http.MethodGet, handlers...) } diff --git a/pkg/api/signup.go b/pkg/api/signup.go index b0de30901b2..f3568d03daf 100644 --- a/pkg/api/signup.go +++ b/pkg/api/signup.go @@ -25,7 +25,7 @@ func GetSignUpOptions(c *models.ReqContext) response.Response { } // POST /api/user/signup -func SignUp(c *models.ReqContext) response.Response { +func (hs *HTTPServer) SignUp(c *models.ReqContext) response.Response { form := dtos.SignUpForm{} if err := web.Bind(c.Req, &form); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) @@ -35,7 +35,7 @@ func SignUp(c *models.ReqContext) response.Response { } existing := models.GetUserByLoginQuery{LoginOrEmail: form.Email} - if err := bus.Dispatch(c.Req.Context(), &existing); err == nil { + if err := hs.SQLStore.GetUserByLogin(c.Req.Context(), &existing); err == nil { return response.Error(422, "User with same email address already exists", nil) } @@ -51,7 +51,7 @@ func SignUp(c *models.ReqContext) response.Response { } cmd.RemoteAddr = c.Req.RemoteAddr - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.CreateTempUser(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to create signup", err) } @@ -86,7 +86,7 @@ func (hs *HTTPServer) SignUpStep2(c *models.ReqContext) response.Response { // verify email if setting.VerifyEmailEnabled { - if ok, rsp := verifyUserSignUpEmail(c.Req.Context(), form.Email, form.Code); !ok { + if ok, rsp := hs.verifyUserSignUpEmail(c.Req.Context(), form.Email, form.Code); !ok { return rsp } createUserCmd.EmailVerified = true @@ -110,19 +110,19 @@ func (hs *HTTPServer) SignUpStep2(c *models.ReqContext) response.Response { } // mark temp user as completed - if ok, rsp := updateTempUserStatus(c.Req.Context(), form.Code, models.TmpUserCompleted); !ok { + if ok, rsp := hs.updateTempUserStatus(c.Req.Context(), form.Code, models.TmpUserCompleted); !ok { return rsp } // check for pending invites invitesQuery := models.GetTempUsersQuery{Email: form.Email, Status: models.TmpUserInvitePending} - if err := bus.Dispatch(c.Req.Context(), &invitesQuery); err != nil { + if err := hs.SQLStore.GetTempUsersQuery(c.Req.Context(), &invitesQuery); err != nil { return response.Error(500, "Failed to query database for invites", err) } apiResponse := util.DynMap{"message": "User sign up completed successfully", "code": "redirect-to-landing-page"} for _, invite := range invitesQuery.Result { - if ok, rsp := applyUserInvite(c.Req.Context(), user, invite, false); !ok { + if ok, rsp := hs.applyUserInvite(c.Req.Context(), user, invite, false); !ok { return rsp } apiResponse["code"] = "redirect-to-select-org" @@ -138,10 +138,10 @@ func (hs *HTTPServer) SignUpStep2(c *models.ReqContext) response.Response { return response.JSON(200, apiResponse) } -func verifyUserSignUpEmail(ctx context.Context, email string, code string) (bool, response.Response) { +func (hs *HTTPServer) verifyUserSignUpEmail(ctx context.Context, email string, code string) (bool, response.Response) { query := models.GetTempUserByCodeQuery{Code: code} - if err := bus.Dispatch(ctx, &query); err != nil { + if err := hs.SQLStore.GetTempUserByCode(ctx, &query); err != nil { if errors.Is(err, models.ErrTempUserNotFound) { return false, response.Error(404, "Invalid email verification code", nil) } diff --git a/pkg/api/stars.go b/pkg/api/stars.go index ff65c4b225d..e3594fc0eb7 100644 --- a/pkg/api/stars.go +++ b/pkg/api/stars.go @@ -1,33 +1,44 @@ package api import ( + "net/http" + "strconv" + "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/web" ) -func StarDashboard(c *models.ReqContext) response.Response { - cmd := models.StarDashboardCommand{UserId: c.UserId, DashboardId: c.ParamsInt64(":id")} +func (hs *HTTPServer) StarDashboard(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + cmd := models.StarDashboardCommand{UserId: c.UserId, DashboardId: id} if cmd.DashboardId <= 0 { return response.Error(400, "Missing dashboard id", nil) } - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.StarDashboard(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to star dashboard", err) } return response.Success("Dashboard starred!") } -func UnstarDashboard(c *models.ReqContext) response.Response { - cmd := models.UnstarDashboardCommand{UserId: c.UserId, DashboardId: c.ParamsInt64(":id")} +func (hs *HTTPServer) UnstarDashboard(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + cmd := models.UnstarDashboardCommand{UserId: c.UserId, DashboardId: id} if cmd.DashboardId <= 0 { return response.Error(400, "Missing dashboard id", nil) } - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.UnstarDashboard(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to unstar dashboard", err) } diff --git a/pkg/api/team.go b/pkg/api/team.go index 907a05c9c1a..c695db38b4c 100644 --- a/pkg/api/team.go +++ b/pkg/api/team.go @@ -3,11 +3,12 @@ package api import ( "errors" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" @@ -19,7 +20,7 @@ func (hs *HTTPServer) CreateTeam(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - accessControlEnabled := hs.Cfg.FeatureToggles["accesscontrol"] + accessControlEnabled := hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) if !accessControlEnabled && c.OrgRole == models.ROLE_VIEWER { return response.Error(403, "Not allowed to create team.", nil) } @@ -37,8 +38,7 @@ func (hs *HTTPServer) CreateTeam(c *models.ReqContext) response.Response { // the SignedInUser is an empty struct therefore // an additional check whether it is an actual user is required if c.SignedInUser.IsRealUser() { - if err := addTeamMember(hs.SQLStore, c.SignedInUser.UserId, c.OrgId, team.Id, false, - models.PERMISSION_ADMIN); err != nil { + if err := addOrUpdateTeamMember(c.Req.Context(), hs.TeamPermissionsService, c.SignedInUser.UserId, c.OrgId, team.Id, models.PERMISSION_ADMIN.String()); err != nil { c.Logger.Error("Could not add creator to team", "error", err) } } else { @@ -55,17 +55,23 @@ func (hs *HTTPServer) CreateTeam(c *models.ReqContext) response.Response { // PUT /api/teams/:teamId func (hs *HTTPServer) UpdateTeam(c *models.ReqContext) response.Response { cmd := models.UpdateTeamCommand{} + var err error if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } cmd.OrgId = c.OrgId - cmd.Id = c.ParamsInt64(":teamId") - - if err := hs.teamGuardian.CanAdmin(c.Req.Context(), cmd.OrgId, cmd.Id, c.SignedInUser); err != nil { - return response.Error(403, "Not allowed to update team", err) + cmd.Id, err = strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - if err := hs.Bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if err := hs.teamGuardian.CanAdmin(c.Req.Context(), cmd.OrgId, cmd.Id, c.SignedInUser); err != nil { + return response.Error(403, "Not allowed to update team", err) + } + } + + if err := hs.SQLStore.UpdateTeam(c.Req.Context(), &cmd); err != nil { if errors.Is(err, models.ErrTeamNameTaken) { return response.Error(400, "Team name taken", err) } @@ -78,14 +84,19 @@ func (hs *HTTPServer) UpdateTeam(c *models.ReqContext) response.Response { // DELETE /api/teams/:teamId func (hs *HTTPServer) DeleteTeamByID(c *models.ReqContext) response.Response { orgId := c.OrgId - teamId := c.ParamsInt64(":teamId") + teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) + } user := c.SignedInUser - if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, user); err != nil { - return response.Error(403, "Not allowed to delete team", err) + if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, user); err != nil { + return response.Error(403, "Not allowed to delete team", err) + } } - if err := hs.Bus.Dispatch(c.Req.Context(), &models.DeleteTeamCommand{OrgId: orgId, Id: teamId}); err != nil { + if err := hs.SQLStore.DeleteTeam(c.Req.Context(), &models.DeleteTeamCommand{OrgId: orgId, Id: teamId}); err != nil { if errors.Is(err, models.ErrTeamNotFound) { return response.Error(404, "Failed to delete Team. ID not found", nil) } @@ -121,7 +132,7 @@ func (hs *HTTPServer) SearchTeams(c *models.ReqContext) response.Response { HiddenUsers: hs.Cfg.HiddenUsers, } - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + if err := hs.SQLStore.SearchTeams(c.Req.Context(), &query); err != nil { return response.Error(500, "Failed to search Teams", err) } @@ -137,14 +148,18 @@ func (hs *HTTPServer) SearchTeams(c *models.ReqContext) response.Response { // GET /api/teams/:teamId func (hs *HTTPServer) GetTeamByID(c *models.ReqContext) response.Response { + teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) + } query := models.GetTeamByIdQuery{ OrgId: c.OrgId, - Id: c.ParamsInt64(":teamId"), + Id: teamId, SignedInUser: c.SignedInUser, HiddenUsers: hs.Cfg.HiddenUsers, } - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + if err := hs.SQLStore.GetTeamById(c.Req.Context(), &query); err != nil { if errors.Is(err, models.ErrTeamNotFound) { return response.Error(404, "Team not found", err) } @@ -158,11 +173,17 @@ func (hs *HTTPServer) GetTeamByID(c *models.ReqContext) response.Response { // GET /api/teams/:teamId/preferences func (hs *HTTPServer) GetTeamPreferences(c *models.ReqContext) response.Response { - teamId := c.ParamsInt64(":teamId") + teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) + } + orgId := c.OrgId - if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { - return response.Error(403, "Not allowed to view team preferences.", err) + if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { + return response.Error(403, "Not allowed to view team preferences.", err) + } } return hs.getPreferencesFor(c.Req.Context(), orgId, 0, teamId) @@ -174,11 +195,18 @@ func (hs *HTTPServer) UpdateTeamPreferences(c *models.ReqContext) response.Respo if err := web.Bind(c.Req, &dtoCmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - teamId := c.ParamsInt64(":teamId") + + teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) + } + orgId := c.OrgId - if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { - return response.Error(403, "Not allowed to update team preferences.", err) + if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { + return response.Error(403, "Not allowed to update team preferences.", err) + } } return hs.updatePreferencesFor(c.Req.Context(), orgId, 0, teamId, &dtoCmd) diff --git a/pkg/api/team_members.go b/pkg/api/team_members.go index bc6f7d5a68b..cc0cd45daf4 100644 --- a/pkg/api/team_members.go +++ b/pkg/api/team_members.go @@ -1,23 +1,31 @@ package api import ( + "context" "errors" + "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" ) // GET /api/teams/:teamId/members func (hs *HTTPServer) GetTeamMembers(c *models.ReqContext) response.Response { - query := models.GetTeamMembersQuery{OrgId: c.OrgId, TeamId: c.ParamsInt64(":teamId")} + teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) + } - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + query := models.GetTeamMembersQuery{OrgId: c.OrgId, TeamId: teamId} + + if err := hs.SQLStore.GetTeamMembers(c.Req.Context(), &query); err != nil { return response.Error(500, "Failed to get Team Members", err) } @@ -44,26 +52,32 @@ func (hs *HTTPServer) GetTeamMembers(c *models.ReqContext) response.Response { // POST /api/teams/:teamId/members func (hs *HTTPServer) AddTeamMember(c *models.ReqContext) response.Response { cmd := models.AddTeamMemberCommand{} + var err error if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } cmd.OrgId = c.OrgId - cmd.TeamId = c.ParamsInt64(":teamId") - - if err := hs.teamGuardian.CanAdmin(c.Req.Context(), cmd.OrgId, cmd.TeamId, c.SignedInUser); err != nil { - return response.Error(403, "Not allowed to add team member", err) + cmd.TeamId, err = strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - err := addTeamMember(hs.SQLStore, cmd.UserId, cmd.OrgId, cmd.TeamId, cmd.External, cmd.Permission) + if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if err := hs.teamGuardian.CanAdmin(c.Req.Context(), cmd.OrgId, cmd.TeamId, c.SignedInUser); err != nil { + return response.Error(403, "Not allowed to add team member", err) + } + } + + isTeamMember, err := hs.SQLStore.IsTeamMember(c.OrgId, cmd.TeamId, cmd.UserId) if err != nil { - if errors.Is(err, models.ErrTeamNotFound) { - return response.Error(404, "Team not found", nil) - } - - if errors.Is(err, models.ErrTeamMemberAlreadyAdded) { - return response.Error(400, "User is already added to this team", nil) - } + return response.Error(500, "Failed to add team member.", err) + } + if isTeamMember { + return response.Error(400, "User is already added to this team", nil) + } + err = addOrUpdateTeamMember(c.Req.Context(), hs.TeamPermissionsService, cmd.UserId, cmd.OrgId, cmd.TeamId, getPermissionName(cmd.Permission)) + if err != nil { return response.Error(500, "Failed to add Member to Team", err) } @@ -78,46 +92,67 @@ func (hs *HTTPServer) UpdateTeamMember(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - teamId := c.ParamsInt64(":teamId") + teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) + } + userId, err := strconv.ParseInt(web.Params(c.Req)[":userId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "userId is invalid", err) + } orgId := c.OrgId - if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { - return response.Error(403, "Not allowed to update team member", err) - } - - if c.OrgRole != models.ROLE_ADMIN { - cmd.ProtectLastAdmin = true - } - - cmd.TeamId = teamId - cmd.UserId = c.ParamsInt64(":userId") - cmd.OrgId = orgId - - if err := hs.Bus.Dispatch(c.Req.Context(), &cmd); err != nil { - if errors.Is(err, models.ErrTeamMemberNotFound) { - return response.Error(404, "Team member not found.", nil) + if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { + return response.Error(403, "Not allowed to update team member", err) } + } + + isTeamMember, err := hs.SQLStore.IsTeamMember(orgId, teamId, userId) + if err != nil { + return response.Error(500, "Failed to update team member.", err) + } + if !isTeamMember { + return response.Error(404, "Team member not found.", nil) + } + + err = addOrUpdateTeamMember(c.Req.Context(), hs.TeamPermissionsService, userId, orgId, teamId, getPermissionName(cmd.Permission)) + if err != nil { return response.Error(500, "Failed to update team member.", err) } return response.Success("Team member updated") } +func getPermissionName(permission models.PermissionType) string { + permissionName := permission.String() + // Team member permission is 0, which maps to an empty string. + // However, we want the team permission service to display "Member" for team members. This is a hack to make it work. + if permissionName == "" { + permissionName = "Member" + } + return permissionName +} + // DELETE /api/teams/:teamId/members/:userId func (hs *HTTPServer) RemoveTeamMember(c *models.ReqContext) response.Response { orgId := c.OrgId - teamId := c.ParamsInt64(":teamId") - userId := c.ParamsInt64(":userId") - - if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { - return response.Error(403, "Not allowed to remove team member", err) + teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamId is invalid", err) + } + userId, err := strconv.ParseInt(web.Params(c.Req)[":userId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "userId is invalid", err) } - protectLastAdmin := false - if c.OrgRole != models.ROLE_ADMIN { - protectLastAdmin = true + if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { + return response.Error(403, "Not allowed to remove team member", err) + } } - if err := hs.Bus.Dispatch(c.Req.Context(), &models.RemoveTeamMemberCommand{OrgId: orgId, TeamId: teamId, UserId: userId, ProtectLastAdmin: protectLastAdmin}); err != nil { + teamIDString := strconv.FormatInt(teamId, 10) + if _, err := hs.TeamPermissionsService.SetUserPermission(c.Req.Context(), orgId, userId, teamIDString, ""); err != nil { if errors.Is(err, models.ErrTeamNotFound) { return response.Error(404, "Team not found", nil) } @@ -131,10 +166,13 @@ func (hs *HTTPServer) RemoveTeamMember(c *models.ReqContext) response.Response { return response.Success("Team Member removed") } -// addTeamMember adds a team member. +// addOrUpdateTeamMember adds or updates a team member. // // Stubbable by tests. -var addTeamMember = func(sqlStore *sqlstore.SQLStore, userID, orgID, teamID int64, isExternal bool, - permission models.PermissionType) error { - return sqlStore.AddTeamMember(userID, orgID, teamID, isExternal, permission) +var addOrUpdateTeamMember = func(ctx context.Context, resourcePermissionService *resourcepermissions.Service, userID, orgID, teamID int64, permission string) error { + teamIDString := strconv.FormatInt(teamID, 10) + if _, err := resourcePermissionService.SetUserPermission(ctx, orgID, userID, teamIDString, permission); err != nil { + return fmt.Errorf("failed setting permissions for user %d in team %d: %w", userID, teamID, err) + } + return nil } diff --git a/pkg/api/team_members_test.go b/pkg/api/team_members_test.go index af942d44033..37dd49b424f 100644 --- a/pkg/api/team_members_test.go +++ b/pkg/api/team_members_test.go @@ -3,38 +3,54 @@ package api import ( "context" "encoding/json" + "fmt" + "math/rand" "net/http" + "strings" "testing" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/licensing" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/services/teamguardian/database" + "github.com/grafana/grafana/pkg/services/teamguardian/manager" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func setUpGetTeamMembersHandler() { - bus.AddHandler("test", func(ctx context.Context, query *models.GetTeamMembersQuery) error { - query.Result = []*models.TeamMemberDTO{ - {Email: "testUser@grafana.com", Login: testUserLogin}, - {Email: "user1@grafana.com", Login: "user1"}, - {Email: "user2@grafana.com", Login: "user2"}, +func setUpGetTeamMembersHandler(t *testing.T, sqlStore *sqlstore.SQLStore) { + const testOrgID int64 = 1 + var userCmd models.CreateUserCommand + team, err := sqlStore.CreateTeam("group1 name", "test1@test.com", testOrgID) + require.NoError(t, err) + for i := 0; i < 3; i++ { + userCmd = models.CreateUserCommand{ + Email: fmt.Sprint("user", i, "@test.com"), + Name: fmt.Sprint("user", i), + Login: fmt.Sprint("loginuser", i), } - return nil - }) + // user + user, err := sqlStore.CreateUser(context.Background(), userCmd) + require.NoError(t, err) + err = sqlStore.AddTeamMember(user.Id, testOrgID, team.Id, false, 1) + require.NoError(t, err) + } } func TestTeamMembersAPIEndpoint_userLoggedIn(t *testing.T) { settings := setting.NewCfg() + sqlStore := sqlstore.InitTestDB(t) hs := &HTTPServer{ - Cfg: settings, - License: &licensing.OSSLicensingService{}, + Cfg: settings, + License: &licensing.OSSLicensingService{}, + SQLStore: sqlStore, } loggedInUserScenarioWithRole(t, "When calling GET on", "GET", "api/teams/1/members", "api/teams/:teamId/members", models.ROLE_ADMIN, func(sc *scenarioContext) { - setUpGetTeamMembersHandler() + setUpGetTeamMembersHandler(t, sqlStore) sc.handlerFunc = hs.GetTeamMembers sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() @@ -56,7 +72,7 @@ func TestTeamMembersAPIEndpoint_userLoggedIn(t *testing.T) { loggedInUserScenarioWithRole(t, "When calling GET on", "GET", "api/teams/1/members", "api/teams/:teamId/members", models.ROLE_ADMIN, func(sc *scenarioContext) { - setUpGetTeamMembersHandler() + setUpGetTeamMembersHandler(t, sqlStore) sc.handlerFunc = hs.GetTeamMembers sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() @@ -66,9 +82,281 @@ func TestTeamMembersAPIEndpoint_userLoggedIn(t *testing.T) { var resp []models.TeamMemberDTO err := json.Unmarshal(sc.resp.Body.Bytes(), &resp) require.NoError(t, err) - assert.Len(t, resp, 2) - assert.Equal(t, testUserLogin, resp[0].Login) - assert.Equal(t, "user2", resp[1].Login) + assert.Len(t, resp, 3) + assert.Equal(t, "loginuser0", resp[0].Login) + assert.Equal(t, "loginuser1", resp[1].Login) + assert.Equal(t, "loginuser2", resp[2].Login) }) }) } + +func createUser(db *sqlstore.SQLStore, orgId int64, t *testing.T) int64 { + user, err := db.CreateUser(context.Background(), models.CreateUserCommand{ + Login: fmt.Sprintf("TestUser%d", rand.Int()), + OrgId: orgId, + Password: "password", + }) + require.NoError(t, err) + + return user.Id +} + +func setupTeamTestScenario(userCount int, db *sqlstore.SQLStore, t *testing.T) int64 { + user, err := db.CreateUser(context.Background(), models.CreateUserCommand{SkipOrgSetup: true, Login: testUserLogin}) + require.NoError(t, err) + testOrg, err := db.CreateOrgWithMember("TestOrg", user.Id) + require.NoError(t, err) + + team, err := db.CreateTeam("test", "test@test.com", testOrg.Id) + require.NoError(t, err) + + for i := 0; i < userCount; i++ { + userId := createUser(db, testOrg.Id, t) + require.NoError(t, err) + + err = db.AddTeamMember(userId, testOrg.Id, team.Id, false, 0) + require.NoError(t, err) + } + + return testOrg.Id +} + +var ( + teamMemberAddRoute = "/api/teams/%s/members" + createTeamMemberCmd = `{"userId": %d}` + teamMemberUpdateRoute = "/api/teams/%s/members/%s" + updateTeamMemberCmd = `{"permission": %d}` + teamMemberDeleteRoute = "/api/teams/%s/members/%s" +) + +func TestAddTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { + cfg := setting.NewCfg() + cfg.EditorsCanAdmin = true + sc := setupHTTPServerWithCfg(t, true, false, cfg) + guardian := manager.ProvideService(database.ProvideTeamGuardianStore(sc.db)) + sc.hs.teamGuardian = guardian + + teamMemberCount := 3 + testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, t) + + setInitCtxSignedInOrgAdmin(sc.initCtx) + newUserId := createUser(sc.db, testOrgId, t) + input := strings.NewReader(fmt.Sprintf(createTeamMemberCmd, newUserId)) + t.Run("Organisation admins can add a team member", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPost, fmt.Sprintf(teamMemberAddRoute, "1"), input, t) + assert.Equal(t, http.StatusOK, response.Code) + }) + + setInitCtxSignedInEditor(sc.initCtx) + sc.initCtx.IsGrafanaAdmin = true + newUserId = createUser(sc.db, testOrgId, t) + input = strings.NewReader(fmt.Sprintf(createTeamMemberCmd, newUserId)) + t.Run("Editors cannot add team members", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPost, fmt.Sprintf(teamMemberAddRoute, "1"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + err := sc.db.AddTeamMember(sc.initCtx.UserId, 1, 1, false, 0) + require.NoError(t, err) + input = strings.NewReader(fmt.Sprintf(createTeamMemberCmd, newUserId)) + t.Run("Team members cannot add team members", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPost, fmt.Sprintf(teamMemberAddRoute, "1"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + err = sc.db.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ + UserId: sc.initCtx.UserId, + OrgId: 1, + TeamId: 1, + Permission: models.PERMISSION_ADMIN, + }) + require.NoError(t, err) + input = strings.NewReader(fmt.Sprintf(createTeamMemberCmd, newUserId)) + t.Run("Team admins can add a team member", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPost, fmt.Sprintf(teamMemberAddRoute, "1"), input, t) + assert.Equal(t, http.StatusOK, response.Code) + }) +} + +func TestAddTeamMembersAPIEndpoint_FGAC(t *testing.T) { + sc := setupHTTPServer(t, true, true) + sc.hs.License = &licensing.OSSLicensingService{} + + teamMemberCount := 3 + testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, t) + + setInitCtxSignedInViewer(sc.initCtx) + newUserId := createUser(sc.db, testOrgId, t) + input := strings.NewReader(fmt.Sprintf(createTeamMemberCmd, newUserId)) + t.Run("Access control allows adding a team member with the right permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodPost, fmt.Sprintf(teamMemberAddRoute, "1"), input, t) + assert.Equal(t, http.StatusOK, response.Code) + }) + + setInitCtxSignedInOrgAdmin(sc.initCtx) + newUserId = createUser(sc.db, testOrgId, t) + input = strings.NewReader(fmt.Sprintf(teamCmd, newUserId)) + t.Run("Access control prevents from adding a team member with the wrong permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsRead, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodPost, fmt.Sprintf(teamMemberAddRoute, "1"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + setInitCtxSignedInViewer(sc.initCtx) + t.Run("Access control prevents adding a team member with incorrect scope", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}, 1) + response := callAPI(sc.server, http.MethodPost, fmt.Sprintf(teamMemberAddRoute, "1"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) +} + +func TestUpdateTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { + cfg := setting.NewCfg() + cfg.EditorsCanAdmin = true + sc := setupHTTPServerWithCfg(t, true, false, cfg) + guardian := manager.ProvideService(database.ProvideTeamGuardianStore(sc.db)) + sc.hs.teamGuardian = guardian + + teamMemberCount := 3 + setupTeamTestScenario(teamMemberCount, sc.db, t) + + setInitCtxSignedInOrgAdmin(sc.initCtx) + input := strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, models.PERMISSION_ADMIN)) + t.Run("Organisation admins can update a team member", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(teamMemberUpdateRoute, "1", "2"), input, t) + assert.Equal(t, http.StatusOK, response.Code) + }) + + setInitCtxSignedInEditor(sc.initCtx) + sc.initCtx.IsGrafanaAdmin = true + input = strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, 0)) + t.Run("Editors cannot update team members", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(teamMemberUpdateRoute, "1", "2"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + err := sc.db.AddTeamMember(sc.initCtx.UserId, 1, 1, false, 0) + require.NoError(t, err) + input = strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, 0)) + t.Run("Team members cannot update team members", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(teamMemberUpdateRoute, "1", "2"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + err = sc.db.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ + UserId: sc.initCtx.UserId, + OrgId: 1, + TeamId: 1, + Permission: models.PERMISSION_ADMIN, + }) + require.NoError(t, err) + input = strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, 0)) + t.Run("Team admins can update a team member", func(t *testing.T) { + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(teamMemberUpdateRoute, "1", "2"), input, t) + assert.Equal(t, http.StatusOK, response.Code) + }) +} + +func TestUpdateTeamMembersAPIEndpoint_FGAC(t *testing.T) { + sc := setupHTTPServer(t, true, true) + sc.hs.License = &licensing.OSSLicensingService{} + + teamMemberCount := 3 + setupTeamTestScenario(teamMemberCount, sc.db, t) + + setInitCtxSignedInViewer(sc.initCtx) + input := strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, models.PERMISSION_ADMIN)) + t.Run("Access control allows updating a team member with the right permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(teamMemberUpdateRoute, "1", "2"), input, t) + assert.Equal(t, http.StatusOK, response.Code) + }) + + setInitCtxSignedInOrgAdmin(sc.initCtx) + input = strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, models.PERMISSION_ADMIN)) + t.Run("Access control prevents updating a team member with the wrong permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsRead, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(teamMemberUpdateRoute, "1", "2"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + setInitCtxSignedInViewer(sc.initCtx) + t.Run("Access control prevents updating a team member with incorrect scope", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(teamMemberUpdateRoute, "1", "2"), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) +} + +func TestDeleteTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { + cfg := setting.NewCfg() + cfg.EditorsCanAdmin = true + sc := setupHTTPServerWithCfg(t, true, false, cfg) + guardian := manager.ProvideService(database.ProvideTeamGuardianStore(sc.db)) + sc.hs.teamGuardian = guardian + + teamMemberCount := 3 + setupTeamTestScenario(teamMemberCount, sc.db, t) + + setInitCtxSignedInOrgAdmin(sc.initCtx) + t.Run("Organisation admins can remove a team member", func(t *testing.T) { + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(teamMemberDeleteRoute, "1", "2"), nil, t) + assert.Equal(t, http.StatusOK, response.Code) + }) + + setInitCtxSignedInEditor(sc.initCtx) + sc.initCtx.IsGrafanaAdmin = true + t.Run("Editors cannot remove team members", func(t *testing.T) { + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(teamMemberDeleteRoute, "1", "3"), nil, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + err := sc.db.AddTeamMember(sc.initCtx.UserId, 1, 1, false, 0) + require.NoError(t, err) + t.Run("Team members cannot remove team members", func(t *testing.T) { + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(teamMemberDeleteRoute, "1", "3"), nil, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + err = sc.db.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ + UserId: sc.initCtx.UserId, + OrgId: 1, + TeamId: 1, + Permission: models.PERMISSION_ADMIN, + }) + require.NoError(t, err) + t.Run("Team admins can remove a team member", func(t *testing.T) { + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(teamMemberDeleteRoute, "1", "3"), nil, t) + assert.Equal(t, http.StatusOK, response.Code) + }) +} + +func TestDeleteTeamMembersAPIEndpoint_FGAC(t *testing.T) { + sc := setupHTTPServer(t, true, true) + sc.hs.License = &licensing.OSSLicensingService{} + + teamMemberCount := 3 + setupTeamTestScenario(teamMemberCount, sc.db, t) + + setInitCtxSignedInViewer(sc.initCtx) + t.Run("Access control allows removing a team member with the right permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(teamMemberDeleteRoute, "1", "2"), nil, t) + assert.Equal(t, http.StatusOK, response.Code) + }) + + setInitCtxSignedInOrgAdmin(sc.initCtx) + t.Run("Access control prevents removing a team member with the wrong permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsRead, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(teamMemberDeleteRoute, "1", "3"), nil, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) + + setInitCtxSignedInViewer(sc.initCtx) + t.Run("Access control prevents removing a team member with incorrect scope", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}, 1) + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(teamMemberDeleteRoute, "1", "3"), nil, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) +} diff --git a/pkg/api/team_test.go b/pkg/api/team_test.go index 6667895ea5c..0e2a0a8c620 100644 --- a/pkg/api/team_test.go +++ b/pkg/api/team_test.go @@ -2,16 +2,16 @@ package api import ( "context" + "encoding/json" "fmt" "net/http" "strings" "testing" - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" @@ -32,80 +32,56 @@ func (stub *testLogger) Warn(testMessage string, ctx ...interface{}) { func TestTeamAPIEndpoint(t *testing.T) { t.Run("Given two teams", func(t *testing.T) { - mockResult := models.SearchTeamQueryResult{ - Teams: []*models.TeamDTO{ - {Name: "team1"}, - {Name: "team2"}, - }, - TotalCount: 2, - } + hs := setupSimpleHTTPServer(nil) + hs.SQLStore = sqlstore.InitTestDB(t) - hs := &HTTPServer{ - Cfg: setting.NewCfg(), - } - - loggedInUserScenario(t, "When calling GET on", "/api/teams/search", func(sc *scenarioContext) { - var sentLimit int - var sendPage int - bus.AddHandler("test", func(ctx context.Context, query *models.SearchTeamsQuery) error { - query.Result = mockResult - - sentLimit = query.Limit - sendPage = query.Page - - return nil - }) + loggedInUserScenario(t, "When calling GET on", "/api/teams/search", "/api/teams/search", func(sc *scenarioContext) { + _, err := hs.SQLStore.CreateTeam("team1", "", 1) + require.NoError(t, err) + _, err = hs.SQLStore.CreateTeam("team2", "", 1) + require.NoError(t, err) sc.handlerFunc = hs.SearchTeams sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() - - assert.Equal(t, 1000, sentLimit) - assert.Equal(t, 1, sendPage) - - respJSON, err := simplejson.NewJson(sc.resp.Body.Bytes()) + require.Equal(t, http.StatusOK, sc.resp.Code) + var resp models.SearchTeamQueryResult + err = json.Unmarshal(sc.resp.Body.Bytes(), &resp) require.NoError(t, err) - assert.Equal(t, 2, respJSON.Get("totalCount").MustInt()) - assert.Equal(t, 2, len(respJSON.Get("teams").MustArray())) + assert.EqualValues(t, 2, resp.TotalCount) + assert.Equal(t, 2, len(resp.Teams)) }) - loggedInUserScenario(t, "When calling GET on", "/api/teams/search", func(sc *scenarioContext) { - var sentLimit int - var sendPage int - bus.AddHandler("test", func(ctx context.Context, query *models.SearchTeamsQuery) error { - query.Result = mockResult - - sentLimit = query.Limit - sendPage = query.Page - - return nil - }) + loggedInUserScenario(t, "When calling GET on", "/api/teams/search", "/api/teams/search", func(sc *scenarioContext) { + _, err := hs.SQLStore.CreateTeam("team1", "", 1) + require.NoError(t, err) + _, err = hs.SQLStore.CreateTeam("team2", "", 1) + require.NoError(t, err) sc.handlerFunc = hs.SearchTeams sc.fakeReqWithParams("GET", sc.url, map[string]string{"perpage": "10", "page": "2"}).exec() + require.Equal(t, http.StatusOK, sc.resp.Code) + var resp models.SearchTeamQueryResult + err = json.Unmarshal(sc.resp.Body.Bytes(), &resp) + require.NoError(t, err) - assert.Equal(t, 10, sentLimit) - assert.Equal(t, 2, sendPage) + assert.EqualValues(t, 2, resp.TotalCount) + assert.Equal(t, 0, len(resp.Teams)) }) }) t.Run("When creating team with API key", func(t *testing.T) { - defer bus.ClearBusHandlers() - - hs := &HTTPServer{ - Cfg: setting.NewCfg(), - Bus: bus.GetBus(), - } + hs := setupSimpleHTTPServer(nil) hs.Cfg.EditorsCanAdmin = true teamName := "team foo" // TODO: Use a fake SQLStore when it's represented by an interface - origCreateTeam := createTeam - origAddTeamMember := addTeamMember + orgCreateTeam := createTeam + orgAddTeamMember := addOrUpdateTeamMember t.Cleanup(func() { - createTeam = origCreateTeam - addTeamMember = origAddTeamMember + createTeam = orgCreateTeam + addOrUpdateTeamMember = orgAddTeamMember }) createTeamCalled := 0 @@ -115,8 +91,8 @@ func TestTeamAPIEndpoint(t *testing.T) { } addTeamMemberCalled := 0 - addTeamMember = func(sqlStore *sqlstore.SQLStore, userID, orgID, teamID int64, isExternal bool, - permission models.PermissionType) error { + addOrUpdateTeamMember = func(ctx context.Context, resourcePermissionService *resourcepermissions.Service, userID, orgID, teamID int64, + permission string) error { addTeamMemberCalled++ return nil } @@ -158,16 +134,20 @@ func TestTeamAPIEndpoint(t *testing.T) { }) } -var ( - createTeamURL = "/api/teams/" - createTeamCmd = `{"name": "MyTestTeam%d"}` +const ( + createTeamURL = "/api/teams/" + detailTeamURL = "/api/teams/%d" + detailTeamPreferenceURL = "/api/teams/%d/preferences" + teamCmd = `{"name": "MyTestTeam%d"}` + teamPreferenceCmd = `{"theme": "dark"}` + teamPreferenceCmdLight = `{"theme": "light"}` ) func TestTeamAPIEndpoint_CreateTeam_LegacyAccessControl(t *testing.T) { sc := setupHTTPServer(t, true, false) setInitCtxSignedInOrgAdmin(sc.initCtx) - input := strings.NewReader(fmt.Sprintf(createTeamCmd, 1)) + input := strings.NewReader(fmt.Sprintf(teamCmd, 1)) t.Run("Organisation admin can create a team", func(t *testing.T) { response := callAPI(sc.server, http.MethodPost, createTeamURL, input, t) assert.Equal(t, http.StatusOK, response.Code) @@ -175,9 +155,9 @@ func TestTeamAPIEndpoint_CreateTeam_LegacyAccessControl(t *testing.T) { setInitCtxSignedInEditor(sc.initCtx) sc.initCtx.IsGrafanaAdmin = true - input = strings.NewReader(fmt.Sprintf(createTeamCmd, 2)) + input = strings.NewReader(fmt.Sprintf(teamCmd, 2)) t.Run("Org editor and server admin cannot create a team", func(t *testing.T) { - response := callAPI(sc.server, http.MethodPost, createTeamURL, strings.NewReader(createTeamCmd), t) + response := callAPI(sc.server, http.MethodPost, createTeamURL, strings.NewReader(teamCmd), t) assert.Equal(t, http.StatusForbidden, response.Code) }) } @@ -188,7 +168,7 @@ func TestTeamAPIEndpoint_CreateTeam_LegacyAccessControl_EditorsCanAdmin(t *testi sc := setupHTTPServerWithCfg(t, true, false, cfg) setInitCtxSignedInEditor(sc.initCtx) - input := strings.NewReader(fmt.Sprintf(createTeamCmd, 1)) + input := strings.NewReader(fmt.Sprintf(teamCmd, 1)) t.Run("Editors can create a team if editorsCanAdmin is set to true", func(t *testing.T) { response := callAPI(sc.server, http.MethodPost, createTeamURL, input, t) assert.Equal(t, http.StatusOK, response.Code) @@ -199,17 +179,161 @@ func TestTeamAPIEndpoint_CreateTeam_FGAC(t *testing.T) { sc := setupHTTPServer(t, true, true) setInitCtxSignedInViewer(sc.initCtx) - input := strings.NewReader(fmt.Sprintf(createTeamCmd, 1)) + input := strings.NewReader(fmt.Sprintf(teamCmd, 1)) t.Run("Access control allows creating teams with the correct permissions", func(t *testing.T) { - setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: ActionTeamsCreate}}, 1) + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsCreate}}, 1) response := callAPI(sc.server, http.MethodPost, createTeamURL, input, t) assert.Equal(t, http.StatusOK, response.Code) }) - input = strings.NewReader(fmt.Sprintf(createTeamCmd, 2)) + input = strings.NewReader(fmt.Sprintf(teamCmd, 2)) t.Run("Access control prevents creating teams with the incorrect permissions", func(t *testing.T) { setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: "teams:invalid"}}, accesscontrol.GlobalOrgID) response := callAPI(sc.server, http.MethodPost, createTeamURL, input, t) assert.Equal(t, http.StatusForbidden, response.Code) }) } + +// Given a team with a user, when the user is granted X permission, +// Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsWrite with teams:id:1 scope +// else return 403 +func TestTeamAPIEndpoint_UpdateTeam_FGAC(t *testing.T) { + sc := setupHTTPServer(t, true, true) + sc.db = sqlstore.InitTestDB(t) + _, err := sc.db.CreateTeam("team1", "", 1) + + require.NoError(t, err) + + setInitCtxSignedInViewer(sc.initCtx) + + input := strings.NewReader(fmt.Sprintf(teamCmd, 1)) + t.Run("Access control allows updating teams with the correct permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(detailTeamURL, 1), input, t) + assert.Equal(t, http.StatusOK, response.Code) + + teamQuery := &models.GetTeamByIdQuery{OrgId: 1, SignedInUser: sc.initCtx.SignedInUser, Id: 1, Result: &models.TeamDTO{}} + err := sc.db.GetTeamById(context.Background(), teamQuery) + require.NoError(t, err) + assert.Equal(t, "MyTestTeam1", teamQuery.Result.Name) + }) + + input = strings.NewReader(fmt.Sprintf(teamCmd, 2)) + t.Run("Access control allows updating teams with the correct global permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:*"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(detailTeamURL, 1), input, t) + assert.Equal(t, http.StatusOK, response.Code) + + teamQuery := &models.GetTeamByIdQuery{OrgId: 1, SignedInUser: sc.initCtx.SignedInUser, Id: 1, Result: &models.TeamDTO{}} + err := sc.db.GetTeamById(context.Background(), teamQuery) + require.NoError(t, err) + assert.Equal(t, "MyTestTeam2", teamQuery.Result.Name) + }) + + input = strings.NewReader(fmt.Sprintf(teamCmd, 3)) + t.Run("Access control prevents updating teams with the incorrect permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:2"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(detailTeamURL, 1), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + + teamQuery := &models.GetTeamByIdQuery{OrgId: 1, SignedInUser: sc.initCtx.SignedInUser, Id: 1, Result: &models.TeamDTO{}} + err := sc.db.GetTeamById(context.Background(), teamQuery) + assert.NoError(t, err) + assert.Equal(t, "MyTestTeam2", teamQuery.Result.Name) + }) +} + +// Given a team with a user, when the user is granted X permission, +// Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsDelete with teams:id:1 scope +// else return 403 +func TestTeamAPIEndpoint_DeleteTeam_FGAC(t *testing.T) { + sc := setupHTTPServer(t, true, true) + sc.db = sqlstore.InitTestDB(t) + _, err := sc.db.CreateTeam("team1", "", 1) + require.NoError(t, err) + + setInitCtxSignedInViewer(sc.initCtx) + + t.Run("Access control prevents deleting teams with the incorrect permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsDelete, Scope: "teams:id:7"}}, 1) + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(detailTeamURL, 1), http.NoBody, t) + assert.Equal(t, http.StatusForbidden, response.Code) + + teamQuery := &models.GetTeamByIdQuery{OrgId: 1, SignedInUser: sc.initCtx.SignedInUser, Id: 1, Result: &models.TeamDTO{}} + err := sc.db.GetTeamById(context.Background(), teamQuery) + require.NoError(t, err) + }) + + t.Run("Access control allows deleting teams with the correct permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsDelete, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodDelete, fmt.Sprintf(detailTeamURL, 1), http.NoBody, t) + assert.Equal(t, http.StatusOK, response.Code) + + teamQuery := &models.GetTeamByIdQuery{OrgId: 1, SignedInUser: sc.initCtx.SignedInUser, Id: 1, Result: &models.TeamDTO{}} + err := sc.db.GetTeamById(context.Background(), teamQuery) + require.ErrorIs(t, err, models.ErrTeamNotFound) + }) +} + +// Given a team with a user, when the user is granted X permission, +// Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsRead with teams:id:1 scope +// else return 403 +func TestTeamAPIEndpoint_GetTeamPreferences_FGAC(t *testing.T) { + sc := setupHTTPServer(t, true, true) + sc.db = sqlstore.InitTestDB(t) + _, err := sc.db.CreateTeam("team1", "", 1) + + require.NoError(t, err) + + setInitCtxSignedInViewer(sc.initCtx) + + t.Run("Access control allows getting team preferences with the correct permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, + []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsRead, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodGet, fmt.Sprintf(detailTeamPreferenceURL, 1), http.NoBody, t) + assert.Equal(t, http.StatusOK, response.Code) + }) + + t.Run("Access control prevents getting team preferences with the incorrect permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsRead, Scope: "teams:id:2"}}, 1) + response := callAPI(sc.server, http.MethodGet, fmt.Sprintf(detailTeamPreferenceURL, 1), http.NoBody, t) + assert.Equal(t, http.StatusForbidden, response.Code) + }) +} + +// Given a team with a user, when the user is granted X permission, +// Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsWrite with teams:id:1 scope +// else return 403 +func TestTeamAPIEndpoint_UpdateTeamPreferences_FGAC(t *testing.T) { + sc := setupHTTPServer(t, true, true) + sc.db = sqlstore.InitTestDB(t) + _, err := sc.db.CreateTeam("team1", "", 1) + + require.NoError(t, err) + + setInitCtxSignedInViewer(sc.initCtx) + + input := strings.NewReader(teamPreferenceCmd) + t.Run("Access control allows updating team preferences with the correct permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:1"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(detailTeamPreferenceURL, 1), input, t) + assert.Equal(t, http.StatusOK, response.Code) + + prefQuery := &models.GetPreferencesQuery{OrgId: 1, TeamId: 1, Result: &models.Preferences{}} + err := sc.db.GetPreferences(context.Background(), prefQuery) + require.NoError(t, err) + assert.Equal(t, "dark", prefQuery.Result.Theme) + }) + + input = strings.NewReader(teamPreferenceCmdLight) + t.Run("Access control prevents updating team preferences with the incorrect permissions", func(t *testing.T) { + setAccessControlPermissions(sc.acmock, []*accesscontrol.Permission{{Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:2"}}, 1) + response := callAPI(sc.server, http.MethodPut, fmt.Sprintf(detailTeamPreferenceURL, 1), input, t) + assert.Equal(t, http.StatusForbidden, response.Code) + + prefQuery := &models.GetPreferencesQuery{OrgId: 1, TeamId: 1, Result: &models.Preferences{}} + err := sc.db.GetPreferences(context.Background(), prefQuery) + assert.NoError(t, err) + assert.Equal(t, "dark", prefQuery.Result.Theme) + }) +} diff --git a/pkg/api/user.go b/pkg/api/user.go index 85c0207d4f3..fac5aa8acbf 100644 --- a/pkg/api/user.go +++ b/pkg/api/user.go @@ -5,10 +5,10 @@ import ( "errors" "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/setting" @@ -23,13 +23,17 @@ func (hs *HTTPServer) GetSignedInUser(c *models.ReqContext) response.Response { // GET /api/users/:id func (hs *HTTPServer) GetUserByID(c *models.ReqContext) response.Response { - return hs.getUserUserProfile(c, c.ParamsInt64(":id")) + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + return hs.getUserUserProfile(c, id) } func (hs *HTTPServer) getUserUserProfile(c *models.ReqContext, userID int64) response.Response { query := models.GetUserProfileQuery{UserId: userID} - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + if err := hs.SQLStore.GetUserProfile(c.Req.Context(), &query); err != nil { if errors.Is(err, models.ErrUserNotFound) { return response.Error(404, models.ErrUserNotFound.Error(), nil) } @@ -38,7 +42,7 @@ func (hs *HTTPServer) getUserUserProfile(c *models.ReqContext, userID int64) res getAuthQuery := models.GetAuthInfoQuery{UserId: userID} query.Result.AuthLabels = []string{} - if err := bus.Dispatch(c.Req.Context(), &getAuthQuery); err == nil { + if err := hs.authInfoService.GetAuthInfo(c.Req.Context(), &getAuthQuery); err == nil { authLabel := GetAuthProviderLabel(getAuthQuery.Result.AuthModule) query.Result.AuthLabels = append(query.Result.AuthLabels, authLabel) query.Result.IsExternal = true @@ -72,9 +76,9 @@ func (hs *HTTPServer) getGlobalUserAccessControlMetadata(c *models.ReqContext, u } // GET /api/users/lookup -func GetUserByLoginOrEmail(c *models.ReqContext) response.Response { +func (hs *HTTPServer) GetUserByLoginOrEmail(c *models.ReqContext) response.Response { query := models.GetUserByLoginQuery{LoginOrEmail: c.Query("loginOrEmail")} - if err := bus.Dispatch(c.Req.Context(), &query); err != nil { + if err := hs.SQLStore.GetUserByLogin(c.Req.Context(), &query); err != nil { if errors.Is(err, models.ErrUserNotFound) { return response.Error(404, models.ErrUserNotFound.Error(), nil) } @@ -96,7 +100,7 @@ func GetUserByLoginOrEmail(c *models.ReqContext) response.Response { } // POST /api/user -func UpdateSignedInUser(c *models.ReqContext) response.Response { +func (hs *HTTPServer) UpdateSignedInUser(c *models.ReqContext) response.Response { cmd := models.UpdateUserCommand{} if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) @@ -110,38 +114,48 @@ func UpdateSignedInUser(c *models.ReqContext) response.Response { } } cmd.UserId = c.UserId - return handleUpdateUser(c.Req.Context(), cmd) + return hs.handleUpdateUser(c.Req.Context(), cmd) } // POST /api/users/:id -func UpdateUser(c *models.ReqContext) response.Response { +func (hs *HTTPServer) UpdateUser(c *models.ReqContext) response.Response { cmd := models.UpdateUserCommand{} + var err error if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - cmd.UserId = c.ParamsInt64(":id") - return handleUpdateUser(c.Req.Context(), cmd) + cmd.UserId, err = strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + return hs.handleUpdateUser(c.Req.Context(), cmd) } // POST /api/users/:id/using/:orgId -func UpdateUserActiveOrg(c *models.ReqContext) response.Response { - userID := c.ParamsInt64(":id") - orgID := c.ParamsInt64(":orgId") +func (hs *HTTPServer) UpdateUserActiveOrg(c *models.ReqContext) response.Response { + userID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + orgID, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "orgId is invalid", err) + } - if !validateUsingOrg(c.Req.Context(), userID, orgID) { + if !hs.validateUsingOrg(c.Req.Context(), userID, orgID) { return response.Error(401, "Not a valid organization", nil) } cmd := models.SetUsingOrgCommand{UserId: userID, OrgId: orgID} - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.SetUsingOrg(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to change active organization", err) } return response.Success("Active organization changed") } -func handleUpdateUser(ctx context.Context, cmd models.UpdateUserCommand) response.Response { +func (hs *HTTPServer) handleUpdateUser(ctx context.Context, cmd models.UpdateUserCommand) response.Response { if len(cmd.Login) == 0 { cmd.Login = cmd.Email if len(cmd.Login) == 0 { @@ -149,7 +163,7 @@ func handleUpdateUser(ctx context.Context, cmd models.UpdateUserCommand) respons } } - if err := bus.Dispatch(ctx, &cmd); err != nil { + if err := hs.SQLStore.UpdateUser(ctx, &cmd); err != nil { return response.Error(500, "Failed to update user", err) } @@ -157,24 +171,28 @@ func handleUpdateUser(ctx context.Context, cmd models.UpdateUserCommand) respons } // GET /api/user/orgs -func GetSignedInUserOrgList(c *models.ReqContext) response.Response { - return getUserOrgList(c.Req.Context(), c.UserId) +func (hs *HTTPServer) GetSignedInUserOrgList(c *models.ReqContext) response.Response { + return hs.getUserOrgList(c.Req.Context(), c.UserId) } // GET /api/user/teams -func GetSignedInUserTeamList(c *models.ReqContext) response.Response { - return getUserTeamList(c.Req.Context(), c.OrgId, c.UserId) +func (hs *HTTPServer) GetSignedInUserTeamList(c *models.ReqContext) response.Response { + return hs.getUserTeamList(c.Req.Context(), c.OrgId, c.UserId) } // GET /api/users/:id/teams -func GetUserTeams(c *models.ReqContext) response.Response { - return getUserTeamList(c.Req.Context(), c.OrgId, c.ParamsInt64(":id")) +func (hs *HTTPServer) GetUserTeams(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + return hs.getUserTeamList(c.Req.Context(), c.OrgId, id) } -func getUserTeamList(ctx context.Context, orgID int64, userID int64) response.Response { +func (hs *HTTPServer) getUserTeamList(ctx context.Context, orgID int64, userID int64) response.Response { query := models.GetTeamsByUserQuery{OrgId: orgID, UserId: userID} - if err := bus.Dispatch(ctx, &query); err != nil { + if err := hs.SQLStore.GetTeamsByUser(ctx, &query); err != nil { return response.Error(500, "Failed to get user teams", err) } @@ -185,24 +203,28 @@ func getUserTeamList(ctx context.Context, orgID int64, userID int64) response.Re } // GET /api/users/:id/orgs -func GetUserOrgList(c *models.ReqContext) response.Response { - return getUserOrgList(c.Req.Context(), c.ParamsInt64(":id")) +func (hs *HTTPServer) GetUserOrgList(c *models.ReqContext) response.Response { + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } + return hs.getUserOrgList(c.Req.Context(), id) } -func getUserOrgList(ctx context.Context, userID int64) response.Response { +func (hs *HTTPServer) getUserOrgList(ctx context.Context, userID int64) response.Response { query := models.GetUserOrgListQuery{UserId: userID} - if err := bus.Dispatch(ctx, &query); err != nil { + if err := hs.SQLStore.GetUserOrgList(ctx, &query); err != nil { return response.Error(500, "Failed to get user organizations", err) } return response.JSON(200, query.Result) } -func validateUsingOrg(ctx context.Context, userID int64, orgID int64) bool { +func (hs *HTTPServer) validateUsingOrg(ctx context.Context, userID int64, orgID int64) bool { query := models.GetUserOrgListQuery{UserId: userID} - if err := bus.Dispatch(ctx, &query); err != nil { + if err := hs.SQLStore.GetUserOrgList(ctx, &query); err != nil { return false } @@ -218,16 +240,19 @@ func validateUsingOrg(ctx context.Context, userID int64, orgID int64) bool { } // POST /api/user/using/:id -func UserSetUsingOrg(c *models.ReqContext) response.Response { - orgID := c.ParamsInt64(":id") +func (hs *HTTPServer) UserSetUsingOrg(c *models.ReqContext) response.Response { + orgID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } - if !validateUsingOrg(c.Req.Context(), c.UserId, orgID) { + if !hs.validateUsingOrg(c.Req.Context(), c.UserId, orgID) { return response.Error(401, "Not a valid organization", nil) } cmd := models.SetUsingOrgCommand{UserId: c.UserId, OrgId: orgID} - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.SetUsingOrg(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to change active organization", err) } @@ -236,22 +261,26 @@ func UserSetUsingOrg(c *models.ReqContext) response.Response { // GET /profile/switch-org/:id func (hs *HTTPServer) ChangeActiveOrgAndRedirectToHome(c *models.ReqContext) { - orgID := c.ParamsInt64(":id") + orgID, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + c.JsonApiErr(http.StatusBadRequest, "id is invalid", err) + return + } - if !validateUsingOrg(c.Req.Context(), c.UserId, orgID) { + if !hs.validateUsingOrg(c.Req.Context(), c.UserId, orgID) { hs.NotFoundHandler(c) } cmd := models.SetUsingOrgCommand{UserId: c.UserId, OrgId: orgID} - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.SetUsingOrg(c.Req.Context(), &cmd); err != nil { hs.NotFoundHandler(c) } c.Redirect(hs.Cfg.AppSubURL + "/") } -func ChangeUserPassword(c *models.ReqContext) response.Response { +func (hs *HTTPServer) ChangeUserPassword(c *models.ReqContext) response.Response { cmd := models.ChangeUserPasswordCommand{} if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) @@ -262,7 +291,7 @@ func ChangeUserPassword(c *models.ReqContext) response.Response { userQuery := models.GetUserByIdQuery{Id: c.UserId} - if err := bus.Dispatch(c.Req.Context(), &userQuery); err != nil { + if err := hs.SQLStore.GetUserById(c.Req.Context(), &userQuery); err != nil { return response.Error(500, "Could not read user from database", err) } @@ -285,7 +314,7 @@ func ChangeUserPassword(c *models.ReqContext) response.Response { return response.Error(500, "Failed to encode password", err) } - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.ChangeUserPassword(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to change user password", err) } @@ -297,8 +326,11 @@ func redirectToChangePassword(c *models.ReqContext) { c.Redirect("/profile/password", 302) } -func SetHelpFlag(c *models.ReqContext) response.Response { - flag := c.ParamsInt64(":id") +func (hs *HTTPServer) SetHelpFlag(c *models.ReqContext) response.Response { + flag, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "id is invalid", err) + } bitmask := &c.HelpFlags1 bitmask.AddFlag(models.HelpFlags1(flag)) @@ -308,20 +340,20 @@ func SetHelpFlag(c *models.ReqContext) response.Response { HelpFlags1: *bitmask, } - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.SetUserHelpFlag(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to update help flag", err) } return response.JSON(200, &util.DynMap{"message": "Help flag set", "helpFlags1": cmd.HelpFlags1}) } -func ClearHelpFlags(c *models.ReqContext) response.Response { +func (hs *HTTPServer) ClearHelpFlags(c *models.ReqContext) response.Response { cmd := models.SetUserHelpFlagCommand{ UserId: c.UserId, HelpFlags1: models.HelpFlags1(0), } - if err := bus.Dispatch(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.SetUserHelpFlag(c.Req.Context(), &cmd); err != nil { return response.Error(500, "Failed to update help flag", err) } diff --git a/pkg/api/user_test.go b/pkg/api/user_test.go index 7d2370c2030..e9aa77d46c3 100644 --- a/pkg/api/user_test.go +++ b/pkg/api/user_test.go @@ -2,18 +2,23 @@ package api import ( "context" + "encoding/json" "fmt" "net/http" "testing" "time" + "github.com/grafana/grafana/pkg/api/dtos" + "github.com/grafana/grafana/pkg/services/login/authinfoservice" "github.com/grafana/grafana/pkg/services/searchusers/filters" + "github.com/grafana/grafana/pkg/services/secrets/database" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" + "golang.org/x/oauth2" "github.com/grafana/grafana/pkg/services/searchusers" - "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" @@ -23,10 +28,11 @@ import ( func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { settings := setting.NewCfg() - hs := &HTTPServer{Cfg: settings} - sqlStore := sqlstore.InitTestDB(t) - hs.SQLStore = sqlStore + hs := &HTTPServer{ + Cfg: settings, + SQLStore: sqlStore, + } mockResult := models.SearchUserQueryResult{ Users: []*models.UserSearchHitDTO{ @@ -36,60 +42,67 @@ func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { TotalCount: 2, } - loggedInUserScenario(t, "When calling GET on", "api/users/:id", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "api/users/1", "api/users/:id", func(sc *scenarioContext) { fakeNow := time.Date(2019, 2, 11, 17, 30, 40, 0, time.UTC) - bus.AddHandler("test", func(ctx context.Context, query *models.GetUserProfileQuery) error { - query.Result = models.UserProfileDTO{ - Id: int64(1), - Email: "daniel@grafana.com", - Name: "Daniel", - Login: "danlee", - OrgId: int64(2), - IsGrafanaAdmin: true, - IsDisabled: false, - IsExternal: false, - UpdatedAt: fakeNow, - CreatedAt: fakeNow, - } - return nil - }) + secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) + srv := authinfoservice.ProvideAuthInfoService(bus.New(), sqlStore, &authinfoservice.OSSUserProtectionImpl{}, secretsService) + hs.authInfoService = srv - bus.AddHandler("test", func(ctx context.Context, query *models.GetAuthInfoQuery) error { - query.Result = &models.UserAuth{ - AuthModule: models.AuthModuleLDAP, - } - return nil - }) + createUserCmd := models.CreateUserCommand{ + Email: fmt.Sprint("user", "@test.com"), + Name: "user", + Login: "loginuser", + IsAdmin: true, + } + user, err := sqlStore.CreateUser(context.Background(), createUserCmd) + require.Nil(t, err) sc.handlerFunc = hs.GetUserByID - avatarUrl := dtos.GetGravatarUrl("daniel@grafana.com") - sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() - expected := fmt.Sprintf(` - { - "id": 1, - "email": "daniel@grafana.com", - "name": "Daniel", - "login": "danlee", - "theme": "", - "orgId": 2, - "isGrafanaAdmin": true, - "isDisabled": false, - "isExternal": true, - "authLabels": [ - "LDAP" - ], - "avatarUrl": "%s", - "updatedAt": "2019-02-11T17:30:40Z", - "createdAt": "2019-02-11T17:30:40Z" - } - `, avatarUrl) + token := &oauth2.Token{ + AccessToken: "testaccess", + RefreshToken: "testrefresh", + Expiry: time.Now(), + TokenType: "Bearer", + } + idToken := "testidtoken" + token = token.WithExtra(map[string]interface{}{"id_token": idToken}) + query := &models.GetUserByAuthInfoQuery{Login: "loginuser", AuthModule: "test", AuthId: "test"} + cmd := &models.UpdateAuthInfoCommand{ + UserId: user.Id, + AuthId: query.AuthId, + AuthModule: query.AuthModule, + OAuthToken: token, + } + err = srv.UpdateAuthInfo(context.Background(), cmd) + require.NoError(t, err) + avatarUrl := dtos.GetGravatarUrl("@test.com") + sc.fakeReqWithParams("GET", sc.url, map[string]string{"id": fmt.Sprintf("%v", user.Id)}).exec() + expected := models.UserProfileDTO{ + Id: 1, + Email: "user@test.com", + Name: "user", + Login: "loginuser", + OrgId: 1, + IsGrafanaAdmin: true, + AuthLabels: []string{}, + CreatedAt: fakeNow, + UpdatedAt: fakeNow, + AvatarUrl: avatarUrl, + } + + var resp models.UserProfileDTO require.Equal(t, http.StatusOK, sc.resp.Code) - require.JSONEq(t, expected, sc.resp.Body.String()) + err = json.Unmarshal(sc.resp.Body.Bytes(), &resp) + require.NoError(t, err) + resp.CreatedAt = fakeNow + resp.UpdatedAt = fakeNow + resp.AvatarUrl = avatarUrl + require.EqualValues(t, expected, resp) }) - loggedInUserScenario(t, "When calling GET on", "/api/users/lookup", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "/api/users/lookup", "/api/users/lookup", func(sc *scenarioContext) { fakeNow := time.Date(2019, 2, 11, 17, 30, 40, 0, time.UTC) bus.AddHandler("test", func(ctx context.Context, query *models.GetUserByLoginQuery) error { require.Equal(t, "danlee", query.LoginOrEmail) @@ -109,33 +122,28 @@ func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { return nil }) + createUserCmd := models.CreateUserCommand{ + Email: fmt.Sprint("admin", "@test.com"), + Name: "admin", + Login: "admin", + IsAdmin: true, + } + _, err := sqlStore.CreateUser(context.Background(), createUserCmd) + require.Nil(t, err) - sc.handlerFunc = GetUserByLoginOrEmail - sc.fakeReqWithParams("GET", sc.url, map[string]string{"loginOrEmail": "danlee"}).exec() - - expected := ` - { - "id": 1, - "email": "daniel@grafana.com", - "name": "Daniel", - "login": "danlee", - "theme": "light", - "orgId": 2, - "isGrafanaAdmin": true, - "isDisabled": false, - "authLabels": null, - "isExternal": false, - "avatarUrl": "", - "updatedAt": "2019-02-11T17:30:40Z", - "createdAt": "2019-02-11T17:30:40Z" - } - ` + sc.handlerFunc = hs.GetUserByLoginOrEmail + sc.fakeReqWithParams("GET", sc.url, map[string]string{"loginOrEmail": "admin@test.com"}).exec() + var resp models.UserProfileDTO require.Equal(t, http.StatusOK, sc.resp.Code) - require.JSONEq(t, expected, sc.resp.Body.String()) + err = json.Unmarshal(sc.resp.Body.Bytes(), &resp) + require.NoError(t, err) + require.Equal(t, "admin", resp.Login) + require.Equal(t, "admin@test.com", resp.Email) + require.True(t, resp.IsGrafanaAdmin) }) - loggedInUserScenario(t, "When calling GET on", "/api/users", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "/api/users", "/api/users", func(sc *scenarioContext) { var sentLimit int var sendPage int bus.AddHandler("test", func(ctx context.Context, query *models.SearchUsersQuery) error { @@ -159,7 +167,7 @@ func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { assert.Equal(t, 2, len(respJSON.MustArray())) }) - loggedInUserScenario(t, "When calling GET with page and limit querystring parameters on", "/api/users", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET with page and limit querystring parameters on", "/api/users", "/api/users", func(sc *scenarioContext) { var sentLimit int var sendPage int bus.AddHandler("test", func(ctx context.Context, query *models.SearchUsersQuery) error { @@ -179,7 +187,7 @@ func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { assert.Equal(t, 2, sendPage) }) - loggedInUserScenario(t, "When calling GET on", "/api/users/search", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET on", "/api/users/search", "/api/users/search", func(sc *scenarioContext) { var sentLimit int var sendPage int bus.AddHandler("test", func(ctx context.Context, query *models.SearchUsersQuery) error { @@ -205,7 +213,7 @@ func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { assert.Equal(t, 2, len(respJSON.Get("users").MustArray())) }) - loggedInUserScenario(t, "When calling GET with page and perpage querystring parameters on", "/api/users/search", func(sc *scenarioContext) { + loggedInUserScenario(t, "When calling GET with page and perpage querystring parameters on", "/api/users/search", "/api/users/search", func(sc *scenarioContext) { var sentLimit int var sendPage int bus.AddHandler("test", func(ctx context.Context, query *models.SearchUsersQuery) error { diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go index 6a363289eb4..e3f207ef814 100644 --- a/pkg/bus/bus.go +++ b/pkg/bus/bus.go @@ -7,8 +7,9 @@ import ( "reflect" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/setting" - "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" ) // HandlerFunc defines a handler function interface. @@ -55,9 +56,11 @@ type InProcBus struct { listeners map[string][]HandlerFunc listenersWithCtx map[string][]HandlerFunc txMng TransactionManager + tracer tracing.Tracer } -func ProvideBus() *InProcBus { +func ProvideBus(tracer tracing.Tracer) *InProcBus { + globalBus.tracer = tracer return globalBus } @@ -71,7 +74,7 @@ var globalBus = New() // New initialize the bus func New() *InProcBus { - return &InProcBus{ + bus := &InProcBus{ logger: log.New("bus"), handlers: make(map[string]HandlerFunc), handlersWithCtx: make(map[string]HandlerFunc), @@ -79,6 +82,8 @@ func New() *InProcBus { listenersWithCtx: make(map[string][]HandlerFunc), txMng: &noopTransactionManager{}, } + bus.tracer = tracing.InitializeForBus() + return bus } // Want to get rid of global bus @@ -95,10 +100,10 @@ func (b *InProcBus) SetTransactionManager(tm TransactionManager) { func (b *InProcBus) Dispatch(ctx context.Context, msg Msg) error { var msgName = reflect.TypeOf(msg).Elem().Name() - span, ctx := opentracing.StartSpanFromContext(ctx, "bus - "+msgName) - defer span.Finish() + ctx, span := b.tracer.Start(ctx, "bus - "+msgName) + defer span.End() - span.SetTag("msg", msgName) + span.SetAttributes("msg", msgName, attribute.Key("msg").String(msgName)) withCtx := true var handler = b.handlersWithCtx[msgName] @@ -148,11 +153,10 @@ func (b *InProcBus) Publish(ctx context.Context, msg Msg) error { return err } } + _, span := b.tracer.Start(ctx, "bus - "+msgName) + defer span.End() - span, _ := opentracing.StartSpanFromContext(ctx, "bus - "+msgName) - defer span.Finish() - - span.SetTag("msg", msgName) + span.SetAttributes("msg", msgName, attribute.Key("msg").String(msgName)) return nil } diff --git a/pkg/bus/bus_test.go b/pkg/bus/bus_test.go index c22f4cf79b1..05789ef1f8b 100644 --- a/pkg/bus/bus_test.go +++ b/pkg/bus/bus_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/stretchr/testify/require" ) @@ -15,6 +16,9 @@ type testQuery struct { func TestDispatch(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -23,7 +27,8 @@ func TestDispatch(t *testing.T) { return nil }) - err := bus.Dispatch(context.Background(), &testQuery{}) + require.NoError(t, err) + err = bus.Dispatch(context.Background(), &testQuery{}) require.NoError(t, err) require.True(t, invoked, "expected handler to be called") @@ -31,14 +36,20 @@ func TestDispatch(t *testing.T) { func TestDispatch_NoRegisteredHandler(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer - err := bus.Dispatch(context.Background(), &testQuery{}) + err = bus.Dispatch(context.Background(), &testQuery{}) require.Equal(t, err, ErrHandlerNotFound, "expected bus to return HandlerNotFound since no handler is registered") } func TestDispatch_ContextHandler(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -47,7 +58,7 @@ func TestDispatch_ContextHandler(t *testing.T) { return nil }) - err := bus.Dispatch(context.Background(), &testQuery{}) + err = bus.Dispatch(context.Background(), &testQuery{}) require.NoError(t, err) require.True(t, invoked, "expected handler to be called") @@ -55,6 +66,9 @@ func TestDispatch_ContextHandler(t *testing.T) { func TestDispatchCtx(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -63,7 +77,7 @@ func TestDispatchCtx(t *testing.T) { return nil }) - err := bus.Dispatch(context.Background(), &testQuery{}) + err = bus.Dispatch(context.Background(), &testQuery{}) require.NoError(t, err) require.True(t, invoked, "expected handler to be called") @@ -71,6 +85,9 @@ func TestDispatchCtx(t *testing.T) { func TestDispatchCtx_NoContextHandler(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -79,7 +96,7 @@ func TestDispatchCtx_NoContextHandler(t *testing.T) { return nil }) - err := bus.Dispatch(context.Background(), &testQuery{}) + err = bus.Dispatch(context.Background(), &testQuery{}) require.NoError(t, err) require.True(t, invoked, "expected handler to be called") @@ -87,14 +104,20 @@ func TestDispatchCtx_NoContextHandler(t *testing.T) { func TestDispatchCtx_NoRegisteredHandler(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer - err := bus.Dispatch(context.Background(), &testQuery{}) + err = bus.Dispatch(context.Background(), &testQuery{}) require.Equal(t, err, ErrHandlerNotFound, "expected bus to return HandlerNotFound since no handler is registered") } func TestQuery(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer want := "hello from handler" @@ -105,7 +128,7 @@ func TestQuery(t *testing.T) { q := &testQuery{} - err := bus.Dispatch(context.Background(), q) + err = bus.Dispatch(context.Background(), q) require.NoError(t, err, "unable to dispatch query") require.Equal(t, want, q.Resp) @@ -113,17 +136,23 @@ func TestQuery(t *testing.T) { func TestQuery_HandlerReturnsError(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer bus.AddHandler(func(ctx context.Context, query *testQuery) error { return errors.New("handler error") }) - err := bus.Dispatch(context.Background(), &testQuery{}) + err = bus.Dispatch(context.Background(), &testQuery{}) require.Error(t, err, "expected error but got none") } func TestEventPublish(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -132,7 +161,7 @@ func TestEventPublish(t *testing.T) { return nil }) - err := bus.Publish(context.Background(), &testQuery{}) + err = bus.Publish(context.Background(), &testQuery{}) require.NoError(t, err, "unable to publish event") require.True(t, invoked) @@ -140,13 +169,19 @@ func TestEventPublish(t *testing.T) { func TestEventPublish_NoRegisteredListener(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer - err := bus.Publish(context.Background(), &testQuery{}) + err = bus.Publish(context.Background(), &testQuery{}) require.NoError(t, err, "unable to publish event") } func TestEventCtxPublishCtx(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -155,7 +190,7 @@ func TestEventCtxPublishCtx(t *testing.T) { return nil }) - err := bus.Publish(context.Background(), &testQuery{}) + err = bus.Publish(context.Background(), &testQuery{}) require.NoError(t, err, "unable to publish event") require.True(t, invoked) @@ -163,13 +198,19 @@ func TestEventCtxPublishCtx(t *testing.T) { func TestEventPublishCtx_NoRegisteredListener(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer - err := bus.Publish(context.Background(), &testQuery{}) + err = bus.Publish(context.Background(), &testQuery{}) require.NoError(t, err, "unable to publish event") } func TestEventPublishCtx(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -178,7 +219,7 @@ func TestEventPublishCtx(t *testing.T) { return nil }) - err := bus.Publish(context.Background(), &testQuery{}) + err = bus.Publish(context.Background(), &testQuery{}) require.NoError(t, err, "unable to publish event") require.True(t, invoked) @@ -186,6 +227,9 @@ func TestEventPublishCtx(t *testing.T) { func TestEventCtxPublish(t *testing.T) { bus := New() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + bus.tracer = tracer var invoked bool @@ -194,7 +238,7 @@ func TestEventCtxPublish(t *testing.T) { return nil }) - err := bus.Publish(context.Background(), &testQuery{}) + err = bus.Publish(context.Background(), &testQuery{}) require.NoError(t, err, "unable to publish event") require.True(t, invoked) diff --git a/pkg/cmd/grafana-cli/commands/commands.go b/pkg/cmd/grafana-cli/commands/commands.go index cabaa44f3fa..9ae1f6e7f52 100644 --- a/pkg/cmd/grafana-cli/commands/commands.go +++ b/pkg/cmd/grafana-cli/commands/commands.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/runner" "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/sqlstore/migrations" "github.com/grafana/grafana/pkg/setting" @@ -50,7 +51,11 @@ func runDbCommand(command func(commandLine utils.CommandLine, sqlStore *sqlstore return errutil.Wrap("failed to load configuration", err) } - sqlStore, err := sqlstore.ProvideService(cfg, nil, bus.GetBus(), &migrations.OSSMigrations{}) + tracer, err := tracing.ProvideService(cfg) + if err != nil { + return errutil.Wrap("failed to initialize tracer service", err) + } + sqlStore, err := sqlstore.ProvideService(cfg, nil, bus.GetBus(), &migrations.OSSMigrations{}, tracer) if err != nil { return errutil.Wrap("failed to initialize SQL store", err) } diff --git a/pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_secrets.go b/pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_secrets.go index 020eccfa864..9994dddd9e6 100644 --- a/pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_secrets.go +++ b/pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_secrets.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/runner" "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -178,7 +179,7 @@ func (s alertingSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm } func ReEncryptSecrets(_ utils.CommandLine, runner runner.Runner) error { - if !runner.SettingsProvider.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle) { + if !runner.SettingsProvider.IsFeatureToggleEnabled(featuremgmt.FlagEnvelopeEncryption) { logger.Warn("Envelope encryption is not enabled, quitting...") return nil } diff --git a/pkg/cmd/grafana-cli/runner/wire.go b/pkg/cmd/grafana-cli/runner/wire.go index 41bd04de809..665bd0c48d0 100644 --- a/pkg/cmd/grafana-cli/runner/wire.go +++ b/pkg/cmd/grafana-cli/runner/wire.go @@ -10,7 +10,9 @@ import ( "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/localcache" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/infra/usagestats" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/secrets" secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -22,7 +24,10 @@ import ( var wireSet = wire.NewSet( New, localcache.ProvideService, + tracing.ProvideService, bus.ProvideBus, + featuremgmt.ProvideManagerService, + featuremgmt.ProvideToggles, wire.Bind(new(bus.Bus), new(*bus.InProcBus)), sqlstore.ProvideService, wire.InterfaceValue(new(usagestats.Service), noOpUsageStats{}), @@ -71,6 +76,6 @@ func (noOpRouteRegister) Group(string, func(routing.RouteRegister), ...web.Handl func (noOpRouteRegister) Insert(string, func(routing.RouteRegister), ...web.Handler) {} -func (noOpRouteRegister) Register(routing.Router) {} +func (noOpRouteRegister) Register(routing.Router, ...routing.RegisterNamedMiddleware) {} func (noOpRouteRegister) Reset() {} diff --git a/pkg/expr/mathexp/funcs.go b/pkg/expr/mathexp/funcs.go index a2deb4387f8..681d3d9afbd 100644 --- a/pkg/expr/mathexp/funcs.go +++ b/pkg/expr/mathexp/funcs.go @@ -53,6 +53,21 @@ var builtins = map[string]parse.Func{ VariantReturn: true, F: isNumber, }, + "round": { + Args: []parse.ReturnType{parse.TypeVariantSet}, + VariantReturn: true, + F: round, + }, + "ceil": { + Args: []parse.ReturnType{parse.TypeVariantSet}, + VariantReturn: true, + F: ceil, + }, + "floor": { + Args: []parse.ReturnType{parse.TypeVariantSet}, + VariantReturn: true, + F: floor, + }, } // abs returns the absolute value for each result in NumberSet, SeriesSet, or Scalar @@ -253,3 +268,42 @@ func perNullableFloat(e *State, val Value, floatF func(x *float64) *float64) (Va return newVal, nil } + +// round returns the rounded value for each result in NumberSet, SeriesSet, or Scalar +func round(e *State, varSet Results) (Results, error) { + newRes := Results{} + for _, res := range varSet.Values { + newVal, err := perFloat(e, res, math.Round) + if err != nil { + return newRes, err + } + newRes.Values = append(newRes.Values, newVal) + } + return newRes, nil +} + +// ceil returns the rounded up value for each result in NumberSet, SeriesSet, or Scalar +func ceil(e *State, varSet Results) (Results, error) { + newRes := Results{} + for _, res := range varSet.Values { + newVal, err := perFloat(e, res, math.Ceil) + if err != nil { + return newRes, err + } + newRes.Values = append(newRes.Values, newVal) + } + return newRes, nil +} + +// floor returns the rounded down value for each result in NumberSet, SeriesSet, or Scalar +func floor(e *State, varSet Results) (Results, error) { + newRes := Results{} + for _, res := range varSet.Values { + newVal, err := perFloat(e, res, math.Floor) + if err != nil { + return newRes, err + } + newRes.Values = append(newRes.Values, newVal) + } + return newRes, nil +} diff --git a/pkg/expr/mathexp/reduce.go b/pkg/expr/mathexp/reduce.go index f306cd24a3d..be8c7816c36 100644 --- a/pkg/expr/mathexp/reduce.go +++ b/pkg/expr/mathexp/reduce.go @@ -69,6 +69,17 @@ func Count(fv *Float64Field) *float64 { return &f } +func Last(fv *Float64Field) *float64 { + var f float64 + if fv.Len() == 0 { + f = math.NaN() + return &f + } + v := fv.GetValue(fv.Len() - 1) + f = *v + return &f +} + // Reduce turns the Series into a Number based on the given reduction function func (s Series) Reduce(refID, rFunc string) (Number, error) { var l data.Labels @@ -90,6 +101,8 @@ func (s Series) Reduce(refID, rFunc string) (Number, error) { f = Max(&floatField) case "count": f = Count(&floatField) + case "last": + f = Last(&floatField) default: return number, fmt.Errorf("reduction %v not implemented", rFunc) } diff --git a/pkg/expr/mathexp/reduce_test.go b/pkg/expr/mathexp/reduce_test.go index 7d83c123b71..2bd13780018 100644 --- a/pkg/expr/mathexp/reduce_test.go +++ b/pkg/expr/mathexp/reduce_test.go @@ -214,6 +214,19 @@ func TestSeriesReduce(t *testing.T) { }, }, }, + { + name: "last empty series", + red: "last", + varToReduce: "A", + vars: seriesEmpty, + errIs: require.NoError, + resultsIs: require.Equal, + results: Results{ + []Value{ + makeNumber("", nil, NaN), + }, + }, + }, } for _, tt := range tests { diff --git a/pkg/expr/nodes.go b/pkg/expr/nodes.go index 444a09be3a1..7a1f90c2bbe 100644 --- a/pkg/expr/nodes.go +++ b/pkg/expr/nodes.go @@ -254,8 +254,16 @@ func (dn *DSNode) Execute(ctx context.Context, vars mathexp.Vars, s *Service) (m } } + dataSource := dn.datasource.Type for _, frame := range qr.Frames { logger.Debug("expression datasource query (seriesSet)", "query", refID) + // Check for TimeSeriesTypeNot in InfluxDB queries. A data frame of this type will cause + // the WideToMany() function to error out, which results in unhealthy alerts. + // This check should be removed once inconsistencies in data source responses are solved. + if frame.TimeSeriesSchema().Type == data.TimeSeriesTypeNot && dataSource == models.DS_INFLUXDB { + logger.Warn("ignoring InfluxDB data frame due to missing numeric fields", "frame", frame) + continue + } series, err := WideToMany(frame) if err != nil { return mathexp.Results{}, err diff --git a/pkg/infra/httpclient/httpclientprovider/azure_middleware.go b/pkg/infra/httpclient/httpclientprovider/azure_middleware.go index ef6db9ac9ab..93beebc45b1 100644 --- a/pkg/infra/httpclient/httpclientprovider/azure_middleware.go +++ b/pkg/infra/httpclient/httpclientprovider/azure_middleware.go @@ -16,17 +16,11 @@ const azureMiddlewareName = "AzureAuthentication.Provider" func AzureMiddleware(cfg *setting.Cfg) httpclient.Middleware { return httpclient.NamedMiddlewareFunc(azureMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper { - if enabled, err := isAzureAuthenticationEnabled(opts.CustomOptions); err != nil { - return errorResponse(err) - } else if !enabled { - return next - } - credentials, err := getAzureCredentials(opts.CustomOptions) if err != nil { return errorResponse(err) } else if credentials == nil { - credentials = getDefaultAzureCredentials(cfg) + return next } tokenProvider, err := aztokenprovider.NewAzureAccessTokenProvider(cfg, credentials) @@ -49,17 +43,6 @@ func errorResponse(err error) http.RoundTripper { }) } -func isAzureAuthenticationEnabled(customOptions map[string]interface{}) (bool, error) { - if untypedValue, ok := customOptions["_azureAuth"]; !ok { - return false, nil - } else if value, ok := untypedValue.(bool); !ok { - err := fmt.Errorf("the field 'azureAuth' should be a bool") - return false, err - } else { - return value, nil - } -} - func getAzureCredentials(customOptions map[string]interface{}) (azcredentials.AzureCredentials, error) { if untypedValue, ok := customOptions["_azureCredentials"]; !ok { return nil, nil @@ -71,16 +54,6 @@ func getAzureCredentials(customOptions map[string]interface{}) (azcredentials.Az } } -func getDefaultAzureCredentials(cfg *setting.Cfg) azcredentials.AzureCredentials { - if cfg.Azure.ManagedIdentityEnabled { - return &azcredentials.AzureManagedIdentityCredentials{} - } else { - return &azcredentials.AzureClientSecretCredentials{ - AzureCloud: cfg.Azure.Cloud, - } - } -} - func getAzureEndpointResourceId(customOptions map[string]interface{}) (*url.URL, error) { var value string if untypedValue, ok := customOptions["azureEndpointResourceId"]; !ok { diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go index f1b4ec367aa..7aab8e0c8cb 100644 --- a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go @@ -8,6 +8,8 @@ import ( sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/metrics/metricutil" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" "github.com/mwitkow/go-conntrack" ) @@ -15,12 +17,12 @@ import ( var newProviderFunc = sdkhttpclient.NewProvider // New creates a new HTTP client provider with pre-configured middlewares. -func New(cfg *setting.Cfg) *sdkhttpclient.Provider { +func New(cfg *setting.Cfg, tracer tracing.Tracer, features featuremgmt.FeatureToggles) *sdkhttpclient.Provider { logger := log.New("httpclient") userAgent := fmt.Sprintf("Grafana/%s", cfg.BuildVersion) middlewares := []sdkhttpclient.Middleware{ - TracingMiddleware(logger), + TracingMiddleware(logger, tracer), DataSourceMetricsMiddleware(), SetUserAgentMiddleware(userAgent), sdkhttpclient.BasicAuthenticationMiddleware(), @@ -34,7 +36,7 @@ func New(cfg *setting.Cfg) *sdkhttpclient.Provider { setDefaultTimeoutOptions(cfg) - if cfg.FeatureToggles["httpclientprovider_azure_auth"] { + if features.IsEnabled(featuremgmt.FlagHttpclientproviderAzureAuth) { middlewares = append(middlewares, AzureMiddleware(cfg)) } diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go index c9a3bdd4d76..9fbafaa5342 100644 --- a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go @@ -4,6 +4,8 @@ import ( "testing" sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" ) @@ -19,7 +21,9 @@ func TestHTTPClientProvider(t *testing.T) { t.Cleanup(func() { newProviderFunc = origNewProviderFunc }) - _ = New(&setting.Cfg{SigV4AuthEnabled: false}) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + _ = New(&setting.Cfg{SigV4AuthEnabled: false}, tracer, featuremgmt.WithFeatures()) require.Len(t, providerOpts, 1) o := providerOpts[0] require.Len(t, o.Middlewares, 6) @@ -41,7 +45,9 @@ func TestHTTPClientProvider(t *testing.T) { t.Cleanup(func() { newProviderFunc = origNewProviderFunc }) - _ = New(&setting.Cfg{SigV4AuthEnabled: true}) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + _ = New(&setting.Cfg{SigV4AuthEnabled: true}, tracer, featuremgmt.WithFeatures()) require.Len(t, providerOpts, 1) o := providerOpts[0] require.Len(t, o.Middlewares, 7) diff --git a/pkg/infra/httpclient/httpclientprovider/tracing_middleware.go b/pkg/infra/httpclient/httpclientprovider/tracing_middleware.go index f95cc102dec..6531eeb4d75 100644 --- a/pkg/infra/httpclient/httpclientprovider/tracing_middleware.go +++ b/pkg/infra/httpclient/httpclientprovider/tracing_middleware.go @@ -1,12 +1,16 @@ package httpclientprovider import ( + "fmt" "net/http" + "strconv" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/ext" + "github.com/grafana/grafana/pkg/infra/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) const ( @@ -14,32 +18,26 @@ const ( httpContentLengthTagKey = "http.content_length" ) -func TracingMiddleware(logger log.Logger) httpclient.Middleware { +func TracingMiddleware(logger log.Logger, tracer tracing.Tracer) httpclient.Middleware { return httpclient.NamedMiddlewareFunc(TracingMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper { return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { - span, ctx := opentracing.StartSpanFromContext(req.Context(), "HTTP Outgoing Request") - defer span.Finish() + ctx, span := tracer.Start(req.Context(), "HTTP Outgoing Request", trace.WithSpanKind(trace.SpanKindClient)) + defer span.End() req = req.WithContext(ctx) for k, v := range opts.Labels { - span.SetTag(k, v) - } - - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)); err != nil { - logger.Error("Failed to inject span context instance", "err", err) + span.SetAttributes(k, v, attribute.Key(k).String(v)) } + tracer.Inject(ctx, req.Header, span) res, err := next.RoundTrip(req) - ext.HTTPUrl.Set(span, req.URL.String()) - ext.HTTPMethod.Set(span, req.Method) - ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum) + span.SetAttributes("http.url", req.URL.String(), attribute.String("http.url", req.URL.String())) + span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method)) + // ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum) if err != nil { - ext.Error.Set(span, true) + span.RecordError(err) return res, err } @@ -47,12 +45,12 @@ func TracingMiddleware(logger log.Logger) httpclient.Middleware { // we avoid measuring contentlength less than zero because it indicates // that the content size is unknown. https://godoc.org/github.com/badu/http#Response if res.ContentLength > 0 { - span.SetTag(httpContentLengthTagKey, res.ContentLength) + span.SetAttributes(httpContentLengthTagKey, res.ContentLength, attribute.Key(httpContentLengthTagKey).Int64(res.ContentLength)) } - ext.HTTPStatusCode.Set(span, uint16(res.StatusCode)) + span.SetAttributes("http.status_code", res.StatusCode, attribute.Int("http.status_code", res.StatusCode)) if res.StatusCode >= 400 { - ext.Error.Set(span, true) + span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(res.StatusCode))) } } diff --git a/pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go index 4f9e29fdde8..5b3fdede7ee 100644 --- a/pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go +++ b/pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go @@ -8,21 +8,20 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/ext" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/stretchr/testify/require" - jaeger "github.com/uber/jaeger-client-go" ) func TestTracingMiddleware(t *testing.T) { - setupTracing(t) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) t.Run("GET request that returns 200 OK should start and capture span", func(t *testing.T) { finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{StatusCode: http.StatusOK, Request: req}, nil }) - mw := TracingMiddleware(log.New("test")) + mw := TracingMiddleware(log.New("test"), tracer) rt := mw.CreateMiddleware(httpclient.Options{ Labels: map[string]string{ "l1": "v1", @@ -44,24 +43,8 @@ func TestTracingMiddleware(t *testing.T) { require.NoError(t, res.Body.Close()) } - sp := opentracing.SpanFromContext(res.Request.Context()) + _, sp := tracer.Start(ctx, "test") require.NotNil(t, sp) - jsp, ok := sp.(*jaeger.Span) - require.True(t, ok) - require.Equal(t, "HTTP Outgoing Request", jsp.OperationName()) - require.Len(t, jsp.Tags(), 8) - expectedTags := opentracing.Tags{ - string(ext.HTTPMethod): http.MethodGet, - string(ext.HTTPStatusCode): uint16(http.StatusOK), - string(ext.HTTPUrl): "http://test.com/query", - "l1": "v1", - "l2": "v2", - jaeger.SamplerParamTagKey: true, - jaeger.SamplerTypeTagKey: jaeger.SamplerTypeConst, - string(ext.SpanKind): ext.SpanKindRPCClientEnum, - } - require.EqualValues(t, expectedTags, jsp.Tags()) - require.Contains(t, req.Header, "Uber-Trace-Id") }) t.Run("GET request that returns 400 Bad Request should start and capture span", func(t *testing.T) { @@ -69,7 +52,7 @@ func TestTracingMiddleware(t *testing.T) { return &http.Response{StatusCode: http.StatusBadRequest, Request: req}, nil }) - mw := TracingMiddleware(log.New("test")) + mw := TracingMiddleware(log.New("test"), tracer) rt := mw.CreateMiddleware(httpclient.Options{ Labels: map[string]string{ "l1": "v1", @@ -91,25 +74,8 @@ func TestTracingMiddleware(t *testing.T) { require.NoError(t, res.Body.Close()) } - sp := opentracing.SpanFromContext(res.Request.Context()) + _, sp := tracer.Start(res.Request.Context(), "test") require.NotNil(t, sp) - jsp, ok := sp.(*jaeger.Span) - require.True(t, ok) - require.Equal(t, "HTTP Outgoing Request", jsp.OperationName()) - require.Len(t, jsp.Tags(), 9) - expectedTags := opentracing.Tags{ - string(ext.Error): true, - string(ext.HTTPMethod): http.MethodGet, - string(ext.HTTPStatusCode): uint16(http.StatusBadRequest), - string(ext.HTTPUrl): "http://test.com/query", - "l1": "v1", - "l2": "v2", - jaeger.SamplerParamTagKey: true, - jaeger.SamplerTypeTagKey: jaeger.SamplerTypeConst, - string(ext.SpanKind): ext.SpanKindRPCClientEnum, - } - require.EqualValues(t, expectedTags, jsp.Tags()) - require.Contains(t, req.Header, "Uber-Trace-Id") }) t.Run("POST request that returns 200 OK should start and capture span", func(t *testing.T) { @@ -117,7 +83,7 @@ func TestTracingMiddleware(t *testing.T) { return &http.Response{StatusCode: http.StatusOK, Request: req, ContentLength: 10}, nil }) - mw := TracingMiddleware(log.New("test")) + mw := TracingMiddleware(log.New("test"), tracer) rt := mw.CreateMiddleware(httpclient.Options{ Labels: map[string]string{ "l1": "v1", @@ -139,35 +105,7 @@ func TestTracingMiddleware(t *testing.T) { require.NoError(t, res.Body.Close()) } - sp := opentracing.SpanFromContext(res.Request.Context()) + _, sp := tracer.Start(res.Request.Context(), "test") require.NotNil(t, sp) - jsp, ok := sp.(*jaeger.Span) - require.True(t, ok) - require.Equal(t, "HTTP Outgoing Request", jsp.OperationName()) - require.Len(t, jsp.Tags(), 9) - expectedTags := opentracing.Tags{ - httpContentLengthTagKey: int64(10), - string(ext.HTTPMethod): http.MethodPost, - string(ext.HTTPStatusCode): uint16(http.StatusOK), - string(ext.HTTPUrl): "http://test.com/query", - "l1": "v1", - "l2": "v2", - jaeger.SamplerParamTagKey: true, - jaeger.SamplerTypeTagKey: jaeger.SamplerTypeConst, - string(ext.SpanKind): ext.SpanKindRPCClientEnum, - } - require.EqualValues(t, expectedTags, jsp.Tags()) - require.Contains(t, req.Header, "Uber-Trace-Id") - }) -} - -func setupTracing(t *testing.T) { - t.Helper() - - tracer, closer := jaeger.NewTracer("test", jaeger.NewConstSampler(true), jaeger.NewNullReporter()) - opentracing.SetGlobalTracer(tracer) - t.Cleanup(func() { - require.NoError(t, closer.Close()) - opentracing.SetGlobalTracer(opentracing.NoopTracer{}) }) } diff --git a/pkg/infra/log/log.go b/pkg/infra/log/log.go index f8d074b33dd..ba531ca633f 100644 --- a/pkg/infra/log/log.go +++ b/pkg/infra/log/log.go @@ -13,13 +13,14 @@ import ( "time" gokitlog "github.com/go-kit/log" - "github.com/go-kit/log/term" "github.com/go-stack/stack" - "github.com/grafana/grafana/pkg/infra/log/level" - "github.com/grafana/grafana/pkg/util" - "github.com/grafana/grafana/pkg/util/errutil" "github.com/mattn/go-isatty" "gopkg.in/ini.v1" + + "github.com/grafana/grafana/pkg/infra/log/level" + "github.com/grafana/grafana/pkg/infra/log/term" + "github.com/grafana/grafana/pkg/util" + "github.com/grafana/grafana/pkg/util/errutil" ) var loggersToClose []DisposableHandler @@ -27,11 +28,20 @@ var loggersToReload []ReloadableHandler var filters map[string]level.Option var Root MultiLoggers +const ( + // top 7 calls in the stack are within logger + DefaultCallerDepth = 7 + CallerContextKey = "caller" +) + func init() { loggersToClose = make([]DisposableHandler, 0) loggersToReload = make([]ReloadableHandler, 0) filters = map[string]level.Option{} - Root.AddLogger(gokitlog.NewLogfmtLogger(os.Stderr), "info", filters) + + // Use console by default + format := getLogFormat("console") + Root.AddLogger(format(os.Stderr), "info", filters) } type LogWithFilters struct { @@ -99,24 +109,13 @@ func (ml MultiLoggers) Log(keyvals ...interface{}) error { return nil } -// we need to implement new function for multiloggers +// New creates a new logger from the existing one with additional context func (ml MultiLoggers) New(ctx ...interface{}) MultiLoggers { - var newloger MultiLoggers - for _, logWithFilter := range ml.loggers { - logWithFilter.val = gokitlog.With(logWithFilter.val, ctx) - if len(ctx) > 0 { - v, ok := logWithFilter.filters[ctx[0].(string)] - if ok { - logWithFilter.val = level.NewFilter(logWithFilter.val, v) - } else { - logWithFilter.val = level.NewFilter(logWithFilter.val, logWithFilter.maxLevel) - } - } - newloger.loggers = append(newloger.loggers, logWithFilter) - } - return newloger + return with(ml, gokitlog.With, ctx) } +// New creates MultiLoggers with the provided context and caller that is added as a suffix. +// The first element of the context must be the logger name func New(ctx ...interface{}) MultiLoggers { if len(ctx) == 0 { return Root @@ -136,6 +135,28 @@ func New(ctx ...interface{}) MultiLoggers { return newloger } +func with(loggers MultiLoggers, withFunc func(gokitlog.Logger, ...interface{}) gokitlog.Logger, ctx []interface{}) MultiLoggers { + if len(ctx) == 0 { + return loggers + } + var newloger MultiLoggers + for _, l := range loggers.loggers { + l.val = withFunc(l.val, ctx...) + newloger.loggers = append(newloger.loggers, l) + } + return newloger +} + +// WithPrefix adds context that will be added to the log message +func WithPrefix(loggers MultiLoggers, ctx ...interface{}) MultiLoggers { + return with(loggers, gokitlog.WithPrefix, ctx) +} + +// WithSuffix adds context that will be appended at the end of the log message +func WithSuffix(loggers MultiLoggers, ctx ...interface{}) MultiLoggers { + return with(loggers, gokitlog.WithSuffix, ctx) +} + var logLevels = map[string]level.Option{ "trace": level.AllowDebug(), "debug": level.AllowDebug(), @@ -183,39 +204,27 @@ func Stack(skip int) string { return s.String() } -type Formatedlogger func(w io.Writer) gokitlog.Logger - -func terminalColorFn(keyvals ...interface{}) term.FgBgColor { - for i := 0; i < len(keyvals)-1; i += 2 { - if keyvals[i] != level.Key() { - continue - } - switch keyvals[i+1] { - case "trace": - return term.FgBgColor{Fg: term.Gray} - case level.DebugValue(): - return term.FgBgColor{Fg: term.Gray} - case level.InfoValue(): - return term.FgBgColor{Fg: term.Green} - case level.WarnValue(): - return term.FgBgColor{Fg: term.Yellow} - case level.ErrorValue(): - return term.FgBgColor{Fg: term.Red} - case "crit": - return term.FgBgColor{Fg: term.Gray, Bg: term.DarkRed} - default: - return term.FgBgColor{} - } +// StackCaller returns a go-kit Valuer function that returns the stack trace from the place it is called. Argument `skip` allows skipping top n lines from the stack. +func StackCaller(skip int) gokitlog.Valuer { + return func() interface{} { + return Stack(skip + 1) } - return term.FgBgColor{} } +// Caller proxies go-kit/log Caller and returns a Valuer function that returns a file and line from a specified depth +// in the callstack +func Caller(depth int) gokitlog.Valuer { + return gokitlog.Caller(depth) +} + +type Formatedlogger func(w io.Writer) gokitlog.Logger + func getLogFormat(format string) Formatedlogger { switch format { case "console": if isatty.IsTerminal(os.Stdout.Fd()) { return func(w io.Writer) gokitlog.Logger { - return term.NewColorLogger(w, gokitlog.NewLogfmtLogger, terminalColorFn) + return term.NewTerminalLogger(w) } } return func(w io.Writer) gokitlog.Logger { diff --git a/pkg/infra/log/term/terminal_logger.go b/pkg/infra/log/term/terminal_logger.go new file mode 100644 index 00000000000..860ef27c7b6 --- /dev/null +++ b/pkg/infra/log/term/terminal_logger.go @@ -0,0 +1,274 @@ +package term + +import ( + "bytes" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "sync" + "time" + + gokitlog "github.com/go-kit/log" + "github.com/grafana/grafana/pkg/infra/log/level" +) + +const ( + timeFormat = "2006-01-02T15:04:05-0700" + termTimeFormat = "01-02|15:04:05" + floatFormat = 'f' + termMsgJust = 40 + errorKey = "LOG15_ERROR" +) + +type terminalLogger struct { + w io.Writer +} + +// NewTerminalLogger returns a logger that encodes keyvals to the Writer in +// a format optimized for human readability on a terminal with color-coded +// level output and terser human friendly timestamp. +// This format should only be used for interactive programs or while developing. +// +// [TIME] [LEVEL] MESSAGE key=value key=value ... +// +// Example: +// +// [May 16 20:58:45] [DBUG] remove route ns=haproxy addr=127.0.0.1:50002 +// +func NewTerminalLogger(w io.Writer) gokitlog.Logger { + return &terminalLogger{w} +} + +func (l terminalLogger) Log(keyvals ...interface{}) error { + r := getRecord(keyvals) + + b := &bytes.Buffer{} + lvl := strings.ToUpper(r.level.String()) + if r.color > 0 { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %s ", r.color, lvl, r.time.Format(termTimeFormat), r.msg) // lgtm[go/log-injection] + } else { + fmt.Fprintf(b, "[%s] [%s] %s ", lvl, r.time.Format(termTimeFormat), r.msg) // lgtm[go/log-injection] + } + + // try to justify the log output for short messages + if len(r.keyvals) > 0 && len(r.msg) < termMsgJust { + b.Write(bytes.Repeat([]byte{' '}, termMsgJust-len(r.msg))) + } + + // print the keys logfmt style + logfmt(b, r.keyvals, r.color) + _, err := l.w.Write(b.Bytes()) + return err +} + +type record struct { + msg string + time time.Time + level level.Value + color int + keyvals []interface{} +} + +func getRecord(keyvals ...interface{}) *record { + r := &record{ + color: 0, + level: level.InfoValue(), + keyvals: []interface{}{}, + } + + keyvals = keyvals[0].([]interface{}) + + if len(keyvals) == 0 { + return nil + } + if len(keyvals)%2 == 1 { + keyvals = append(keyvals, nil) + } + + for i := 0; i < len(keyvals); i += 2 { + k, v := keyvals[i], keyvals[i+1] + + if k == "t" { + t, ok := v.(fmt.Stringer) + if ok { + time, err := time.Parse("2006-01-02T15:04:05.999999999-0700", t.String()) + if err == nil { + r.time = time + continue + } + } + + t3, ok := v.(string) + if ok { + // from alerting: 2022-01-26T12:03:41.655107858-08:00 + time, err := time.Parse("2006-01-02T15:04:05.999999999-07:00", t3) + if err == nil { + r.time = time + continue + } + } + } + + if keyvals[i] == "msg" { + r.msg = v.(string) + continue + } + + if k == level.Key() { + switch v { + case "trace": + case level.DebugValue(): + r.level = level.DebugValue() + r.color = 36 + case level.InfoValue(): + r.level = level.InfoValue() + r.color = 32 + case level.WarnValue(): + r.level = level.WarnValue() + r.color = 33 + case level.ErrorValue(): + r.level = level.ErrorValue() + r.color = 31 + case "crit": + r.level = level.ErrorValue() + r.color = 35 + } + + continue + } + + r.keyvals = append(r.keyvals, k) + r.keyvals = append(r.keyvals, v) + } + + return r +} + +func logfmt(buf *bytes.Buffer, ctx []interface{}, color int) { + for i := 0; i < len(ctx); i += 2 { + if i != 0 { + buf.WriteByte(' ') + } + + k, ok := ctx[i].(string) + v := formatLogfmtValue(ctx[i+1]) + if !ok { + k, v = errorKey, formatLogfmtValue(k) + } + + // XXX: we should probably check that all of your key bytes aren't invalid + if color > 0 { + fmt.Fprintf(buf, "\x1b[%dm%s\x1b[0m=%s", color, k, v) // lgtm[go/log-injection] + } else { + buf.WriteString(k) + buf.WriteByte('=') + buf.WriteString(v) + } + } + + buf.WriteByte('\n') +} + +func formatShared(value interface{}) (result interface{}) { + defer func() { + if err := recover(); err != nil { + if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() { + result = "nil" + } else { + panic(err) + } + } + }() + + switch v := value.(type) { + case time.Time: + return v.Format(timeFormat) + + case error: + return v.Error() + + case fmt.Stringer: + return v.String() + + default: + return v + } +} + +// formatValue formats a value for serialization +func formatLogfmtValue(value interface{}) string { + if value == nil { + return "nil" + } + + if t, ok := value.(time.Time); ok { + // Performance optimization: No need for escaping since the provided + // timeFormat doesn't have any escape characters, and escaping is + // expensive. + return t.Format(timeFormat) + } + value = formatShared(value) + switch v := value.(type) { + case bool: + return strconv.FormatBool(v) + case float32: + return strconv.FormatFloat(float64(v), floatFormat, 3, 64) + case float64: + return strconv.FormatFloat(v, floatFormat, 3, 64) + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return fmt.Sprintf("%d", value) + case string: + return escapeString(v) + default: + return escapeString(fmt.Sprintf("%+v", value)) + } +} + +var stringBufPool = sync.Pool{ + New: func() interface{} { return new(bytes.Buffer) }, +} + +func escapeString(s string) string { + needsQuotes := false + needsEscape := false + for _, r := range s { + if r <= ' ' || r == '=' || r == '"' { + needsQuotes = true + } + if r == '\\' || r == '"' || r == '\n' || r == '\r' || r == '\t' { + needsEscape = true + } + } + if !needsEscape && !needsQuotes { + return s + } + e := stringBufPool.Get().(*bytes.Buffer) + e.WriteByte('"') + for _, r := range s { + switch r { + case '\\', '"': + e.WriteByte('\\') + e.WriteByte(byte(r)) + case '\n': + e.WriteString("\\n") + case '\r': + e.WriteString("\\r") + case '\t': + e.WriteString("\\t") + default: + e.WriteRune(r) + } + } + e.WriteByte('"') + var ret string + if needsQuotes { + ret = e.String() + } else { + ret = string(e.Bytes()[1 : e.Len()-1]) + } + e.Reset() + stringBufPool.Put(e) + return ret +} diff --git a/pkg/infra/tracing/opentelemetry_tracing.go b/pkg/infra/tracing/opentelemetry_tracing.go index 7f93c6de9ad..abe7fcb2fe2 100644 --- a/pkg/infra/tracing/opentelemetry_tracing.go +++ b/pkg/infra/tracing/opentelemetry_tracing.go @@ -2,13 +2,16 @@ package tracing import ( "context" + "net/http" "time" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/setting" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" tracesdk "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" @@ -16,25 +19,27 @@ import ( ) type Tracer interface { - Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) Run(context.Context) error + Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) + Inject(context.Context, http.Header, Span) } type Span interface { End() - SetAttributes(kv ...attribute.KeyValue) + SetAttributes(key string, value interface{}, kv attribute.KeyValue) + SetName(name string) + SetStatus(code codes.Code, description string) + RecordError(err error, options ...trace.EventOption) + AddEvents(keys []string, values []EventValue) } -var ( - GlobalTracer trace.Tracer -) - -type OpentelemetryTracingService struct { +type Opentelemetry struct { enabled bool address string log log.Logger tracerProvider *tracesdk.TracerProvider + tracer trace.Tracer Cfg *setting.Cfg } @@ -43,7 +48,12 @@ type OpentelemetrySpan struct { span trace.Span } -func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error { +type EventValue struct { + Str string + Num int64 +} + +func (ots *Opentelemetry) parseSettingsOpentelemetry() error { section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger") if err != nil { return err @@ -57,7 +67,7 @@ func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error { return nil } -func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerProvider, error) { +func (ots *Opentelemetry) initTracerProvider() (*tracesdk.TracerProvider, error) { // Create the Jaeger exporter exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address))) if err != nil { @@ -76,7 +86,7 @@ func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerPr return tp, nil } -func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error { +func (ots *Opentelemetry) initOpentelemetryTracer() error { tp, err := ots.initTracerProvider() if err != nil { return err @@ -89,15 +99,18 @@ func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error { } ots.tracerProvider = tp - GlobalTracer = otel.GetTracerProvider().Tracer("component-main") + ots.tracer = otel.GetTracerProvider().Tracer("component-main") return nil } -func (ots *OpentelemetryTracingService) Run(ctx context.Context) error { +func (ots *Opentelemetry) Run(ctx context.Context) error { <-ctx.Done() ots.log.Info("Closing tracing") + if ots.tracerProvider == nil { + return nil + } ctxShutdown, cancel := context.WithTimeout(ctx, time.Second*5) defer cancel() @@ -108,16 +121,47 @@ func (ots *OpentelemetryTracingService) Run(ctx context.Context) error { return nil } -func (ots *OpentelemetryTracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { - ctx, span := GlobalTracer.Start(ctx, spanName) - oSpan := OpentelemetrySpan{span: span} - return ctx, oSpan +func (ots *Opentelemetry) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { + ctx, span := ots.tracer.Start(ctx, spanName) + opentelemetrySpan := OpentelemetrySpan{ + span: span, + } + return ctx, opentelemetrySpan +} + +func (ots *Opentelemetry) Inject(ctx context.Context, header http.Header, _ Span) { + otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(header)) } func (s OpentelemetrySpan) End() { s.span.End() } -func (s OpentelemetrySpan) SetAttributes(kv ...attribute.KeyValue) { - s.span.SetAttributes(kv...) +func (s OpentelemetrySpan) SetAttributes(key string, value interface{}, kv attribute.KeyValue) { + s.span.SetAttributes(kv) +} + +func (s OpentelemetrySpan) SetName(name string) { + s.span.SetName(name) +} + +func (s OpentelemetrySpan) SetStatus(code codes.Code, description string) { + s.span.SetStatus(code, description) +} + +func (s OpentelemetrySpan) RecordError(err error, options ...trace.EventOption) { + for _, o := range options { + s.span.RecordError(err, o) + } +} + +func (s OpentelemetrySpan) AddEvents(keys []string, values []EventValue) { + for i, v := range values { + if v.Num != 0 { + s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).String(v.Str))) + } + if v.Str != "" { + s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).Int64(v.Num))) + } + } } diff --git a/pkg/infra/tracing/test_helper.go b/pkg/infra/tracing/test_helper.go new file mode 100644 index 00000000000..88474c700a9 --- /dev/null +++ b/pkg/infra/tracing/test_helper.go @@ -0,0 +1,16 @@ +package tracing + +func InitializeTracerForTest() (Tracer, error) { + ots := &Opentelemetry{} + err := ots.initOpentelemetryTracer() + if err != nil { + return ots, err + } + return ots, err +} + +func InitializeForBus() Tracer { + ots := &Opentelemetry{} + _ = ots.initOpentelemetryTracer() + return ots +} diff --git a/pkg/infra/tracing/tracing.go b/pkg/infra/tracing/tracing.go index df1c01efdb1..95df49b4727 100644 --- a/pkg/infra/tracing/tracing.go +++ b/pkg/infra/tracing/tracing.go @@ -4,17 +4,21 @@ import ( "context" "fmt" "io" + "net/http" "os" "strings" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/setting" - "go.opentelemetry.io/otel/attribute" - trace "go.opentelemetry.io/otel/trace" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" + ol "github.com/opentracing/opentracing-go/log" jaegercfg "github.com/uber/jaeger-client-go/config" "github.com/uber/jaeger-client-go/zipkin" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + trace "go.opentelemetry.io/otel/trace" ) const ( @@ -23,7 +27,7 @@ const ( ) func ProvideService(cfg *setting.Cfg) (Tracer, error) { - ts := &TracingService{ + ts := &Opentracing{ Cfg: cfg, log: log.New("tracing"), } @@ -36,7 +40,7 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) { return ts, ts.initGlobalTracer() } - ots := &OpentelemetryTracingService{ + ots := &Opentelemetry{ Cfg: cfg, log: log.New("tracing"), } @@ -48,7 +52,7 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) { return ots, ots.initOpentelemetryTracer() } -type TracingService struct { +type Opentracing struct { enabled bool address string customTags map[string]string @@ -67,7 +71,7 @@ type OpentracingSpan struct { span opentracing.Span } -func (ts *TracingService) parseSettings() error { +func (ts *Opentracing) parseSettings() error { var section, err = ts.Cfg.Raw.GetSection("tracing.jaeger") if err != nil { return err @@ -94,7 +98,7 @@ func (ts *TracingService) parseSettings() error { return nil } -func (ts *TracingService) initJaegerCfg() (jaegercfg.Configuration, error) { +func (ts *Opentracing) initJaegerCfg() (jaegercfg.Configuration, error) { cfg := jaegercfg.Configuration{ ServiceName: "grafana", Disabled: !ts.enabled, @@ -116,7 +120,7 @@ func (ts *TracingService) initJaegerCfg() (jaegercfg.Configuration, error) { return cfg, nil } -func (ts *TracingService) initGlobalTracer() error { +func (ts *Opentracing) initGlobalTracer() error { cfg, err := ts.initJaegerCfg() if err != nil { return err @@ -151,11 +155,10 @@ func (ts *TracingService) initGlobalTracer() error { opentracing.SetGlobalTracer(tracer) ts.closer = closer - return nil } -func (ts *TracingService) Run(ctx context.Context) error { +func (ts *Opentracing) Run(ctx context.Context) error { <-ctx.Done() if ts.closer != nil { @@ -166,22 +169,60 @@ func (ts *TracingService) Run(ctx context.Context) error { return nil } -func (ts *TracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { +func (ts *Opentracing) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { span, ctx := opentracing.StartSpanFromContext(ctx, spanName) - oSpan := OpentracingSpan{ - span: span, + opentracingSpan := OpentracingSpan{span: span} + return ctx, opentracingSpan +} + +func (ts *Opentracing) Inject(ctx context.Context, header http.Header, span Span) { + opentracingSpan, ok := span.(OpentracingSpan) + if !ok { + logger.Error("Failed to cast opentracing span") + } + err := opentracing.GlobalTracer().Inject( + opentracingSpan.span.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(header)) + + if err != nil { + logger.Error("Failed to inject span context instance", "err", err) } - return ctx, oSpan } func (s OpentracingSpan) End() { s.span.Finish() } -func (s OpentracingSpan) SetAttributes(kv ...attribute.KeyValue) { - for k, v := range kv { - s.span.SetTag(fmt.Sprint(k), v) +func (s OpentracingSpan) SetAttributes(key string, value interface{}, kv attribute.KeyValue) { + s.span.SetTag(key, value) +} + +func (s OpentracingSpan) SetName(name string) { + s.span.SetOperationName(name) +} + +func (s OpentracingSpan) SetStatus(code codes.Code, description string) { + ext.Error.Set(s.span, true) +} + +func (s OpentracingSpan) RecordError(err error, options ...trace.EventOption) { + ext.Error.Set(s.span, true) +} + +func (s OpentracingSpan) AddEvents(keys []string, values []EventValue) { + fields := []ol.Field{} + for i, v := range values { + if v.Str != "" { + field := ol.String(keys[i], v.Str) + fields = append(fields, field) + } + if v.Num != 0 { + field := ol.Int64(keys[i], v.Num) + fields = append(fields, field) + } } + s.span.LogFields(fields...) } func splitTagSettings(input string) map[string]string { diff --git a/pkg/infra/tracing/tracing_test.go b/pkg/infra/tracing/tracing_test.go index fa97b5675bd..d40390b5276 100644 --- a/pkg/infra/tracing/tracing_test.go +++ b/pkg/infra/tracing/tracing_test.go @@ -42,7 +42,7 @@ func TestGroupSplit(t *testing.T) { } func TestInitJaegerCfg_Default(t *testing.T) { - ts := &TracingService{} + ts := &Opentracing{} cfg, err := ts.initJaegerCfg() require.NoError(t, err) @@ -50,7 +50,7 @@ func TestInitJaegerCfg_Default(t *testing.T) { } func TestInitJaegerCfg_Enabled(t *testing.T) { - ts := &TracingService{enabled: true} + ts := &Opentracing{enabled: true} cfg, err := ts.initJaegerCfg() require.NoError(t, err) @@ -66,7 +66,7 @@ func TestInitJaegerCfg_DisabledViaEnv(t *testing.T) { require.NoError(t, err) }() - ts := &TracingService{enabled: true} + ts := &Opentracing{enabled: true} cfg, err := ts.initJaegerCfg() require.NoError(t, err) @@ -81,7 +81,7 @@ func TestInitJaegerCfg_EnabledViaEnv(t *testing.T) { require.NoError(t, err) }() - ts := &TracingService{enabled: false} + ts := &Opentracing{enabled: false} cfg, err := ts.initJaegerCfg() require.NoError(t, err) @@ -96,7 +96,7 @@ func TestInitJaegerCfg_InvalidEnvVar(t *testing.T) { require.NoError(t, err) }() - ts := &TracingService{} + ts := &Opentracing{} _, err = ts.initJaegerCfg() require.EqualError(t, err, "cannot parse env var JAEGER_DISABLED=totallybogus: strconv.ParseBool: parsing \"totallybogus\": invalid syntax") } @@ -108,7 +108,7 @@ func TestInitJaegerCfg_EnabledViaHost(t *testing.T) { }() cfg := setting.NewCfg() - ts := &TracingService{Cfg: cfg} + ts := &Opentracing{Cfg: cfg} _, err := ts.Cfg.Raw.NewSection("tracing.jaeger") require.NoError(t, err) require.NoError(t, ts.parseSettings()) @@ -128,7 +128,7 @@ func TestInitJaegerCfg_EnabledViaHostPort(t *testing.T) { }() cfg := setting.NewCfg() - ts := &TracingService{Cfg: cfg} + ts := &Opentracing{Cfg: cfg} _, err := ts.Cfg.Raw.NewSection("tracing.jaeger") require.NoError(t, err) require.NoError(t, ts.parseSettings()) diff --git a/pkg/infra/usagestats/service/api.go b/pkg/infra/usagestats/service/api.go new file mode 100644 index 00000000000..be27e79c394 --- /dev/null +++ b/pkg/infra/usagestats/service/api.go @@ -0,0 +1,27 @@ +package service + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/middleware" + "github.com/grafana/grafana/pkg/models" +) + +const rootUrl = "/api/admin" + +func (uss *UsageStats) registerAPIEndpoints() { + uss.RouteRegister.Group(rootUrl, func(subrouter routing.RouteRegister) { + subrouter.Get("/usage-report-preview", middleware.ReqGrafanaAdmin, routing.Wrap(uss.getUsageReportPreview)) + }) +} + +func (uss *UsageStats) getUsageReportPreview(ctx *models.ReqContext) response.Response { + usageReport, err := uss.GetUsageReport(ctx.Req.Context()) + if err != nil { + return response.Error(http.StatusInternalServerError, "failed to get usage report", err) + } + + return response.JSON(http.StatusOK, usageReport) +} diff --git a/pkg/infra/usagestats/service/api_test.go b/pkg/infra/usagestats/service/api_test.go new file mode 100644 index 00000000000..62cf46d9d97 --- /dev/null +++ b/pkg/infra/usagestats/service/api_test.go @@ -0,0 +1,134 @@ +package service + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path" + "testing" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/web" + "github.com/stretchr/testify/require" +) + +func TestApi_getUsageStats(t *testing.T) { + type getUsageStatsTestCase struct { + desc string + expectedStatus int + expectedCall bool + IsGrafanaAdmin bool + enabled bool + } + tests := []getUsageStatsTestCase{ + { + desc: "expect usage stats", + enabled: true, + IsGrafanaAdmin: true, + expectedCall: true, + expectedStatus: 200, + }, + { + desc: "expect usage stat preview still there after disabling", + enabled: false, + IsGrafanaAdmin: true, + expectedCall: true, + expectedStatus: 200, + }, + { + desc: "expect http status 403 when not admin", + enabled: false, + IsGrafanaAdmin: false, + expectedCall: false, + expectedStatus: 403, + }, + } + + uss := createService(t, setting.Cfg{}) + uss.registerAPIEndpoints() + getSystemStatsWasCalled := false + + uss.Bus.AddHandler(func(ctx context.Context, query *models.GetSystemStatsQuery) error { + query.Result = &models.SystemStats{} + getSystemStatsWasCalled = true + return nil + }) + + uss.Bus.AddHandler(func(ctx context.Context, query *models.GetDataSourceStatsQuery) error { + query.Result = []*models.DataSourceStats{} + return nil + }) + + uss.Bus.AddHandler(func(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error { + query.Result = []*models.DataSource{} + return nil + }) + + uss.Bus.AddHandler(func(ctx context.Context, query *models.GetDataSourceAccessStatsQuery) error { + query.Result = []*models.DataSourceAccessStats{} + return nil + }) + + uss.Bus.AddHandler(func(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error { + query.Result = []*models.NotifierUsageStats{} + return nil + }) + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + getSystemStatsWasCalled = false + uss.Cfg.ReportingEnabled = tt.enabled + server := setupTestServer(t, &models.SignedInUser{OrgId: 1, IsGrafanaAdmin: tt.IsGrafanaAdmin}, uss) + + usageStats, recorder := getUsageStats(t, server) + require.Equal(t, tt.expectedCall, getSystemStatsWasCalled) + require.Equal(t, tt.expectedStatus, recorder.Code) + + if tt.expectedStatus == http.StatusOK { + require.NotNil(t, usageStats) + } + }) + } +} + +func getUsageStats(t *testing.T, server *web.Mux) (*models.SystemStats, *httptest.ResponseRecorder) { + req, err := http.NewRequest(http.MethodGet, "/api/admin/usage-report-preview", http.NoBody) + require.NoError(t, err) + recorder := httptest.NewRecorder() + server.ServeHTTP(recorder, req) + + var usageStats *models.SystemStats + if recorder.Code == http.StatusOK { + require.NoError(t, json.NewDecoder(recorder.Body).Decode(&usageStats)) + } + return usageStats, recorder +} + +func setupTestServer(t *testing.T, user *models.SignedInUser, service *UsageStats) *web.Mux { + server := web.New() + server.UseMiddleware(web.Renderer(path.Join(setting.StaticRootPath, "views"), "[[", "]]")) + server.Use(contextProvider(&testContext{user})) + service.RouteRegister.Register(server) + return server +} + +type testContext struct { + user *models.SignedInUser +} + +func contextProvider(tc *testContext) web.Handler { + return func(c *web.Context) { + signedIn := tc.user != nil + reqCtx := &models.ReqContext{ + Context: c, + SignedInUser: tc.user, + IsSignedIn: signedIn, + SkipCache: true, + Logger: log.New("test"), + } + c.Map(reqCtx) + } +} diff --git a/pkg/infra/usagestats/service/service.go b/pkg/infra/usagestats/service/service.go index e9300a83fc4..13ac15b2be1 100644 --- a/pkg/infra/usagestats/service/service.go +++ b/pkg/infra/usagestats/service/service.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/kvstore" "github.com/grafana/grafana/pkg/infra/log" @@ -22,6 +23,7 @@ type UsageStats struct { pluginStore plugins.Store SocialService social.Service kvStore *kvstore.NamespacedKVStore + RouteRegister routing.RouteRegister log log.MultiLoggers @@ -33,18 +35,22 @@ type UsageStats struct { } func ProvideService(cfg *setting.Cfg, bus bus.Bus, sqlStore *sqlstore.SQLStore, pluginStore plugins.Store, - socialService social.Service, kvStore kvstore.KVStore) *UsageStats { + socialService social.Service, kvStore kvstore.KVStore, routeRegister routing.RouteRegister, +) *UsageStats { s := &UsageStats{ Cfg: cfg, Bus: bus, SQLStore: sqlStore, oauthProviders: socialService.GetOAuthProviders(), + RouteRegister: routeRegister, pluginStore: pluginStore, kvStore: kvstore.WithNamespace(kvStore, 0, "infra.usagestats"), log: log.New("infra.usagestats"), startTime: time.Now(), } + s.registerAPIEndpoints() + return s } diff --git a/pkg/infra/usagestats/service/usage_stats_test.go b/pkg/infra/usagestats/service/usage_stats_test.go index fa9f53bae81..bbbae5c6644 100644 --- a/pkg/infra/usagestats/service/usage_stats_test.go +++ b/pkg/infra/usagestats/service/usage_stats_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/kvstore" @@ -619,5 +620,6 @@ func createService(t *testing.T, cfg setting.Cfg) *UsageStats { kvStore: kvstore.WithNamespace(kvstore.ProvideService(sqlStore), 0, "infra.usagestats"), log: log.New("infra.usagestats"), startTime: time.Now().Add(-1 * time.Minute), + RouteRegister: routing.NewRouteRegister(), } } diff --git a/pkg/login/brute_force_login_protection_test.go b/pkg/login/brute_force_login_protection_test.go index 53667404c1a..a0fd5be034e 100644 --- a/pkg/login/brute_force_login_protection_test.go +++ b/pkg/login/brute_force_login_protection_test.go @@ -62,6 +62,7 @@ func TestValidateLoginAttempts(t *testing.T) { withLoginAttempts(t, tc.loginAttempts) query := &models.LoginUserQuery{Username: "user", Cfg: tc.cfg} + err := validateLoginAttempts(context.Background(), query) require.Equal(t, tc.expected, err) }) diff --git a/pkg/login/social/generic_oauth.go b/pkg/login/social/generic_oauth.go index e20cee43fb4..2ad76148885 100644 --- a/pkg/login/social/generic_oauth.go +++ b/pkg/login/social/generic_oauth.go @@ -231,13 +231,19 @@ func (s *SocialGenericOAuth) extractFromToken(token *oauth2.Token) *UserInfoJson return nil } - var header map[string]string + var header map[string]interface{} if err := json.Unmarshal(headerBytes, &header); err != nil { s.log.Error("Error deserializing header", "error", err) return nil } - if compression, ok := header["zip"]; ok { + if compressionVal, exists := header["zip"]; exists { + compression, ok := compressionVal.(string) + if !ok { + s.log.Warn("Unknown compression algorithm") + return nil + } + if compression != "DEF" { s.log.Warn("Unknown compression algorithm", "algorithm", compression) return nil diff --git a/pkg/login/social/generic_oauth_test.go b/pkg/login/social/generic_oauth_test.go index b7df2e97cee..9be66f1e3ac 100644 --- a/pkg/login/social/generic_oauth_test.go +++ b/pkg/login/social/generic_oauth_test.go @@ -7,15 +7,16 @@ import ( "testing" "time" - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/infra/log/level" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/oauth2" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/log/level" ) func newLogger(name string, lev string) log.Logger { - logger := log.Root.New("logger", name) + logger := log.New(name) logger.AddLogger(logger, lev, map[string]level.Option{}) return logger } @@ -726,6 +727,14 @@ func TestPayloadCompression(t *testing.T) { }, ExpectedEmail: "john.doe@example.com", }, + { + Name: "Given a valid DEFLATE compressed id_token with numeric header, return userInfo", + OAuth2Extra: map[string]interface{}{ + // Generated from https://token.dev/ + "id_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiIsInZlciI6NH0.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTY0MjUxNjYwNSwiZXhwIjoxNjQyNTIwMjA1LCJlbWFpbCI6ImpvaG4uZG9lQGV4YW1wbGUuY29tIn0.ANndoPWIHNjKPG8na7UUq7nan1RgF8-ze8STU31RXcA", + }, + ExpectedEmail: "john.doe@example.com", + }, { Name: "Given an invalid DEFLATE compressed id_token, return nil", OAuth2Extra: map[string]interface{}{ diff --git a/pkg/middleware/auth_test.go b/pkg/middleware/auth_test.go index 6bb56ab55ad..dc66ae33b10 100644 --- a/pkg/middleware/auth_test.go +++ b/pkg/middleware/auth_test.go @@ -17,7 +17,6 @@ func TestMiddlewareAuth(t *testing.T) { middlewareScenario(t, "ReqSignIn true and unauthenticated request", func(t *testing.T, sc *scenarioContext) { sc.m.Get("/secure", reqSignIn, sc.defaultHandler) - sc.fakeReq("GET", "/secure").exec() assert.Equal(t, 302, sc.resp.Code) diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 861053d75d6..7cd7b958e81 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -47,7 +47,7 @@ func AddDefaultResponseHeaders(cfg *setting.Cfg) web.Handler { // addSecurityHeaders adds HTTP(S) response headers that enable various security protections in the client's browser. func addSecurityHeaders(w web.ResponseWriter, cfg *setting.Cfg) { - if (cfg.Protocol == setting.HTTPSScheme || cfg.Protocol == setting.HTTP2Scheme) && cfg.StrictTransportSecurity { + if cfg.StrictTransportSecurity { strictHeaderValues := []string{fmt.Sprintf("max-age=%v", cfg.StrictTransportSecurityMaxAge)} if cfg.StrictTransportSecurityPreload { strictHeaderValues = append(strictHeaderValues, "preload") diff --git a/pkg/middleware/middleware_jwt_auth_test.go b/pkg/middleware/middleware_jwt_auth_test.go index b6f73f3ceeb..770d7b1c802 100644 --- a/pkg/middleware/middleware_jwt_auth_test.go +++ b/pkg/middleware/middleware_jwt_auth_test.go @@ -5,11 +5,12 @@ import ( "errors" "testing" + "github.com/stretchr/testify/assert" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/contexthandler" "github.com/grafana/grafana/pkg/setting" - "github.com/stretchr/testify/assert" ) func TestMiddlewareJWTAuth(t *testing.T) { @@ -29,6 +30,10 @@ func TestMiddlewareJWTAuth(t *testing.T) { cfg.JWTAuthEmailClaim = "foo-email" } + configureAutoSignUp := func(cfg *setting.Cfg) { + cfg.JWTAuthAutoSignUp = true + } + token := "some-token" middlewareScenario(t, "Valid token with valid login claim", func(t *testing.T, sc *scenarioContext) { @@ -37,6 +42,7 @@ func TestMiddlewareJWTAuth(t *testing.T) { sc.jwtAuthService.VerifyProvider = func(ctx context.Context, token string) (models.JWTClaims, error) { verifiedToken = token return models.JWTClaims{ + "sub": myUsername, "foo-username": myUsername, }, nil } @@ -64,6 +70,7 @@ func TestMiddlewareJWTAuth(t *testing.T) { sc.jwtAuthService.VerifyProvider = func(ctx context.Context, token string) (models.JWTClaims, error) { verifiedToken = token return models.JWTClaims{ + "sub": myEmail, "foo-email": myEmail, }, nil } @@ -85,11 +92,72 @@ func TestMiddlewareJWTAuth(t *testing.T) { assert.Equal(t, myEmail, sc.context.Email) }, configure, configureEmailClaim) + middlewareScenario(t, "Valid token with no user and auto_sign_up disabled", func(t *testing.T, sc *scenarioContext) { + myEmail := "vladimir@example.com" + var verifiedToken string + sc.jwtAuthService.VerifyProvider = func(ctx context.Context, token string) (models.JWTClaims, error) { + verifiedToken = token + return models.JWTClaims{ + "sub": myEmail, + "name": "Vladimir Example", + "foo-email": myEmail, + }, nil + } + bus.AddHandler("get-sign-user", func(ctx context.Context, query *models.GetSignedInUserQuery) error { + return models.ErrUserNotFound + }) + + sc.fakeReq("GET", "/").withJWTAuthHeader(token).exec() + assert.Equal(t, verifiedToken, token) + assert.Equal(t, 401, sc.resp.Code) + assert.Equal(t, contexthandler.UserNotFound, sc.respJson["message"]) + }, configure, configureEmailClaim) + + middlewareScenario(t, "Valid token with no user and auto_sign_up enabled", func(t *testing.T, sc *scenarioContext) { + myEmail := "vladimir@example.com" + var verifiedToken string + sc.jwtAuthService.VerifyProvider = func(ctx context.Context, token string) (models.JWTClaims, error) { + verifiedToken = token + return models.JWTClaims{ + "sub": myEmail, + "name": "Vladimir Example", + "foo-email": myEmail, + }, nil + } + bus.AddHandler("get-sign-user", func(ctx context.Context, query *models.GetSignedInUserQuery) error { + query.Result = &models.SignedInUser{ + UserId: id, + OrgId: orgID, + Email: query.Email, + } + return nil + }) + bus.AddHandler("upsert-user", func(ctx context.Context, command *models.UpsertUserCommand) error { + command.Result = &models.User{ + Id: id, + Name: command.ExternalUser.Name, + Email: command.ExternalUser.Email, + } + return nil + }) + + sc.fakeReq("GET", "/").withJWTAuthHeader(token).exec() + assert.Equal(t, verifiedToken, token) + assert.Equal(t, 200, sc.resp.Code) + assert.True(t, sc.context.IsSignedIn) + assert.Equal(t, orgID, sc.context.OrgId) + assert.Equal(t, id, sc.context.UserId) + assert.Equal(t, myEmail, sc.context.Email) + }, configure, configureEmailClaim, configureAutoSignUp) + middlewareScenario(t, "Valid token without a login claim", func(t *testing.T, sc *scenarioContext) { var verifiedToken string sc.jwtAuthService.VerifyProvider = func(ctx context.Context, token string) (models.JWTClaims, error) { verifiedToken = token - return models.JWTClaims{"foo": "bar"}, nil + return models.JWTClaims{ + "sub": "baz", + "foo": "bar", + }, nil } sc.fakeReq("GET", "/").withJWTAuthHeader(token).exec() @@ -102,7 +170,10 @@ func TestMiddlewareJWTAuth(t *testing.T) { var verifiedToken string sc.jwtAuthService.VerifyProvider = func(ctx context.Context, token string) (models.JWTClaims, error) { verifiedToken = token - return models.JWTClaims{"foo": "bar"}, nil + return models.JWTClaims{ + "sub": "baz", + "foo": "bar", + }, nil } sc.fakeReq("GET", "/").withJWTAuthHeader(token).exec() diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index bdc6d04ab91..b05fc116a24 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/remotecache" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/login" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/auth" @@ -67,7 +68,6 @@ func TestMiddleWareSecurityHeaders(t *testing.T) { sc.fakeReq("GET", "/api/").exec() assert.Equal(t, "max-age=64000; preload; includeSubDomains", sc.resp.Header().Get("Strict-Transport-Security")) }, func(cfg *setting.Cfg) { - cfg.Protocol = setting.HTTPSScheme cfg.StrictTransportSecurity = true cfg.StrictTransportSecurityMaxAge = 64000 }) @@ -731,7 +731,9 @@ func getContextHandler(t *testing.T, cfg *setting.Cfg) *contexthandler.ContextHa userAuthTokenSvc := auth.NewFakeUserAuthTokenService() renderSvc := &fakeRenderService{} authJWTSvc := models.NewFakeJWTService() - return contexthandler.ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + return contexthandler.ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore, tracer) } type fakeRenderService struct { diff --git a/pkg/middleware/org_redirect.go b/pkg/middleware/org_redirect.go index 16463a0c68d..e6f8b638f52 100644 --- a/pkg/middleware/org_redirect.go +++ b/pkg/middleware/org_redirect.go @@ -44,7 +44,16 @@ func OrgRedirect(cfg *setting.Cfg) web.Handler { return } - newURL := fmt.Sprintf("%s%s?%s", cfg.AppURL, strings.TrimPrefix(c.Req.URL.Path, "/"), c.Req.URL.Query().Encode()) + urlParams := c.Req.URL.Query() + qs := urlParams.Encode() + + if urlParams.Has("kiosk") && urlParams.Get("kiosk") == "" { + urlParams.Del("kiosk") + qs = fmt.Sprintf("%s&kiosk", urlParams.Encode()) + } + + newURL := fmt.Sprintf("%s%s?%s", cfg.AppURL, strings.TrimPrefix(c.Req.URL.Path, "/"), qs) + c.Redirect(newURL, 302) } } diff --git a/pkg/middleware/org_redirect_test.go b/pkg/middleware/org_redirect_test.go index 4018ab9427c..4c4620843f6 100644 --- a/pkg/middleware/org_redirect_test.go +++ b/pkg/middleware/org_redirect_test.go @@ -7,33 +7,68 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestOrgRedirectMiddleware(t *testing.T) { - middlewareScenario(t, "when setting a correct org for the user", func(t *testing.T, sc *scenarioContext) { - sc.withTokenSessionCookie("token") - bus.AddHandler("test", func(ctx context.Context, query *models.SetUsingOrgCommand) error { - return nil + testCases := []struct { + desc string + input string + expStatus int + expLocation string + }{ + { + desc: "when setting a correct org for the user", + input: "/?orgId=3", + expStatus: 302, + expLocation: "/?orgId=3", + }, + { + desc: "when setting a correct org for the user with '&kiosk'", + input: "/?orgId=3&kiosk", + expStatus: 302, + expLocation: "/?orgId=3&kiosk", + }, + { + desc: "when setting a correct org for the user with '&kiosk=", + input: "/?kiosk=&orgId=3", + expStatus: 302, + expLocation: "/?orgId=3&kiosk", + }, + { + desc: "when setting a correct org for the user with '&kiosk=tv'", + input: "/?kiosk=tv&orgId=3", + expStatus: 302, + expLocation: "/?kiosk=tv&orgId=3", + }, + } + + for _, tc := range testCases { + middlewareScenario(t, tc.desc, func(t *testing.T, sc *scenarioContext) { + sc.withTokenSessionCookie("token") + bus.AddHandler("test", func(ctx context.Context, query *models.SetUsingOrgCommand) error { + return nil + }) + + bus.AddHandler("test", func(ctx context.Context, query *models.GetSignedInUserQuery) error { + query.Result = &models.SignedInUser{OrgId: 1, UserId: 12} + return nil + }) + + sc.userAuthTokenService.LookupTokenProvider = func(ctx context.Context, unhashedToken string) (*models.UserToken, error) { + return &models.UserToken{ + UserId: 0, + UnhashedToken: "", + }, nil + } + + sc.m.Get("/", sc.defaultHandler) + sc.fakeReq("GET", tc.input).exec() + + require.Equal(t, tc.expStatus, sc.resp.Code) + require.Equal(t, tc.expLocation, sc.resp.Header().Get("Location")) }) - - bus.AddHandler("test", func(ctx context.Context, query *models.GetSignedInUserQuery) error { - query.Result = &models.SignedInUser{OrgId: 1, UserId: 12} - return nil - }) - - sc.userAuthTokenService.LookupTokenProvider = func(ctx context.Context, unhashedToken string) (*models.UserToken, error) { - return &models.UserToken{ - UserId: 0, - UnhashedToken: "", - }, nil - } - - sc.m.Get("/", sc.defaultHandler) - sc.fakeReq("GET", "/?orgId=3").exec() - - assert.Equal(t, 302, sc.resp.Code) - }) + } middlewareScenario(t, "when setting an invalid org for user", func(t *testing.T, sc *scenarioContext) { sc.withTokenSessionCookie("token") @@ -56,6 +91,6 @@ func TestOrgRedirectMiddleware(t *testing.T) { sc.m.Get("/", sc.defaultHandler) sc.fakeReq("GET", "/?orgId=3").exec() - assert.Equal(t, 404, sc.resp.Code) + require.Equal(t, 404, sc.resp.Code) }) } diff --git a/pkg/middleware/request_metrics.go b/pkg/middleware/request_metrics.go index a29ef94f0a5..6429f95f318 100644 --- a/pkg/middleware/request_metrics.go +++ b/pkg/middleware/request_metrics.go @@ -7,7 +7,7 @@ import ( "time" "github.com/grafana/grafana/pkg/infra/metrics" - "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/web" "github.com/prometheus/client_golang/prometheus" cw "github.com/weaveworks/common/tracing" @@ -45,7 +45,7 @@ func init() { } // RequestMetrics is a middleware handler that instruments the request. -func RequestMetrics(cfg *setting.Cfg) func(handler string) web.Handler { +func RequestMetrics(features featuremgmt.FeatureToggles) func(handler string) web.Handler { return func(handler string) web.Handler { return func(res http.ResponseWriter, req *http.Request, c *web.Context) { rw := res.(web.ResponseWriter) @@ -60,7 +60,7 @@ func RequestMetrics(cfg *setting.Cfg) func(handler string) web.Handler { method := sanitizeMethod(req.Method) // enable histogram and disable summaries + counters for http requests. - if cfg.IsHTTPRequestHistogramDisabled() { + if features.IsEnabled(featuremgmt.FlagDisableHttpRequestHistogram) { duration := time.Since(now).Nanoseconds() / int64(time.Millisecond) metrics.MHttpRequestTotal.WithLabelValues(handler, code, method).Inc() metrics.MHttpRequestSummary.WithLabelValues(handler, code, method).Observe(float64(duration)) diff --git a/pkg/middleware/request_tracing.go b/pkg/middleware/request_tracing.go index 19607010f7f..f877a1f37ef 100644 --- a/pkg/middleware/request_tracing.go +++ b/pkg/middleware/request_tracing.go @@ -4,11 +4,16 @@ import ( "context" "fmt" "net/http" + "strconv" "strings" - opentracing "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/ext" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/web" ) @@ -36,7 +41,7 @@ func RouteOperationNameFromContext(ctx context.Context) (string, bool) { return "", false } -func RequestTracing() web.Handler { +func RequestTracing(tracer tracing.Tracer) web.Handler { return func(res http.ResponseWriter, req *http.Request, c *web.Context) { if strings.HasPrefix(c.Req.URL.Path, "/public/") || c.Req.URL.Path == "robots.txt" { @@ -46,11 +51,9 @@ func RequestTracing() web.Handler { rw := res.(web.ResponseWriter) - tracer := opentracing.GlobalTracer() - wireContext, _ := tracer.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header)) - span := tracer.StartSpan(fmt.Sprintf("HTTP %s %s", req.Method, req.URL.Path), ext.RPCServerOption(wireContext)) + wireContext := otel.GetTextMapPropagator().Extract(req.Context(), propagation.HeaderCarrier(req.Header)) + ctx, span := tracer.Start(req.Context(), fmt.Sprintf("HTTP %s %s", req.Method, req.URL.Path), trace.WithLinks(trace.LinkFromContext(wireContext))) - ctx := opentracing.ContextWithSpan(req.Context(), span) c.Req = req.WithContext(ctx) c.Map(c.Req) @@ -59,17 +62,17 @@ func RequestTracing() web.Handler { // Only call span.Finish when a route operation name have been set, // meaning that not set the span would not be reported. if routeOperation, exists := RouteOperationNameFromContext(c.Req.Context()); exists { - defer span.Finish() - span.SetOperationName(fmt.Sprintf("HTTP %s %s", req.Method, routeOperation)) + defer span.End() + span.SetName(fmt.Sprintf("HTTP %s %s", req.Method, routeOperation)) } status := rw.Status() - ext.HTTPStatusCode.Set(span, uint16(status)) - ext.HTTPUrl.Set(span, req.RequestURI) - ext.HTTPMethod.Set(span, req.Method) + span.SetAttributes("http.status_code", status, attribute.Int("http.status_code", status)) + span.SetAttributes("http.url", req.RequestURI, attribute.String("http.url", req.RequestURI)) + span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method)) if status >= 400 { - ext.Error.Set(span, true) + span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(status))) } } } diff --git a/pkg/models/alert.go b/pkg/models/alert.go index b4da89d143c..452e0fa44c6 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -127,14 +127,6 @@ func (a *Alert) GetTagsFromSettings() []*Tag { return tags } -type SaveAlertsCommand struct { - DashboardId int64 - UserId int64 - OrgId int64 - - Alerts []*Alert -} - type PauseAlertCommand struct { OrgId int64 AlertIds []int64 diff --git a/pkg/models/datasource.go b/pkg/models/datasource.go index 005c4a1bbde..ba4184a848f 100644 --- a/pkg/models/datasource.go +++ b/pkg/models/datasource.go @@ -13,7 +13,15 @@ const ( DS_INFLUXDB_08 = "influxdb_08" DS_ES = "elasticsearch" DS_PROMETHEUS = "prometheus" + DS_ALERTMANAGER = "alertmanager" + DS_JAEGER = "jaeger" + DS_LOKI = "loki" + DS_OPENTSDB = "opentsdb" + DS_TEMPO = "tempo" + DS_ZIPKIN = "zipkin" DS_MYSQL = "mysql" + DS_POSTGRES = "postgres" + DS_MSSQL = "mssql" DS_ACCESS_DIRECT = "direct" DS_ACCESS_PROXY = "proxy" DS_ES_OPEN_DISTRO = "grafana-es-open-distro-datasource" diff --git a/pkg/models/org_user.go b/pkg/models/org_user.go index 5498536de89..a7059f94d06 100644 --- a/pkg/models/org_user.go +++ b/pkg/models/org_user.go @@ -107,11 +107,13 @@ type UpdateOrgUserCommand struct { // QUERIES type GetOrgUsersQuery struct { + UserID int64 OrgId int64 Query string Limit int IsServiceAccount bool + User *SignedInUser Result []*OrgUserDTO } @@ -122,6 +124,7 @@ type SearchOrgUsersQuery struct { Limit int IsServiceAccount bool + User *SignedInUser Result SearchOrgUsersQueryResult } diff --git a/pkg/models/team_member.go b/pkg/models/team_member.go index 1ad8dd97e84..8b7078c5168 100644 --- a/pkg/models/team_member.go +++ b/pkg/models/team_member.go @@ -35,18 +35,16 @@ type AddTeamMemberCommand struct { } type UpdateTeamMemberCommand struct { - UserId int64 `json:"-"` - OrgId int64 `json:"-"` - TeamId int64 `json:"-"` - Permission PermissionType `json:"permission"` - ProtectLastAdmin bool `json:"-"` + UserId int64 `json:"-"` + OrgId int64 `json:"-"` + TeamId int64 `json:"-"` + Permission PermissionType `json:"permission"` } type RemoveTeamMemberCommand struct { - OrgId int64 `json:"-"` - UserId int64 - TeamId int64 - ProtectLastAdmin bool `json:"-"` + OrgId int64 `json:"-"` + UserId int64 + TeamId int64 } // ---------------------- diff --git a/pkg/models/user_auth.go b/pkg/models/user_auth.go index c7f2b01e878..221b899c0e1 100644 --- a/pkg/models/user_auth.go +++ b/pkg/models/user_auth.go @@ -120,8 +120,3 @@ type TeamOrgGroupDTO struct { OrgName string `json:"orgName"` GroupDN string `json:"groupDN"` } - -type GetTeamsForLDAPGroupCommand struct { - Groups []string - Result []TeamOrgGroupDTO -} diff --git a/pkg/plugins/backend_utils.go b/pkg/plugins/backend_utils.go index edb905045d7..075635e9ef4 100644 --- a/pkg/plugins/backend_utils.go +++ b/pkg/plugins/backend_utils.go @@ -2,11 +2,8 @@ package plugins import ( "fmt" - "path/filepath" "runtime" "strings" - - "github.com/grafana/grafana/pkg/setting" ) func ComposePluginStartCommand(executable string) string { @@ -32,14 +29,3 @@ func ComposeRendererStartCommand() string { return fmt.Sprintf("%s_%s_%s%s", "plugin_start", os, strings.ToLower(arch), extension) } - -func CoreDataSourcePathResolver(cfg *setting.Cfg, pluginRootDirName string) PluginPathResolver { - return func() (string, error) { - // override mismatch cloud monitoring plugin - if pluginRootDirName == "stackdriver" { - pluginRootDirName = "cloud-monitoring" - } - - return filepath.Join(cfg.StaticRootPath, "app/plugins/datasource", pluginRootDirName), nil - } -} diff --git a/pkg/plugins/backendplugin/coreplugin/registry.go b/pkg/plugins/backendplugin/coreplugin/registry.go new file mode 100644 index 00000000000..1b27c5cf3a6 --- /dev/null +++ b/pkg/plugins/backendplugin/coreplugin/registry.go @@ -0,0 +1,112 @@ +package coreplugin + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin" + "github.com/grafana/grafana/pkg/tsdb/azuremonitor" + "github.com/grafana/grafana/pkg/tsdb/cloudmonitoring" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch" + "github.com/grafana/grafana/pkg/tsdb/grafanads" + "github.com/grafana/grafana/pkg/tsdb/graphite" + "github.com/grafana/grafana/pkg/tsdb/influxdb" + "github.com/grafana/grafana/pkg/tsdb/loki" + "github.com/grafana/grafana/pkg/tsdb/mssql" + "github.com/grafana/grafana/pkg/tsdb/mysql" + "github.com/grafana/grafana/pkg/tsdb/opentsdb" + "github.com/grafana/grafana/pkg/tsdb/postgres" + "github.com/grafana/grafana/pkg/tsdb/prometheus" + "github.com/grafana/grafana/pkg/tsdb/tempo" + "github.com/grafana/grafana/pkg/tsdb/testdatasource" +) + +const ( + CloudWatch = "cloudwatch" + CloudMonitoring = "stackdriver" + AzureMonitor = "grafana-azure-monitor-datasource" + Elasticsearch = "elasticsearch" + Graphite = "graphite" + InfluxDB = "influxdb" + Loki = "loki" + OpenTSDB = "opentsdb" + Prometheus = "prometheus" + Tempo = "tempo" + TestData = "testdata" + PostgreSQL = "postgres" + MySQL = "mysql" + MSSQL = "mssql" + Grafana = "grafana" +) + +type Registry struct { + store map[string]backendplugin.PluginFactoryFunc +} + +func NewRegistry(store map[string]backendplugin.PluginFactoryFunc) *Registry { + return &Registry{ + store: store, + } +} + +func ProvideCoreRegistry(am *azuremonitor.Service, cw *cloudwatch.CloudWatchService, cm *cloudmonitoring.Service, + es *elasticsearch.Service, grap *graphite.Service, idb *influxdb.Service, lk *loki.Service, otsdb *opentsdb.Service, + pr *prometheus.Service, t *tempo.Service, td *testdatasource.Service, pg *postgres.Service, my *mysql.Service, + ms *mssql.Service, graf *grafanads.Service) *Registry { + return NewRegistry(map[string]backendplugin.PluginFactoryFunc{ + CloudWatch: asBackendPlugin(cw.Executor), + CloudMonitoring: asBackendPlugin(cm), + AzureMonitor: asBackendPlugin(am), + Elasticsearch: asBackendPlugin(es), + Graphite: asBackendPlugin(grap), + InfluxDB: asBackendPlugin(idb), + Loki: asBackendPlugin(lk), + OpenTSDB: asBackendPlugin(otsdb), + Prometheus: asBackendPlugin(pr), + Tempo: asBackendPlugin(t), + TestData: asBackendPlugin(td), + PostgreSQL: asBackendPlugin(pg), + MySQL: asBackendPlugin(my), + MSSQL: asBackendPlugin(ms), + Grafana: asBackendPlugin(graf), + }) +} + +func (cr *Registry) Get(pluginID string) backendplugin.PluginFactoryFunc { + return cr.store[pluginID] +} + +func (cr *Registry) BackendFactoryProvider() func(_ context.Context, p *plugins.Plugin) backendplugin.PluginFactoryFunc { + return func(_ context.Context, p *plugins.Plugin) backendplugin.PluginFactoryFunc { + if !p.IsCorePlugin() { + return nil + } + + return cr.Get(p.ID) + } +} + +func asBackendPlugin(svc interface{}) backendplugin.PluginFactoryFunc { + opts := backend.ServeOpts{} + if queryHandler, ok := svc.(backend.QueryDataHandler); ok { + opts.QueryDataHandler = queryHandler + } + if resourceHandler, ok := svc.(backend.CallResourceHandler); ok { + opts.CallResourceHandler = resourceHandler + } + if streamHandler, ok := svc.(backend.StreamHandler); ok { + opts.StreamHandler = streamHandler + } + if healthHandler, ok := svc.(backend.CheckHealthHandler); ok { + opts.CheckHealthHandler = healthHandler + } + + if opts.QueryDataHandler != nil || opts.CallResourceHandler != nil || + opts.CheckHealthHandler != nil || opts.StreamHandler != nil { + return New(opts) + } + + return nil +} diff --git a/pkg/plugins/backendplugin/grpcplugin/client_v2.go b/pkg/plugins/backendplugin/grpcplugin/client_v2.go index 8ea5c766ad2..30f6399bca0 100644 --- a/pkg/plugins/backendplugin/grpcplugin/client_v2.go +++ b/pkg/plugins/backendplugin/grpcplugin/client_v2.go @@ -18,7 +18,7 @@ import ( "google.golang.org/grpc/status" ) -type clientV2 struct { +type ClientV2 struct { grpcplugin.DiagnosticsClient grpcplugin.ResourceClient grpcplugin.DataClient @@ -52,7 +52,7 @@ func newClientV2(descriptor PluginDescriptor, logger log.Logger, rpcClient plugi return nil, err } - c := clientV2{} + c := ClientV2{} if rawDiagnostics != nil { if diagnosticsClient, ok := rawDiagnostics.(grpcplugin.DiagnosticsClient); ok { c.DiagnosticsClient = diagnosticsClient @@ -92,7 +92,7 @@ func newClientV2(descriptor PluginDescriptor, logger log.Logger, rpcClient plugi return &c, nil } -func (c *clientV2) CollectMetrics(ctx context.Context) (*backend.CollectMetricsResult, error) { +func (c *ClientV2) CollectMetrics(ctx context.Context) (*backend.CollectMetricsResult, error) { if c.DiagnosticsClient == nil { return &backend.CollectMetricsResult{}, nil } @@ -109,7 +109,7 @@ func (c *clientV2) CollectMetrics(ctx context.Context) (*backend.CollectMetricsR return backend.FromProto().CollectMetricsResponse(protoResp), nil } -func (c *clientV2) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { +func (c *ClientV2) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { if c.DiagnosticsClient == nil { return nil, backendplugin.ErrMethodNotImplemented } @@ -130,7 +130,7 @@ func (c *clientV2) CheckHealth(ctx context.Context, req *backend.CheckHealthRequ return backend.FromProto().CheckHealthResponse(protoResp), nil } -func (c *clientV2) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { +func (c *ClientV2) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { if c.DataClient == nil { return nil, backendplugin.ErrMethodNotImplemented } @@ -149,7 +149,7 @@ func (c *clientV2) QueryData(ctx context.Context, req *backend.QueryDataRequest) return backend.FromProto().QueryDataResponse(protoResp) } -func (c *clientV2) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { +func (c *ClientV2) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { if c.ResourceClient == nil { return backendplugin.ErrMethodNotImplemented } @@ -184,7 +184,7 @@ func (c *clientV2) CallResource(ctx context.Context, req *backend.CallResourceRe } } -func (c *clientV2) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { +func (c *ClientV2) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { if c.StreamClient == nil { return nil, backendplugin.ErrMethodNotImplemented } @@ -195,7 +195,7 @@ func (c *clientV2) SubscribeStream(ctx context.Context, req *backend.SubscribeSt return backend.FromProto().SubscribeStreamResponse(protoResp), nil } -func (c *clientV2) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { +func (c *ClientV2) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { if c.StreamClient == nil { return nil, backendplugin.ErrMethodNotImplemented } @@ -206,7 +206,7 @@ func (c *clientV2) PublishStream(ctx context.Context, req *backend.PublishStream return backend.FromProto().PublishStreamResponse(protoResp), nil } -func (c *clientV2) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { +func (c *ClientV2) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { if c.StreamClient == nil { return backendplugin.ErrMethodNotImplemented } diff --git a/pkg/plugins/backendplugin/provider/provider.go b/pkg/plugins/backendplugin/provider/provider.go new file mode 100644 index 00000000000..4a8a18c4c9a --- /dev/null +++ b/pkg/plugins/backendplugin/provider/provider.go @@ -0,0 +1,61 @@ +package provider + +import ( + "context" + "path/filepath" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin" + "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" + "github.com/grafana/grafana/pkg/plugins/backendplugin/grpcplugin" + "github.com/grafana/grafana/pkg/plugins/backendplugin/pluginextensionv2" +) + +// PluginBackendProvider is a function type for initializing a Plugin backend. +type PluginBackendProvider func(_ context.Context, _ *plugins.Plugin) backendplugin.PluginFactoryFunc + +type Service struct { + providerChain []PluginBackendProvider +} + +func New(providers ...PluginBackendProvider) *Service { + if len(providers) == 0 { + return New(RendererProvider, DefaultProvider) + } + return &Service{ + providerChain: providers, + } +} + +func ProvideService(coreRegistry *coreplugin.Registry) *Service { + return New(coreRegistry.BackendFactoryProvider(), RendererProvider, DefaultProvider) +} + +func (s *Service) BackendFactory(ctx context.Context, p *plugins.Plugin) backendplugin.PluginFactoryFunc { + for _, provider := range s.providerChain { + if factory := provider(ctx, p); factory != nil { + return factory + } + } + return nil +} + +var RendererProvider PluginBackendProvider = func(_ context.Context, p *plugins.Plugin) backendplugin.PluginFactoryFunc { + if !p.IsRenderer() { + return nil + } + cmd := plugins.ComposeRendererStartCommand() + return grpcplugin.NewRendererPlugin(p.ID, filepath.Join(p.PluginDir, cmd), + func(pluginID string, renderer pluginextensionv2.RendererPlugin, logger log.Logger) error { + p.Renderer = renderer + return nil + }, + ) +} + +var DefaultProvider PluginBackendProvider = func(_ context.Context, p *plugins.Plugin) backendplugin.PluginFactoryFunc { + // TODO check for executable + cmd := plugins.ComposePluginStartCommand(p.Executable) + return grpcplugin.NewBackendPlugin(p.ID, filepath.Join(p.PluginDir, cmd)) +} diff --git a/pkg/plugins/config.go b/pkg/plugins/config.go new file mode 100644 index 00000000000..e25f18066e2 --- /dev/null +++ b/pkg/plugins/config.go @@ -0,0 +1,57 @@ +package plugins + +import ( + "github.com/grafana/grafana/pkg/setting" +) + +type Cfg struct { + DevMode bool + + PluginsPath string + + PluginSettings setting.PluginSettings + PluginsAllowUnsigned []string + + EnterpriseLicensePath string + + // AWS Plugin Auth + AWSAllowedAuthProviders []string + AWSAssumeRoleEnabled bool + + // Azure Cloud settings + Azure setting.AzureSettings + + CheckForUpdates bool + + BuildVersion string // TODO Remove + AppSubURL string // TODO Remove +} + +func NewCfg() *Cfg { + return &Cfg{} +} + +func FromGrafanaCfg(grafanaCfg *setting.Cfg) *Cfg { + cfg := &Cfg{} + + cfg.DevMode = grafanaCfg.Env == setting.Dev + cfg.PluginsPath = grafanaCfg.PluginsPath + + cfg.PluginSettings = grafanaCfg.PluginSettings + cfg.PluginsAllowUnsigned = grafanaCfg.PluginsAllowUnsigned + cfg.EnterpriseLicensePath = grafanaCfg.EnterpriseLicensePath + + // AWS + cfg.AWSAllowedAuthProviders = grafanaCfg.AWSAllowedAuthProviders + cfg.AWSAssumeRoleEnabled = grafanaCfg.AWSAssumeRoleEnabled + + // Azure + cfg.Azure = grafanaCfg.Azure + + cfg.CheckForUpdates = grafanaCfg.CheckForUpdates + + cfg.BuildVersion = grafanaCfg.BuildVersion + cfg.AppSubURL = grafanaCfg.AppSubURL + + return cfg +} diff --git a/pkg/plugins/dashboards.go b/pkg/plugins/dashboards.go deleted file mode 100644 index f8973015a50..00000000000 --- a/pkg/plugins/dashboards.go +++ /dev/null @@ -1,17 +0,0 @@ -package plugins - -type PluginDashboardInfoDTO struct { - PluginId string `json:"pluginId"` - Title string `json:"title"` - Imported bool `json:"imported"` - ImportedUri string `json:"importedUri"` - ImportedUrl string `json:"importedUrl"` - Slug string `json:"slug"` - DashboardId int64 `json:"dashboardId"` - FolderId int64 `json:"folderId"` - ImportedRevision int64 `json:"importedRevision"` - Revision int64 `json:"revision"` - Description string `json:"description"` - Path string `json:"path"` - Removed bool `json:"removed"` -} diff --git a/pkg/plugins/ifaces.go b/pkg/plugins/ifaces.go index 8716b64ba21..3e106fb34a8 100644 --- a/pkg/plugins/ifaces.go +++ b/pkg/plugins/ifaces.go @@ -5,7 +5,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins/backendplugin" ) @@ -16,28 +15,16 @@ type Store interface { Plugin(ctx context.Context, pluginID string) (PluginDTO, bool) // Plugins returns plugins by their requested type. Plugins(ctx context.Context, pluginTypes ...Type) []PluginDTO - // Add adds a plugin to the store. Add(ctx context.Context, pluginID, version string) error - // AddWithFactory adds a plugin to the store. - AddWithFactory(ctx context.Context, pluginID string, factory backendplugin.PluginFactoryFunc, resolver PluginPathResolver) error // Remove removes a plugin from the store. Remove(ctx context.Context, pluginID string) error } -type PluginPathResolver func() (string, error) - -type AddOpts struct { - PluginInstallDir, PluginZipURL, PluginRepoURL string -} - // Loader is responsible for loading plugins from the file system. type Loader interface { // Load will return a list of plugins found in the provided file system paths. - Load(paths []string, ignore map[string]struct{}) ([]*Plugin, error) - // LoadWithFactory will return a plugin found in the provided file system path and use the provided factory to - // construct the plugin backend client. - LoadWithFactory(path string, factory backendplugin.PluginFactoryFunc) (*Plugin, error) + Load(ctx context.Context, class Class, paths []string, ignore map[string]struct{}) ([]*Plugin, error) } // Installer is responsible for managing plugins (add / remove) on the file system. @@ -65,6 +52,11 @@ type Client interface { CollectMetrics(ctx context.Context, pluginID string) (*backend.CollectMetricsResult, error) } +// BackendFactoryProvider provides a backend factory for a provided plugin. +type BackendFactoryProvider interface { + BackendFactory(ctx context.Context, p *Plugin) backendplugin.PluginFactoryFunc +} + type RendererManager interface { // Renderer returns a renderer plugin. Renderer() *Plugin @@ -83,20 +75,26 @@ type PluginLoaderAuthorizer interface { CanLoadPlugin(plugin *Plugin) bool } +type PluginDashboardInfoDTO struct { + UID string `json:"uid"` + PluginId string `json:"pluginId"` + Title string `json:"title"` + Imported bool `json:"imported"` + ImportedUri string `json:"importedUri"` + ImportedUrl string `json:"importedUrl"` + Slug string `json:"slug"` + DashboardId int64 `json:"dashboardId"` + FolderId int64 `json:"folderId"` + ImportedRevision int64 `json:"importedRevision"` + Revision int64 `json:"revision"` + Description string `json:"description"` + Path string `json:"path"` + Removed bool `json:"removed"` +} + type PluginDashboardManager interface { // GetPluginDashboards gets dashboards for a certain org/plugin. GetPluginDashboards(ctx context.Context, orgID int64, pluginID string) ([]*PluginDashboardInfoDTO, error) // LoadPluginDashboard loads a plugin dashboard. LoadPluginDashboard(ctx context.Context, pluginID, path string) (*models.Dashboard, error) - // ImportDashboard imports a dashboard. - ImportDashboard(ctx context.Context, pluginID, path string, orgID, folderID int64, dashboardModel *simplejson.Json, - overwrite bool, inputs []ImportDashboardInput, user *models.SignedInUser) (PluginDashboardInfoDTO, - *models.Dashboard, error) -} - -type ImportDashboardInput struct { - Type string `json:"type"` - PluginId string `json:"pluginId"` - Name string `json:"name"` - Value string `json:"value"` } diff --git a/pkg/plugins/manager/dashboard_import_test.go b/pkg/plugins/manager/dashboard_import_test.go deleted file mode 100644 index 41a08c623f1..00000000000 --- a/pkg/plugins/manager/dashboard_import_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package manager - -import ( - "context" - "io/ioutil" - "testing" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/manager/loader" - "github.com/grafana/grafana/pkg/plugins/manager/signature" - "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/setting" - "github.com/stretchr/testify/require" -) - -func TestDashboardImport(t *testing.T) { - pluginScenario(t, "When importing a plugin dashboard", func(t *testing.T, pm *PluginManager) { - origNewDashboardService := dashboards.NewService - t.Cleanup(func() { - dashboards.NewService = origNewDashboardService - }) - mock := &dashboards.FakeDashboardService{} - dashboards.MockDashboardService(mock) - - info, dash, err := pm.ImportDashboard(context.Background(), "test-app", "dashboards/connections.json", 1, 0, nil, false, - []plugins.ImportDashboardInput{ - {Name: "*", Type: "datasource", Value: "graphite"}, - }, &models.SignedInUser{UserId: 1, OrgRole: models.ROLE_ADMIN}) - require.NoError(t, err) - require.NotNil(t, info) - require.NotNil(t, dash) - - resultStr, err := mock.SavedDashboards[0].Dashboard.Data.EncodePretty() - require.NoError(t, err) - expectedBytes, err := ioutil.ReadFile("testdata/test-app/dashboards/connections_result.json") - require.NoError(t, err) - expectedJson, err := simplejson.NewJson(expectedBytes) - require.NoError(t, err) - expectedStr, err := expectedJson.EncodePretty() - require.NoError(t, err) - - require.Equal(t, expectedStr, resultStr) - - panel := mock.SavedDashboards[0].Dashboard.Data.Get("rows").GetIndex(0).Get("panels").GetIndex(0) - require.Equal(t, "graphite", panel.Get("datasource").MustString()) - }) - - t.Run("When evaling dashboard template", func(t *testing.T) { - template, err := simplejson.NewJson([]byte(`{ - "__inputs": [ - { - "name": "DS_NAME", - "type": "datasource" - } - ], - "test": { - "prop": "${DS_NAME}_${DS_NAME}" - } - }`)) - require.NoError(t, err) - - evaluator := &DashTemplateEvaluator{ - template: template, - inputs: []plugins.ImportDashboardInput{ - {Name: "*", Type: "datasource", Value: "my-server"}, - }, - } - - res, err := evaluator.Eval() - require.NoError(t, err) - - require.Equal(t, "my-server_my-server", res.GetPath("test", "prop").MustString()) - - inputs := res.Get("__inputs") - require.Nil(t, inputs.Interface()) - }) -} - -func pluginScenario(t *testing.T, desc string, fn func(*testing.T, *PluginManager)) { - t.Helper() - - t.Run("Given a plugin", func(t *testing.T) { - cfg := &setting.Cfg{ - FeatureToggles: map[string]bool{}, - PluginSettings: setting.PluginSettings{ - "test-app": map[string]string{ - "path": "testdata/test-app", - }, - }, - } - pm := newManager(cfg, nil, loader.New(nil, cfg, &signature.UnsignedPluginAuthorizer{Cfg: cfg}), &sqlstore.SQLStore{}) - err := pm.init() - require.NoError(t, err) - - t.Run(desc, func(t *testing.T) { - fn(t, pm) - }) - }) -} diff --git a/pkg/plugins/manager/dashboards.go b/pkg/plugins/manager/dashboards.go index 42cc5833bf2..a90a8125ac6 100644 --- a/pkg/plugins/manager/dashboards.go +++ b/pkg/plugins/manager/dashboards.go @@ -11,7 +11,6 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/util" ) @@ -41,6 +40,7 @@ func (m *PluginManager) GetPluginDashboards(ctx context.Context, orgID int64, pl } res := &plugins.PluginDashboardInfoDTO{} + res.UID = dashboard.Uid res.Path = include.Path res.PluginId = plugin.ID res.Title = dashboard.Title @@ -49,6 +49,7 @@ func (m *PluginManager) GetPluginDashboards(ctx context.Context, orgID int64, pl // find existing dashboard for _, existingDash := range query.Result { if existingDash.Slug == dashboard.Slug { + res.UID = existingDash.Uid res.DashboardId = existingDash.Id res.Imported = true res.ImportedUri = "db/" + existingDash.Slug @@ -65,6 +66,7 @@ func (m *PluginManager) GetPluginDashboards(ctx context.Context, orgID int64, pl for _, dash := range query.Result { if _, exists := existingMatches[dash.Id]; !exists { result = append(result, &plugins.PluginDashboardInfoDTO{ + UID: dash.Uid, Slug: dash.Slug, DashboardId: dash.Id, Removed: true, @@ -131,62 +133,3 @@ func (m *PluginManager) LoadPluginDashboard(ctx context.Context, pluginID, path return models.NewDashboardFromJson(data), nil } - -func (m *PluginManager) ImportDashboard(ctx context.Context, pluginID, path string, orgID, folderID int64, dashboardModel *simplejson.Json, - overwrite bool, inputs []plugins.ImportDashboardInput, user *models.SignedInUser) (plugins.PluginDashboardInfoDTO, - *models.Dashboard, error) { - var dashboard *models.Dashboard - if pluginID != "" { - var err error - if dashboard, err = m.LoadPluginDashboard(ctx, pluginID, path); err != nil { - return plugins.PluginDashboardInfoDTO{}, &models.Dashboard{}, err - } - } else { - dashboard = models.NewDashboardFromJson(dashboardModel) - } - - evaluator := &DashTemplateEvaluator{ - template: dashboard.Data, - inputs: inputs, - } - - generatedDash, err := evaluator.Eval() - if err != nil { - return plugins.PluginDashboardInfoDTO{}, &models.Dashboard{}, err - } - - saveCmd := models.SaveDashboardCommand{ - Dashboard: generatedDash, - OrgId: orgID, - UserId: user.UserId, - Overwrite: overwrite, - PluginId: pluginID, - FolderId: folderID, - } - - dto := &dashboards.SaveDashboardDTO{ - OrgId: orgID, - Dashboard: saveCmd.GetDashboardModel(), - Overwrite: saveCmd.Overwrite, - User: user, - } - - savedDash, err := dashboards.NewService(m.sqlStore).ImportDashboard(ctx, dto) - if err != nil { - return plugins.PluginDashboardInfoDTO{}, &models.Dashboard{}, err - } - - return plugins.PluginDashboardInfoDTO{ - PluginId: pluginID, - Title: savedDash.Title, - Path: path, - Revision: savedDash.Data.Get("revision").MustInt64(1), - FolderId: savedDash.FolderId, - ImportedUri: "db/" + savedDash.Slug, - ImportedUrl: savedDash.GetUrl(), - ImportedRevision: dashboard.Data.Get("revision").MustInt64(1), - Imported: true, - DashboardId: savedDash.Id, - Slug: savedDash.Slug, - }, savedDash, nil -} diff --git a/pkg/plugins/manager/dashboards_test.go b/pkg/plugins/manager/dashboards_test.go index 34980f3dcea..55397f0bff2 100644 --- a/pkg/plugins/manager/dashboards_test.go +++ b/pkg/plugins/manager/dashboards_test.go @@ -7,24 +7,25 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin/provider" "github.com/grafana/grafana/pkg/plugins/manager/loader" "github.com/grafana/grafana/pkg/plugins/manager/signature" - "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" ) func TestGetPluginDashboards(t *testing.T) { cfg := &setting.Cfg{ - FeatureToggles: map[string]bool{}, PluginSettings: setting.PluginSettings{ "test-app": map[string]string{ "path": "testdata/test-app", }, }, } - pm := newManager(cfg, nil, loader.New(nil, cfg, &signature.UnsignedPluginAuthorizer{Cfg: cfg}), &sqlstore.SQLStore{}) - err := pm.init() + pmCfg := plugins.FromGrafanaCfg(cfg) + pm, err := ProvideService(cfg, loader.New(pmCfg, nil, + signature.NewUnsignedAuthorizer(pmCfg), &provider.Service{})) require.NoError(t, err) bus.AddHandler("test", func(ctx context.Context, query *models.GetDashboardQuery) error { diff --git a/pkg/plugins/manager/loader/finder/finder.go b/pkg/plugins/manager/loader/finder/finder.go index e8f0bbccd87..2838749adb2 100644 --- a/pkg/plugins/manager/loader/finder/finder.go +++ b/pkg/plugins/manager/loader/finder/finder.go @@ -8,33 +8,33 @@ import ( "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) +var walk = util.Walk + type Finder struct { - cfg *setting.Cfg log log.Logger } -func New(cfg *setting.Cfg) Finder { - return Finder{cfg: cfg, log: log.New("plugin.finder")} +func New() Finder { + return Finder{log: log.New("plugin.finder")} } -func (f *Finder) Find(pluginDirs []string) ([]string, error) { +func (f *Finder) Find(pluginPaths []string) ([]string, error) { var pluginJSONPaths []string - for _, dir := range pluginDirs { - exists, err := fs.Exists(dir) + for _, path := range pluginPaths { + exists, err := fs.Exists(path) if err != nil { - f.log.Warn("Error occurred when checking if plugin directory exists", "dir", dir, "err", err) + f.log.Warn("Error occurred when checking if plugin directory exists", "path", path, "err", err) } if !exists { - f.log.Warn("Skipping finding plugins as directory does not exist", "dir", dir) + f.log.Warn("Skipping finding plugins as directory does not exist", "path", path) continue } - paths, err := f.getPluginJSONPaths(dir) + paths, err := f.getAbsPluginJSONPaths(path) if err != nil { return nil, err } @@ -44,18 +44,27 @@ func (f *Finder) Find(pluginDirs []string) ([]string, error) { return pluginJSONPaths, nil } -func (f *Finder) getPluginJSONPaths(dir string) ([]string, error) { +func (f *Finder) getAbsPluginJSONPaths(path string) ([]string, error) { var pluginJSONPaths []string var err error - dir, err = filepath.Abs(dir) + path, err = filepath.Abs(path) if err != nil { return []string{}, err } - if err := util.Walk(dir, true, true, + if err := walk(path, true, true, func(currentPath string, fi os.FileInfo, err error) error { if err != nil { + if errors.Is(err, os.ErrNotExist) { + f.log.Error("Couldn't scan directory since it doesn't exist", "pluginDir", path, "err", err) + return nil + } + if errors.Is(err, os.ErrPermission) { + f.log.Error("Couldn't scan directory due to lack of permissions", "pluginDir", path, "err", err) + return nil + } + return fmt.Errorf("filepath.Walk reported an error for %q: %w", currentPath, err) } @@ -74,15 +83,6 @@ func (f *Finder) getPluginJSONPaths(dir string) ([]string, error) { pluginJSONPaths = append(pluginJSONPaths, currentPath) return nil }); err != nil { - if errors.Is(err, os.ErrNotExist) { - f.log.Debug("Couldn't scan directory since it doesn't exist", "pluginDir", dir, "err", err) - return []string{}, nil - } - if errors.Is(err, os.ErrPermission) { - f.log.Debug("Couldn't scan directory due to lack of permissions", "pluginDir", dir, "err", err) - return []string{}, nil - } - return []string{}, err } diff --git a/pkg/plugins/manager/loader/finder/finder_test.go b/pkg/plugins/manager/loader/finder/finder_test.go index db401563985..093b2fc1bc4 100644 --- a/pkg/plugins/manager/loader/finder/finder_test.go +++ b/pkg/plugins/manager/loader/finder/finder_test.go @@ -2,12 +2,16 @@ package finder import ( "errors" + "fmt" + "os" "strings" "testing" - "github.com/stretchr/testify/assert" - + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFinder_Find(t *testing.T) { @@ -51,7 +55,7 @@ func TestFinder_Find(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - f := New(tc.cfg) + f := New() pluginPaths, err := f.Find(tc.pluginDirs) if (err != nil) && !errors.Is(err, tc.err) { t.Errorf("Find() error = %v, expected error %v", err, tc.err) @@ -65,3 +69,59 @@ func TestFinder_Find(t *testing.T) { }) } } + +func TestFinder_getAbsPluginJSONPaths(t *testing.T) { + t.Run("When scanning a folder that doesn't exists shouldn't return an error", func(t *testing.T) { + origWalk := walk + walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { + return walkFn(path, nil, os.ErrNotExist) + } + t.Cleanup(func() { + walk = origWalk + }) + + finder := &Finder{ + log: log.New(), + } + + paths, err := finder.getAbsPluginJSONPaths("test") + require.NoError(t, err) + require.Empty(t, paths) + }) + + t.Run("When scanning a folder that lacks permission shouldn't return an error", func(t *testing.T) { + origWalk := walk + walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { + return walkFn(path, nil, os.ErrPermission) + } + t.Cleanup(func() { + walk = origWalk + }) + + finder := &Finder{ + log: log.New(), + } + + paths, err := finder.getAbsPluginJSONPaths("test") + require.NoError(t, err) + require.Empty(t, paths) + }) + + t.Run("When scanning a folder that returns a non-handled error should return that error", func(t *testing.T) { + origWalk := walk + walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { + return walkFn(path, nil, fmt.Errorf("random error")) + } + t.Cleanup(func() { + walk = origWalk + }) + + finder := &Finder{ + log: log.New(), + } + + paths, err := finder.getAbsPluginJSONPaths("test") + require.Error(t, err) + require.Empty(t, paths) + }) +} diff --git a/pkg/plugins/manager/loader/initializer/initializer.go b/pkg/plugins/manager/loader/initializer/initializer.go index 17b065c6160..608e4cc66fe 100644 --- a/pkg/plugins/manager/loader/initializer/initializer.go +++ b/pkg/plugins/manager/loader/initializer/initializer.go @@ -1,104 +1,42 @@ package initializer import ( + "context" "fmt" - "net/url" "os" - "path" - "path/filepath" "strings" "github.com/grafana/grafana-aws-sdk/pkg/awsds" - "github.com/gosimple/slug" - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin" - "github.com/grafana/grafana/pkg/plugins/backendplugin/grpcplugin" - "github.com/grafana/grafana/pkg/plugins/backendplugin/pluginextensionv2" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" ) type Initializer struct { - cfg *setting.Cfg - license models.Licensing - log log.Logger + cfg *plugins.Cfg + license models.Licensing + backendProvider plugins.BackendFactoryProvider + log log.Logger } -func New(cfg *setting.Cfg, license models.Licensing) Initializer { +func New(cfg *plugins.Cfg, backendProvider plugins.BackendFactoryProvider, license models.Licensing) Initializer { return Initializer{ - cfg: cfg, - license: license, - log: log.New("plugin.initializer"), + cfg: cfg, + license: license, + backendProvider: backendProvider, + log: log.New("plugin.initializer"), } } -func (i *Initializer) Initialize(p *plugins.Plugin) error { - if len(p.Dependencies.Plugins) == 0 { - p.Dependencies.Plugins = []plugins.Dependency{} - } - - if p.Dependencies.GrafanaVersion == "" { - p.Dependencies.GrafanaVersion = "*" - } - - for _, include := range p.Includes { - if include.Role == "" { - include.Role = models.ROLE_VIEWER - } - } - - i.handleModuleDefaults(p) - - p.Info.Logos.Small = pluginLogoURL(p.Type, p.Info.Logos.Small, p.BaseURL) - p.Info.Logos.Large = pluginLogoURL(p.Type, p.Info.Logos.Large, p.BaseURL) - - for i := 0; i < len(p.Info.Screenshots); i++ { - p.Info.Screenshots[i].Path = evalRelativePluginURLPath(p.Info.Screenshots[i].Path, p.BaseURL, p.Type) - } - - if p.IsApp() { - for _, child := range p.Children { - i.setPathsBasedOnApp(p, child) - } - - // slugify pages - for _, include := range p.Includes { - if include.Slug == "" { - include.Slug = slug.Make(include.Name) - } - if include.Type == "page" && include.DefaultNav { - p.DefaultNavURL = i.cfg.AppSubURL + "/plugins/" + p.ID + "/page/" + include.Slug - } - if include.Type == "dashboard" && include.DefaultNav { - p.DefaultNavURL = i.cfg.AppSubURL + "/dashboard/db/" + include.Slug - } - } - } - - pluginLog := i.log.New("pluginID", p.ID) - p.SetLogger(pluginLog) - +func (i *Initializer) Initialize(ctx context.Context, p *plugins.Plugin) error { if p.Backend { - var backendFactory backendplugin.PluginFactoryFunc - if p.IsRenderer() { - cmd := plugins.ComposeRendererStartCommand() - backendFactory = grpcplugin.NewRendererPlugin(p.ID, filepath.Join(p.PluginDir, cmd), - func(pluginID string, renderer pluginextensionv2.RendererPlugin, logger log.Logger) error { - p.Renderer = renderer - return nil - }, - ) - } else { - cmd := plugins.ComposePluginStartCommand(p.Executable) - backendFactory = grpcplugin.NewBackendPlugin(p.ID, filepath.Join(p.PluginDir, cmd)) + backendFactory := i.backendProvider.BackendFactory(ctx, p) + if backendFactory == nil { + return fmt.Errorf("could not find backend factory for plugin") } - if backendClient, err := backendFactory(p.ID, pluginLog, i.envVars(p)); err != nil { + if backendClient, err := backendFactory(p.ID, p.Logger(), i.envVars(p)); err != nil { return err } else { p.RegisterClient(backendClient) @@ -108,89 +46,6 @@ func (i *Initializer) Initialize(p *plugins.Plugin) error { return nil } -func (i *Initializer) InitializeWithFactory(p *plugins.Plugin, factory backendplugin.PluginFactoryFunc) error { - err := i.Initialize(p) - if err != nil { - return err - } - - if factory != nil { - var err error - - f, err := factory(p.ID, log.New("pluginID", p.ID), []string{}) - if err != nil { - return err - } - p.RegisterClient(f) - } else { - i.log.Warn("Could not initialize core plugin process", "pluginID", p.ID) - return fmt.Errorf("could not initialize plugin %s", p.ID) - } - - return nil -} - -func (i *Initializer) handleModuleDefaults(p *plugins.Plugin) { - if p.IsCorePlugin() { - // Previously there was an assumption that the Core plugins directory - // should be public/app/plugins// - // However this can be an issue if the Core plugins directory is renamed - baseDir := filepath.Base(p.PluginDir) - - // use path package for the following statements because these are not file paths - p.Module = path.Join("app/plugins", string(p.Type), baseDir, "module") - p.BaseURL = path.Join("public/app/plugins", string(p.Type), baseDir) - return - } - - metrics.SetPluginBuildInformation(p.ID, string(p.Type), p.Info.Version, string(p.Signature)) - - p.Module = path.Join("plugins", p.ID, "module") - p.BaseURL = path.Join("public/plugins", p.ID) -} - -func (i *Initializer) setPathsBasedOnApp(parent *plugins.Plugin, child *plugins.Plugin) { - appSubPath := strings.ReplaceAll(strings.Replace(child.PluginDir, parent.PluginDir, "", 1), "\\", "/") - child.IncludedInAppID = parent.ID - child.BaseURL = parent.BaseURL - - if parent.IsCorePlugin() { - child.Module = util.JoinURLFragments("app/plugins/app/"+parent.ID, appSubPath) + "/module" - } else { - child.Module = util.JoinURLFragments("plugins/"+parent.ID, appSubPath) + "/module" - } -} - -func pluginLogoURL(pluginType plugins.Type, path, baseURL string) string { - if path == "" { - return defaultLogoPath(pluginType) - } - - return evalRelativePluginURLPath(path, baseURL, pluginType) -} - -func defaultLogoPath(pluginType plugins.Type) string { - return "public/img/icn-" + string(pluginType) + ".svg" -} - -func evalRelativePluginURLPath(pathStr, baseURL string, pluginType plugins.Type) string { - if pathStr == "" { - return "" - } - - u, _ := url.Parse(pathStr) - if u.IsAbs() { - return pathStr - } - - // is set as default or has already been prefixed with base path - if pathStr == defaultLogoPath(pluginType) || strings.HasPrefix(pathStr, baseURL) { - return pathStr - } - - return path.Join(baseURL, pathStr) -} - func (i *Initializer) envVars(plugin *plugins.Plugin) []string { hostEnv := []string{ fmt.Sprintf("GF_VERSION=%s", i.cfg.BuildVersion), @@ -260,7 +115,7 @@ func (ps pluginSettings) asEnvVar(prefix string, hostEnv []string) []string { return env } -func getPluginSettings(pluginID string, cfg *setting.Cfg) pluginSettings { +func getPluginSettings(pluginID string, cfg *plugins.Cfg) pluginSettings { ps := pluginSettings{} for k, v := range cfg.PluginSettings[pluginID] { if k == "path" || strings.ToLower(k) == "id" { diff --git a/pkg/plugins/manager/loader/initializer/initializer_test.go b/pkg/plugins/manager/loader/initializer/initializer_test.go index 8e66c9d6755..8b6412e8372 100644 --- a/pkg/plugins/manager/loader/initializer/initializer_test.go +++ b/pkg/plugins/manager/loader/initializer/initializer_test.go @@ -1,17 +1,15 @@ package initializer import ( - "path" + "context" "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/backendplugin" - "github.com/grafana/grafana/pkg/setting" ) func TestInitializer_Initialize(t *testing.T) { @@ -36,21 +34,16 @@ func TestInitializer_Initialize(t *testing.T) { } i := &Initializer{ - cfg: setting.NewCfg(), + cfg: plugins.NewCfg(), log: &fakeLogger{}, + backendProvider: &fakeBackendProvider{ + plugin: p, + }, } - err := i.Initialize(p) + err := i.Initialize(context.Background(), p) assert.NoError(t, err) - assert.Equal(t, "public/img/icn-datasource.svg", p.Info.Logos.Small) - assert.Equal(t, "public/img/icn-datasource.svg", p.Info.Logos.Large) - assert.Equal(t, "*", p.Dependencies.GrafanaVersion) - assert.Len(t, p.Includes, 1) - assert.Equal(t, models.ROLE_VIEWER, p.Includes[0].Role) - assert.Equal(t, filepath.Join("app/plugins/datasource", filepath.Base(p.PluginDir), "module"), p.Module) - assert.Equal(t, path.Join("public/app/plugins/datasource", filepath.Base(p.PluginDir)), p.BaseURL) - assert.NotNil(t, p.Logger()) c, exists := p.Client() assert.True(t, exists) assert.NotNil(t, c) @@ -71,147 +64,39 @@ func TestInitializer_Initialize(t *testing.T) { } i := &Initializer{ - cfg: setting.NewCfg(), + cfg: plugins.NewCfg(), log: fakeLogger{}, + backendProvider: &fakeBackendProvider{ + plugin: p, + }, } - err := i.Initialize(p) + err := i.Initialize(context.Background(), p) assert.NoError(t, err) - // TODO add default img to project - assert.Equal(t, "public/img/icn-renderer.svg", p.Info.Logos.Small) - assert.Equal(t, "public/img/icn-renderer.svg", p.Info.Logos.Large) - assert.Equal(t, ">=8.x", p.Dependencies.GrafanaVersion) - assert.Equal(t, "plugins/test/module", p.Module) - assert.Equal(t, "public/plugins/test", p.BaseURL) - assert.NotNil(t, p.Logger()) c, exists := p.Client() assert.True(t, exists) assert.NotNil(t, c) }) - t.Run("external app", func(t *testing.T) { + t.Run("non backend plugin app", func(t *testing.T) { p := &plugins.Plugin{ JSONData: plugins.JSONData{ - ID: "parent-plugin", - Type: plugins.App, - Includes: []*plugins.Includes{ - { - Type: "page", - DefaultNav: true, - Slug: "myCustomSlug", - }, - }, - }, - PluginDir: absCurPath, - Class: plugins.External, - Children: []*plugins.Plugin{ - { - JSONData: plugins.JSONData{ - ID: "child-plugin", - }, - PluginDir: absCurPath, - }, + Backend: false, }, } i := &Initializer{ - cfg: &setting.Cfg{ - AppSubURL: "appSubURL", - }, + cfg: &plugins.Cfg{}, log: fakeLogger{}, + backendProvider: &fakeBackendProvider{ + plugin: p, + }, } - err := i.Initialize(p) + err := i.Initialize(context.Background(), p) assert.NoError(t, err) - assert.Equal(t, "public/img/icn-app.svg", p.Info.Logos.Small) - assert.Equal(t, "public/img/icn-app.svg", p.Info.Logos.Large) - assert.Equal(t, "*", p.Dependencies.GrafanaVersion) - assert.Len(t, p.Includes, 1) - assert.Equal(t, models.ROLE_VIEWER, p.Includes[0].Role) - assert.Equal(t, filepath.Join("plugins", p.ID, "module"), p.Module) - assert.Equal(t, "public/plugins/parent-plugin", p.BaseURL) - assert.NotNil(t, p.Logger()) - c, exists := p.Client() - assert.False(t, exists) - assert.Nil(t, c) - - assert.Len(t, p.Children, 1) - assert.Equal(t, p.ID, p.Children[0].IncludedInAppID) - assert.Equal(t, "public/plugins/parent-plugin", p.Children[0].BaseURL) - assert.Equal(t, "plugins/parent-plugin/module", p.Children[0].Module) - assert.Equal(t, "appSubURL/plugins/parent-plugin/page/myCustomSlug", p.DefaultNavURL) - }) -} - -func TestInitializer_InitializeWithFactory(t *testing.T) { - t.Run("happy path", func(t *testing.T) { - p := &plugins.Plugin{ - JSONData: plugins.JSONData{ - ID: "test-plugin", - Type: plugins.App, - Includes: []*plugins.Includes{ - { - Type: "page", - DefaultNav: true, - Slug: "myCustomSlug", - }, - }, - }, - PluginDir: "test/folder", - Class: plugins.External, - } - i := &Initializer{ - cfg: &setting.Cfg{ - AppSubURL: "appSubURL", - }, - log: fakeLogger{}, - } - - factoryInvoked := false - - factory := backendplugin.PluginFactoryFunc(func(pluginID string, logger log.Logger, env []string) (backendplugin.Plugin, error) { - factoryInvoked = true - return testPlugin{}, nil - }) - - err := i.InitializeWithFactory(p, factory) - assert.NoError(t, err) - - assert.True(t, factoryInvoked) - assert.NotNil(t, p.Logger()) - client, exists := p.Client() - assert.True(t, exists) - assert.NotNil(t, client.(testPlugin)) - }) - - t.Run("invalid factory", func(t *testing.T) { - p := &plugins.Plugin{ - JSONData: plugins.JSONData{ - ID: "test-plugin", - Type: plugins.App, - Includes: []*plugins.Includes{ - { - Type: "page", - DefaultNav: true, - Slug: "myCustomSlug", - }, - }, - }, - PluginDir: "test/folder", - Class: plugins.External, - } - i := &Initializer{ - cfg: &setting.Cfg{ - AppSubURL: "appSubURL", - }, - log: fakeLogger{}, - } - - err := i.InitializeWithFactory(p, nil) - assert.Errorf(t, err, "could not initialize plugin test-plugin") - c, exists := p.Client() assert.False(t, exists) assert.Nil(t, c) @@ -232,7 +117,7 @@ func TestInitializer_envVars(t *testing.T) { } i := &Initializer{ - cfg: &setting.Cfg{ + cfg: &plugins.Cfg{ EnterpriseLicensePath: "/path/to/ent/license", PluginSettings: map[string]map[string]string{ "test": { @@ -242,6 +127,9 @@ func TestInitializer_envVars(t *testing.T) { }, license: licensing, log: fakeLogger{}, + backendProvider: &fakeBackendProvider{ + plugin: p, + }, } envVars := i.envVars(p) @@ -254,33 +142,6 @@ func TestInitializer_envVars(t *testing.T) { }) } -func TestInitializer_setPathsBasedOnApp(t *testing.T) { - t.Run("When setting paths based on core plugin on Windows", func(t *testing.T) { - i := &Initializer{ - cfg: setting.NewCfg(), - log: fakeLogger{}, - } - - child := &plugins.Plugin{ - PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata\\datasources\\datasource", - } - parent := &plugins.Plugin{ - JSONData: plugins.JSONData{ - ID: "testdata", - }, - Class: plugins.Core, - PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata", - BaseURL: "public/app/plugins/app/testdata", - } - - i.setPathsBasedOnApp(parent, child) - - assert.Equal(t, "app/plugins/app/testdata/datasources/datasource/module", child.Module) - assert.Equal(t, "testdata", child.IncludedInAppID) - assert.Equal(t, "public/app/plugins/app/testdata", child.BaseURL) - }) -} - func TestInitializer_getAWSEnvironmentVariables(t *testing.T) { } @@ -334,7 +195,7 @@ func (t *testLicensingService) ContentDeliveryPrefix() string { return "" } -func (t *testLicensingService) LicenseURL(showAdminLicensingPage bool) string { +func (t *testLicensingService) LicenseURL(_ bool) string { return "" } @@ -350,10 +211,6 @@ func (*testLicensingService) FeatureEnabled(feature string) bool { return false } -type testPlugin struct { - backendplugin.Plugin -} - type fakeLogger struct { log.MultiLoggers } @@ -365,3 +222,15 @@ func (f fakeLogger) New(_ ...interface{}) log.MultiLoggers { func (f fakeLogger) Warn(_ string, _ ...interface{}) { } + +type fakeBackendProvider struct { + plugins.BackendFactoryProvider + + plugin *plugins.Plugin +} + +func (f *fakeBackendProvider) BackendFactory(_ context.Context, _ *plugins.Plugin) backendplugin.PluginFactoryFunc { + return func(_ string, _ log.Logger, _ []string) (backendplugin.Plugin, error) { + return f.plugin, nil + } +} diff --git a/pkg/plugins/manager/loader/loader.go b/pkg/plugins/manager/loader/loader.go index f04bff52ec1..db4e36e674f 100644 --- a/pkg/plugins/manager/loader/loader.go +++ b/pkg/plugins/manager/loader/loader.go @@ -1,23 +1,28 @@ package loader import ( + "context" "encoding/json" "errors" - "fmt" + "net/url" "os" + "path" "path/filepath" "runtime" "strings" + "github.com/gosimple/slug" + "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin" "github.com/grafana/grafana/pkg/plugins/manager/loader/finder" "github.com/grafana/grafana/pkg/plugins/manager/loader/initializer" "github.com/grafana/grafana/pkg/plugins/manager/signature" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" ) var ( @@ -28,7 +33,7 @@ var ( var _ plugins.ErrorResolver = (*Loader)(nil) type Loader struct { - cfg *setting.Cfg + cfg *plugins.Cfg pluginFinder finder.Finder pluginInitializer initializer.Initializer signatureValidator signature.Validator @@ -37,80 +42,51 @@ type Loader struct { errs map[string]*plugins.SignatureError } -func ProvideService(license models.Licensing, cfg *setting.Cfg, authorizer plugins.PluginLoaderAuthorizer) (*Loader, error) { - return New(license, cfg, authorizer), nil +func ProvideService(cfg *setting.Cfg, license models.Licensing, authorizer plugins.PluginLoaderAuthorizer, + backendProvider plugins.BackendFactoryProvider) (*Loader, error) { + return New(plugins.FromGrafanaCfg(cfg), license, authorizer, backendProvider), nil } -func New(license models.Licensing, cfg *setting.Cfg, authorizer plugins.PluginLoaderAuthorizer) *Loader { +func New(cfg *plugins.Cfg, license models.Licensing, authorizer plugins.PluginLoaderAuthorizer, + backendProvider plugins.BackendFactoryProvider) *Loader { return &Loader{ cfg: cfg, - pluginFinder: finder.New(cfg), - pluginInitializer: initializer.New(cfg, license), + pluginFinder: finder.New(), + pluginInitializer: initializer.New(cfg, backendProvider, license), signatureValidator: signature.NewValidator(authorizer), errs: make(map[string]*plugins.SignatureError), log: log.New("plugin.loader"), } } -func (l *Loader) Load(paths []string, ignore map[string]struct{}) ([]*plugins.Plugin, error) { +func (l *Loader) Load(ctx context.Context, class plugins.Class, paths []string, ignore map[string]struct{}) ([]*plugins.Plugin, error) { pluginJSONPaths, err := l.pluginFinder.Find(paths) if err != nil { - l.log.Error("plugin finder encountered an error", "err", err) - } - - return l.loadPlugins(pluginJSONPaths, ignore) -} - -func (l *Loader) LoadWithFactory(path string, factory backendplugin.PluginFactoryFunc) (*plugins.Plugin, error) { - p, err := l.load(path, map[string]struct{}{}) - if err != nil { - l.log.Error("failed to load core plugin", "err", err) return nil, err } - err = l.pluginInitializer.InitializeWithFactory(p, factory) - - return p, err + return l.loadPlugins(ctx, class, pluginJSONPaths, ignore) } -func (l *Loader) load(path string, ignore map[string]struct{}) (*plugins.Plugin, error) { - pluginJSONPaths, err := l.pluginFinder.Find([]string{path}) - if err != nil { - l.log.Error("failed to find plugin", "err", err) - return nil, err - } - - loadedPlugins, err := l.loadPlugins(pluginJSONPaths, ignore) - if err != nil { - return nil, err - } - - if len(loadedPlugins) == 0 { - return nil, fmt.Errorf("could not load plugin at path %s", path) - } - - return loadedPlugins[0], nil -} - -func (l *Loader) loadPlugins(pluginJSONPaths []string, existingPlugins map[string]struct{}) ([]*plugins.Plugin, error) { +func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, pluginJSONPaths []string, existingPlugins map[string]struct{}) ([]*plugins.Plugin, error) { var foundPlugins = foundPlugins{} // load plugin.json files and map directory to JSON data for _, pluginJSONPath := range pluginJSONPaths { plugin, err := l.readPluginJSON(pluginJSONPath) if err != nil { - l.log.Warn("Skipping plugin loading as it's plugin.json is invalid", "id", plugin.ID) + l.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "err", err) continue } pluginJSONAbsPath, err := filepath.Abs(pluginJSONPath) if err != nil { - l.log.Warn("Skipping plugin loading as full plugin.json path could not be calculated", "id", plugin.ID) + l.log.Warn("Skipping plugin loading as absolute plugin.json path could not be calculated", "pluginID", plugin.ID, "err", err) continue } if _, dupe := foundPlugins[filepath.Dir(pluginJSONAbsPath)]; dupe { - l.log.Warn("Skipping plugin loading as it's a duplicate", "id", plugin.ID) + l.log.Warn("Skipping plugin loading as it's a duplicate", "pluginID", plugin.ID) continue } foundPlugins[filepath.Dir(pluginJSONAbsPath)] = plugin @@ -121,11 +97,7 @@ func (l *Loader) loadPlugins(pluginJSONPaths []string, existingPlugins map[strin // calculate initial signature state loadedPlugins := make(map[string]*plugins.Plugin) for pluginDir, pluginJSON := range foundPlugins { - plugin := &plugins.Plugin{ - JSONData: pluginJSON, - PluginDir: pluginDir, - Class: l.pluginClass(pluginDir), - } + plugin := createPluginBase(pluginJSON, class, pluginDir, l.log) sig, err := signature.Calculate(l.log, plugin) if err != nil { @@ -160,7 +132,7 @@ func (l *Loader) loadPlugins(pluginJSONPaths []string, existingPlugins map[strin } // validate signatures - verifiedPlugins := []*plugins.Plugin{} + verifiedPlugins := make([]*plugins.Plugin, 0) for _, plugin := range loadedPlugins { signingError := l.signatureValidator.Validate(plugin) if signingError != nil { @@ -188,14 +160,23 @@ func (l *Loader) loadPlugins(pluginJSONPaths []string, existingPlugins map[strin } } + if plugin.IsApp() { + setDefaultNavURL(plugin, l.cfg.AppSubURL) + } + + if plugin.Parent != nil && plugin.Parent.IsApp() { + configureAppChildOPlugin(plugin.Parent, plugin) + } + verifiedPlugins = append(verifiedPlugins, plugin) } for _, p := range verifiedPlugins { - err := l.pluginInitializer.Initialize(p) + err := l.pluginInitializer.Initialize(ctx, p) if err != nil { return nil, err } + metrics.SetPluginBuildInformation(p.ID, string(p.Type), p.Info.Version, string(p.Signature)) } return verifiedPlugins, nil @@ -233,9 +214,107 @@ func (l *Loader) readPluginJSON(pluginJSONPath string) (plugins.JSONData, error) plugin.Name = "Pie Chart (old)" } + if len(plugin.Dependencies.Plugins) == 0 { + plugin.Dependencies.Plugins = []plugins.Dependency{} + } + + if plugin.Dependencies.GrafanaVersion == "" { + plugin.Dependencies.GrafanaVersion = "*" + } + + for _, include := range plugin.Includes { + if include.Role == "" { + include.Role = models.ROLE_VIEWER + } + } + return plugin, nil } +func createPluginBase(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string, logger log.Logger) *plugins.Plugin { + plugin := &plugins.Plugin{ + JSONData: pluginJSON, + PluginDir: pluginDir, + BaseURL: baseURL(pluginJSON, class, pluginDir), + Module: module(pluginJSON, class, pluginDir), + Class: class, + } + + plugin.SetLogger(logger.New("pluginID", plugin.ID)) + setImages(plugin) + + return plugin +} + +func setImages(p *plugins.Plugin) { + p.Info.Logos.Small = pluginLogoURL(p.Type, p.Info.Logos.Small, p.BaseURL) + p.Info.Logos.Large = pluginLogoURL(p.Type, p.Info.Logos.Large, p.BaseURL) + + for i := 0; i < len(p.Info.Screenshots); i++ { + p.Info.Screenshots[i].Path = evalRelativePluginURLPath(p.Info.Screenshots[i].Path, p.BaseURL, p.Type) + } +} + +func setDefaultNavURL(p *plugins.Plugin, appSubURL string) { + // slugify pages + for _, include := range p.Includes { + if include.Slug == "" { + include.Slug = slug.Make(include.Name) + } + if include.Type == "page" && include.DefaultNav { + p.DefaultNavURL = appSubURL + "/plugins/" + p.ID + "/page/" + include.Slug + } + if include.Type == "dashboard" && include.DefaultNav { + p.DefaultNavURL = appSubURL + "/dashboard/db/" + include.Slug + } + } +} + +func configureAppChildOPlugin(parent *plugins.Plugin, child *plugins.Plugin) { + if !parent.IsApp() { + return + } + appSubPath := strings.ReplaceAll(strings.Replace(child.PluginDir, parent.PluginDir, "", 1), "\\", "/") + child.IncludedInAppID = parent.ID + child.BaseURL = parent.BaseURL + + if parent.IsCorePlugin() { + child.Module = util.JoinURLFragments("app/plugins/app/"+parent.ID, appSubPath) + "/module" + } else { + child.Module = util.JoinURLFragments("plugins/"+parent.ID, appSubPath) + "/module" + } +} + +func pluginLogoURL(pluginType plugins.Type, path, baseURL string) string { + if path == "" { + return defaultLogoPath(pluginType) + } + + return evalRelativePluginURLPath(path, baseURL, pluginType) +} + +func defaultLogoPath(pluginType plugins.Type) string { + return "public/img/icn-" + string(pluginType) + ".svg" +} + +func evalRelativePluginURLPath(pathStr, baseURL string, pluginType plugins.Type) string { + if pathStr == "" { + return "" + } + + u, _ := url.Parse(pathStr) + if u.IsAbs() { + return pathStr + } + + // is set as default or has already been prefixed with base path + if pathStr == defaultLogoPath(pluginType) || strings.HasPrefix(pathStr, baseURL) { + return pathStr + } + + return path.Join(baseURL, pathStr) +} + func (l *Loader) PluginErrors() []*plugins.Error { errs := make([]*plugins.Error, 0) for _, err := range l.errs { @@ -248,6 +327,22 @@ func (l *Loader) PluginErrors() []*plugins.Error { return errs } +func baseURL(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) string { + if class == plugins.Core { + return path.Join("public/app/plugins", string(pluginJSON.Type), filepath.Base(pluginDir)) + } + + return path.Join("public/plugins", pluginJSON.ID) +} + +func module(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) string { + if class == plugins.Core { + return path.Join("app/plugins", string(pluginJSON.Type), filepath.Base(pluginDir), "module") + } + + return path.Join("plugins", pluginJSON.ID, "module") +} + func validatePluginJSON(data plugins.JSONData) error { if data.ID == "" || !data.Type.IsValid() { return ErrInvalidPluginJSON @@ -255,41 +350,15 @@ func validatePluginJSON(data plugins.JSONData) error { return nil } -func (l *Loader) pluginClass(pluginDir string) plugins.Class { - isSubDir := func(base, target string) bool { - path, err := filepath.Rel(base, target) - if err != nil { - return false - } - - if !strings.HasPrefix(path, "..") { - return true - } - - return false - } - - corePluginsDir := filepath.Join(l.cfg.StaticRootPath, "app/plugins") - if isSubDir(corePluginsDir, pluginDir) { - return plugins.Core - } - - if isSubDir(l.cfg.BundledPluginsPath, pluginDir) { - return plugins.Bundled - } - - return plugins.External -} - type foundPlugins map[string]plugins.JSONData // stripDuplicates will strip duplicate plugins or plugins that already exist func (f *foundPlugins) stripDuplicates(existingPlugins map[string]struct{}, log log.Logger) { pluginsByID := make(map[string]struct{}) - for path, scannedPlugin := range *f { + for k, scannedPlugin := range *f { if _, existing := existingPlugins[scannedPlugin.ID]; existing { log.Debug("Skipping plugin as it's already installed", "plugin", scannedPlugin.ID) - delete(*f, path) + delete(*f, k) continue } diff --git a/pkg/plugins/manager/loader/loader_test.go b/pkg/plugins/manager/loader/loader_test.go index f1f158a9514..59cb67d8e86 100644 --- a/pkg/plugins/manager/loader/loader_test.go +++ b/pkg/plugins/manager/loader/loader_test.go @@ -1,19 +1,23 @@ package loader import ( + "context" "errors" "path/filepath" "sort" "testing" - "github.com/stretchr/testify/require" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin" + "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" + "github.com/grafana/grafana/pkg/plugins/backendplugin/provider" "github.com/grafana/grafana/pkg/plugins/manager/loader/finder" "github.com/grafana/grafana/pkg/plugins/manager/loader/initializer" "github.com/grafana/grafana/pkg/plugins/manager/signature" @@ -35,16 +39,18 @@ func TestLoader_Load(t *testing.T) { } tests := []struct { name string - cfg *setting.Cfg + class plugins.Class + cfg *plugins.Cfg pluginPaths []string existingPlugins map[string]struct{} want []*plugins.Plugin pluginErrors map[string]*plugins.Error }{ { - name: "Load a Core plugin", - cfg: &setting.Cfg{ - StaticRootPath: corePluginDir, + name: "Load a Core plugin", + class: plugins.Core, + cfg: &plugins.Cfg{ + PluginsPath: corePluginDir, }, pluginPaths: []string{filepath.Join(corePluginDir, "app/plugins/datasource/cloudwatch")}, want: []*plugins.Plugin{ @@ -86,15 +92,16 @@ func TestLoader_Load(t *testing.T) { Module: "app/plugins/datasource/cloudwatch/module", BaseURL: "public/app/plugins/datasource/cloudwatch", PluginDir: filepath.Join(corePluginDir, "app/plugins/datasource/cloudwatch"), - Signature: "internal", - Class: "core", + Signature: plugins.SignatureInternal, + Class: plugins.Core, }, }, }, { - name: "Load a Bundled plugin", - cfg: &setting.Cfg{ - BundledPluginsPath: filepath.Join(parentDir, "testdata"), + name: "Load a Bundled plugin", + class: plugins.Bundled, + cfg: &plugins.Cfg{ + PluginsPath: filepath.Join(parentDir, "testdata"), }, pluginPaths: []string{"../testdata/valid-v2-signature"}, want: []*plugins.Plugin{ @@ -129,12 +136,13 @@ func TestLoader_Load(t *testing.T) { Signature: "valid", SignatureType: plugins.GrafanaSignature, SignatureOrg: "Grafana Labs", - Class: "bundled", + Class: plugins.Bundled, }, }, }, { - name: "Load an External plugin", - cfg: &setting.Cfg{ + name: "Load plugin with symbolic links", + class: plugins.External, + cfg: &plugins.Cfg{ PluginsPath: filepath.Join(parentDir), }, pluginPaths: []string{"../testdata/symbolic-plugin-dirs"}, @@ -210,10 +218,11 @@ func TestLoader_Load(t *testing.T) { }, }, }, { - name: "Load an unsigned plugin (development)", - cfg: &setting.Cfg{ + name: "Load an unsigned plugin (development)", + class: plugins.External, + cfg: &plugins.Cfg{ + DevMode: true, PluginsPath: filepath.Join(parentDir), - Env: "development", }, pluginPaths: []string{"../testdata/unsigned-datasource"}, want: []*plugins.Plugin{ @@ -248,10 +257,10 @@ func TestLoader_Load(t *testing.T) { }, }, }, { - name: "Load an unsigned plugin (production)", - cfg: &setting.Cfg{ + name: "Load an unsigned plugin (production)", + class: plugins.External, + cfg: &plugins.Cfg{ PluginsPath: filepath.Join(parentDir), - Env: "production", }, pluginPaths: []string{"../testdata/unsigned-datasource"}, want: []*plugins.Plugin{}, @@ -263,10 +272,10 @@ func TestLoader_Load(t *testing.T) { }, }, { - name: "Load an unsigned plugin using PluginsAllowUnsigned config (production)", - cfg: &setting.Cfg{ + name: "Load an unsigned plugin using PluginsAllowUnsigned config (production)", + class: plugins.External, + cfg: &plugins.Cfg{ PluginsPath: filepath.Join(parentDir), - Env: "production", PluginsAllowUnsigned: []string{"test"}, }, pluginPaths: []string{"../testdata/unsigned-datasource"}, @@ -298,15 +307,15 @@ func TestLoader_Load(t *testing.T) { Module: "plugins/test/module", BaseURL: "public/plugins/test", PluginDir: filepath.Join(parentDir, "testdata/unsigned-datasource/plugin"), - Signature: "unsigned", + Signature: plugins.SignatureUnsigned, }, }, }, { - name: "Load an unsigned plugin with modified signature (production)", - cfg: &setting.Cfg{ + name: "Load an unsigned plugin with modified signature (production)", + class: plugins.External, + cfg: &plugins.Cfg{ PluginsPath: filepath.Join(parentDir), - Env: "production", }, pluginPaths: []string{"../testdata/lacking-files"}, want: []*plugins.Plugin{}, @@ -318,10 +327,10 @@ func TestLoader_Load(t *testing.T) { }, }, { - name: "Load an unsigned plugin with modified signature using PluginsAllowUnsigned config (production) still includes a signing error", - cfg: &setting.Cfg{ + name: "Load an unsigned plugin with modified signature using PluginsAllowUnsigned config (production) still includes a signing error", + class: plugins.External, + cfg: &plugins.Cfg{ PluginsPath: filepath.Join(parentDir), - Env: "production", PluginsAllowUnsigned: []string{"test"}, }, pluginPaths: []string{"../testdata/lacking-files"}, @@ -333,11 +342,61 @@ func TestLoader_Load(t *testing.T) { }, }, }, + { + name: "Load an app with includes", + class: plugins.External, + cfg: &plugins.Cfg{ + PluginsPath: filepath.Join(parentDir), + PluginsAllowUnsigned: []string{"test-app"}, + }, + pluginPaths: []string{"../testdata/test-app-with-includes"}, + want: []*plugins.Plugin{ + {JSONData: plugins.JSONData{ + ID: "test-app", + Type: "app", + Name: "Test App", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Test Inc.", + URL: "http://test.com", + }, + Description: "Official Grafana Test App & Dashboard bundle", + Version: "1.0.0", + Links: []plugins.InfoLink{ + {Name: "Project site", URL: "http://project.com"}, + {Name: "License & Terms", URL: "http://license.com"}, + }, + Logos: plugins.Logos{ + Small: "public/img/icn-app.svg", + Large: "public/img/icn-app.svg", + }, + Updated: "2015-02-10", + }, + Dependencies: plugins.Dependencies{ + GrafanaDependency: ">=8.0.0", + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + Includes: []*plugins.Includes{ + {Name: "Nginx Memory", Path: "dashboards/memory.json", Type: "dashboard", Role: "Viewer", Slug: "nginx-memory", DefaultNav: true}, + {Name: "Root Page (react)", Type: "page", Role: "Viewer", Path: "/a/my-simple-app", DefaultNav: true, AddToNav: true, Slug: "root-page-react"}, + }, + Backend: false, + }, + DefaultNavURL: "/plugins/test-app/page/root-page-react", + PluginDir: filepath.Join(parentDir, "testdata/test-app-with-includes"), + Class: plugins.External, + Signature: plugins.SignatureUnsigned, + Module: "plugins/test-app/module", + BaseURL: "public/plugins/test-app", + }, + }, + }, } for _, tt := range tests { l := newLoader(tt.cfg) t.Run(tt.name, func(t *testing.T) { - got, err := l.Load(tt.pluginPaths, tt.existingPlugins) + got, err := l.Load(context.Background(), tt.class, tt.pluginPaths, tt.existingPlugins) require.NoError(t, err) if !cmp.Equal(got, tt.want, compareOpts) { t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.want, compareOpts)) @@ -362,7 +421,7 @@ func TestLoader_Load_MultiplePlugins(t *testing.T) { t.Run("Load multiple", func(t *testing.T) { tests := []struct { name string - cfg *setting.Cfg + cfg *plugins.Cfg pluginPaths []string appURL string existingPlugins map[string]struct{} @@ -371,8 +430,7 @@ func TestLoader_Load_MultiplePlugins(t *testing.T) { }{ { name: "Load multiple plugins (broken, valid, unsigned)", - cfg: &setting.Cfg{ - Env: "production", + cfg: &plugins.Cfg{ PluginsPath: filepath.Join(parentDir), }, appURL: "http://localhost:3000", @@ -434,7 +492,7 @@ func TestLoader_Load_MultiplePlugins(t *testing.T) { }) setting.AppUrl = tt.appURL - got, err := l.Load(tt.pluginPaths, tt.existingPlugins) + got, err := l.Load(context.Background(), plugins.External, tt.pluginPaths, tt.existingPlugins) require.NoError(t, err) sort.SliceStable(got, func(i, j int) bool { return got[i].ID < got[j].ID @@ -488,17 +546,17 @@ func TestLoader_Signature_RootURL(t *testing.T) { Executable: "test", }, PluginDir: filepath.Join(parentDir, "/testdata/valid-v2-pvt-signature-root-url-uri/plugin"), - Class: "external", - Signature: "valid", - SignatureType: "private", + Class: plugins.External, + Signature: plugins.SignatureValid, + SignatureType: plugins.PrivateSignature, SignatureOrg: "Will Browne", Module: "plugins/test/module", BaseURL: "public/plugins/test", }, } - l := newLoader(&setting.Cfg{PluginsPath: filepath.Join(parentDir)}) - got, err := l.Load(paths, map[string]struct{}{}) + l := newLoader(&plugins.Cfg{PluginsPath: filepath.Join(parentDir)}) + got, err := l.Load(context.Background(), plugins.External, paths, map[string]struct{}{}) assert.NoError(t, err) if !cmp.Equal(got, expected, compareOpts) { @@ -566,11 +624,11 @@ func TestLoader_Load_DuplicatePlugins(t *testing.T) { }, } - l := newLoader(&setting.Cfg{ + l := newLoader(&plugins.Cfg{ PluginsPath: filepath.Dir(pluginDir), }) - got, err := l.Load([]string{pluginDir, pluginDir}, map[string]struct{}{}) + got, err := l.Load(context.Background(), plugins.External, []string{pluginDir, pluginDir}, map[string]struct{}{}) assert.NoError(t, err) if !cmp.Equal(got, expected, compareOpts) { @@ -580,7 +638,7 @@ func TestLoader_Load_DuplicatePlugins(t *testing.T) { } func TestLoader_loadNestedPlugins(t *testing.T) { - parentDir, err := filepath.Abs("../") + rootDir, err := filepath.Abs("../") if err != nil { t.Errorf("could not construct absolute path of root dir") return @@ -611,11 +669,11 @@ func TestLoader_loadNestedPlugins(t *testing.T) { }, Module: "plugins/test-ds/module", BaseURL: "public/plugins/test-ds", - PluginDir: filepath.Join(parentDir, "testdata/nested-plugins/parent"), - Signature: "valid", + PluginDir: filepath.Join(rootDir, "testdata/nested-plugins/parent"), + Signature: plugins.SignatureValid, SignatureType: plugins.GrafanaSignature, SignatureOrg: "Grafana Labs", - Class: "external", + Class: plugins.External, } child := &plugins.Plugin{ @@ -643,11 +701,11 @@ func TestLoader_loadNestedPlugins(t *testing.T) { }, Module: "plugins/test-panel/module", BaseURL: "public/plugins/test-panel", - PluginDir: filepath.Join(parentDir, "testdata/nested-plugins/parent/nested"), - Signature: "valid", + PluginDir: filepath.Join(rootDir, "testdata/nested-plugins/parent/nested"), + Signature: plugins.SignatureValid, SignatureType: plugins.GrafanaSignature, SignatureOrg: "Grafana Labs", - Class: "external", + Class: plugins.External, } parent.Children = []*plugins.Plugin{child} @@ -655,11 +713,11 @@ func TestLoader_loadNestedPlugins(t *testing.T) { t.Run("Load nested External plugins", func(t *testing.T) { expected := []*plugins.Plugin{parent, child} - l := newLoader(&setting.Cfg{ - PluginsPath: parentDir, + l := newLoader(&plugins.Cfg{ + PluginsPath: rootDir, }) - got, err := l.Load([]string{"../testdata/nested-plugins"}, map[string]struct{}{}) + got, err := l.Load(context.Background(), plugins.External, []string{"../testdata/nested-plugins"}, map[string]struct{}{}) assert.NoError(t, err) // to ensure we can compare with expected @@ -677,11 +735,11 @@ func TestLoader_loadNestedPlugins(t *testing.T) { parent.Children = nil expected := []*plugins.Plugin{parent} - l := newLoader(&setting.Cfg{ - PluginsPath: parentDir, + l := newLoader(&plugins.Cfg{ + PluginsPath: rootDir, }) - got, err := l.Load([]string{"../testdata/nested-plugins"}, map[string]struct{}{ + got, err := l.Load(context.Background(), plugins.External, []string{"../testdata/nested-plugins"}, map[string]struct{}{ "test-panel": {}, }) assert.NoError(t, err) @@ -695,6 +753,179 @@ func TestLoader_loadNestedPlugins(t *testing.T) { t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) } }) + + t.Run("Plugin child field `IncludedInAppID` is set to parent app's plugin ID", func(t *testing.T) { + parent := &plugins.Plugin{ + JSONData: plugins.JSONData{ + ID: "myorgid-simple-app", + Type: "app", + Name: "Simple App", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Your Name", + }, + Links: []plugins.InfoLink{ + {Name: "Website", URL: "https://github.com/grafana/grafana-starter-app"}, + {Name: "License", URL: "https://github.com/grafana/grafana-starter-app/blob/master/LICENSE"}, + }, + Logos: plugins.Logos{ + Small: "public/plugins/myorgid-simple-app/img/logo.svg", + Large: "public/plugins/myorgid-simple-app/img/logo.svg", + }, + Screenshots: []plugins.Screenshots{}, + Description: "Grafana App Plugin Template", + Version: "%VERSION%", + Updated: "%TODAY%", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "7.0.0", + GrafanaDependency: ">=7.0.0", + Plugins: []plugins.Dependency{}, + }, + Includes: []*plugins.Includes{ + { + Name: "Root Page (react)", + Path: "/a/myorgid-simple-app", + Type: "page", + Role: "Viewer", + AddToNav: true, + DefaultNav: true, + Slug: "root-page-react", + }, + { + Name: "Root Page (Tab B)", + Path: "/a/myorgid-simple-app/?tab=b", + Type: "page", + Role: "Viewer", + AddToNav: true, + Slug: "root-page-tab-b", + }, + { + Name: "React Config", + Path: "/plugins/myorgid-simple-app/?page=page2", + Type: "page", + Role: "Admin", + AddToNav: true, + Slug: "react-config", + }, + { + Name: "Streaming Example", + Path: "dashboards/streaming.json", + Type: "dashboard", + Role: "Viewer", + Slug: "streaming-example", + }, + { + Name: "Lots of Stats", + Path: "dashboards/stats.json", + Type: "dashboard", + Role: "Viewer", + Slug: "lots-of-stats", + }, + }, + Backend: false, + }, + Module: "plugins/myorgid-simple-app/module", + BaseURL: "public/plugins/myorgid-simple-app", + PluginDir: filepath.Join(rootDir, "testdata/app-with-child/dist"), + DefaultNavURL: "/plugins/myorgid-simple-app/page/root-page-react", + Signature: plugins.SignatureValid, + SignatureType: plugins.GrafanaSignature, + SignatureOrg: "Grafana Labs", + Class: plugins.External, + } + + child := &plugins.Plugin{ + JSONData: plugins.JSONData{ + ID: "myorgid-simple-panel", + Type: "panel", + Name: "Grafana Panel Plugin Template", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Your Name", + }, + Links: []plugins.InfoLink{ + {Name: "Website", URL: "https://github.com/grafana/grafana-starter-panel"}, + {Name: "License", URL: "https://github.com/grafana/grafana-starter-panel/blob/master/LICENSE"}, + }, + Logos: plugins.Logos{ + Small: "public/plugins/myorgid-simple-panel/img/logo.svg", + Large: "public/plugins/myorgid-simple-panel/img/logo.svg", + }, + Screenshots: []plugins.Screenshots{}, + Description: "Grafana Panel Plugin Template", + Version: "%VERSION%", + Updated: "%TODAY%", + }, + Dependencies: plugins.Dependencies{ + GrafanaDependency: ">=7.0.0", + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + }, + Module: "plugins/myorgid-simple-app/child/module", + BaseURL: "public/plugins/myorgid-simple-app", + PluginDir: filepath.Join(rootDir, "testdata/app-with-child/dist/child"), + IncludedInAppID: parent.ID, + Signature: plugins.SignatureValid, + SignatureType: plugins.GrafanaSignature, + SignatureOrg: "Grafana Labs", + Class: plugins.External, + } + + parent.Children = []*plugins.Plugin{child} + child.Parent = parent + + expected := []*plugins.Plugin{parent, child} + l := newLoader(&plugins.Cfg{ + PluginsPath: rootDir, + }) + + got, err := l.Load(context.Background(), plugins.External, []string{"../testdata/app-with-child"}, map[string]struct{}{}) + assert.NoError(t, err) + + // to ensure we can compare with expected + sort.SliceStable(got, func(i, j int) bool { + return got[i].ID < got[j].ID + }) + + if !cmp.Equal(got, expected, compareOpts) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + } + + t.Run("order of loaded parent and child plugins gives same output", func(t *testing.T) { + parentPluginJSON := filepath.Join(rootDir, "testdata/app-with-child/dist/plugin.json") + childPluginJSON := filepath.Join(rootDir, "testdata/app-with-child/dist/child/plugin.json") + + got, err := l.loadPlugins(context.Background(), plugins.External, []string{ + parentPluginJSON, childPluginJSON}, + map[string]struct{}{}) + assert.NoError(t, err) + + // to ensure we can compare with expected + sort.SliceStable(got, func(i, j int) bool { + return got[i].ID < got[j].ID + }) + + if !cmp.Equal(got, expected, compareOpts) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + } + + got, err = l.loadPlugins(context.Background(), plugins.External, []string{ + childPluginJSON, parentPluginJSON}, + map[string]struct{}{}) + assert.NoError(t, err) + + // to ensure we can compare with expected + sort.SliceStable(got, func(i, j int) bool { + return got[i].ID < got[j].ID + }) + + if !cmp.Equal(got, expected, compareOpts) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + } + }) + }) } func TestLoader_readPluginJSON(t *testing.T) { @@ -740,10 +971,10 @@ func TestLoader_readPluginJSON(t *testing.T) { }, }, Includes: []*plugins.Includes{ - {Name: "Nginx Connections", Path: "dashboards/connections.json", Type: "dashboard"}, - {Name: "Nginx Memory", Path: "dashboards/memory.json", Type: "dashboard"}, - {Name: "Nginx Panel", Type: "panel"}, - {Name: "Nginx Datasource", Type: "datasource"}, + {Name: "Nginx Connections", Path: "dashboards/connections.json", Type: "dashboard", Role: models.ROLE_VIEWER}, + {Name: "Nginx Memory", Path: "dashboards/memory.json", Type: "dashboard", Role: models.ROLE_VIEWER}, + {Name: "Nginx Panel", Type: "panel", Role: models.ROLE_VIEWER}, + {Name: "Nginx Datasource", Type: "datasource", Role: models.ROLE_VIEWER}, }, Backend: false, }, @@ -822,72 +1053,35 @@ func Test_validatePluginJSON(t *testing.T) { } } -func Test_pluginClass(t *testing.T) { - type args struct { - pluginDir string - cfg *setting.Cfg - } - tests := []struct { - name string - args args - expected plugins.Class - }{ - { - name: "Core plugin class", - args: args{ - pluginDir: "/root/app/plugins/test-app", - cfg: &setting.Cfg{ - StaticRootPath: "/root", - }, +func Test_setPathsBasedOnApp(t *testing.T) { + t.Run("When setting paths based on core plugin on Windows", func(t *testing.T) { + child := &plugins.Plugin{ + PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata\\datasources\\datasource", + } + parent := &plugins.Plugin{ + JSONData: plugins.JSONData{ + Type: plugins.App, + ID: "testdata", }, - expected: plugins.Core, - }, - { - name: "Bundled plugin class", - args: args{ - pluginDir: "/test-app", - cfg: &setting.Cfg{ - BundledPluginsPath: "/test-app", - }, - }, - expected: plugins.Bundled, - }, - { - name: "External plugin class", - args: args{ - pluginDir: "/test-app", - cfg: &setting.Cfg{ - PluginsPath: "/test-app", - }, - }, - expected: plugins.External, - }, - { - name: "External plugin class", - args: args{ - pluginDir: "/test-app", - cfg: &setting.Cfg{ - PluginsPath: "/root", - }, - }, - expected: plugins.External, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - l := newLoader(tt.args.cfg) - got := l.pluginClass(tt.args.pluginDir) - assert.Equal(t, tt.expected, got) - }) - } + Class: plugins.Core, + PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata", + BaseURL: "public/app/plugins/app/testdata", + } + + configureAppChildOPlugin(parent, child) + + assert.Equal(t, "app/plugins/app/testdata/datasources/datasource/module", child.Module) + assert.Equal(t, "testdata", child.IncludedInAppID) + assert.Equal(t, "public/app/plugins/app/testdata", child.BaseURL) + }) } -func newLoader(cfg *setting.Cfg) *Loader { +func newLoader(cfg *plugins.Cfg) *Loader { return &Loader{ cfg: cfg, - pluginFinder: finder.New(cfg), - pluginInitializer: initializer.New(cfg, &fakeLicensingService{}), - signatureValidator: signature.NewValidator(&signature.UnsignedPluginAuthorizer{Cfg: cfg}), + pluginFinder: finder.New(), + pluginInitializer: initializer.New(cfg, provider.ProvideService(coreplugin.NewRegistry(make(map[string]backendplugin.PluginFactoryFunc))), &fakeLicensingService{}), + signatureValidator: signature.NewValidator(signature.NewUnsignedAuthorizer(cfg)), errs: make(map[string]*plugins.SignatureError), log: &fakeLogger{}, } @@ -934,6 +1128,10 @@ type fakeLogger struct { log.Logger } +func (fl fakeLogger) New(_ ...interface{}) log.MultiLoggers { + return log.MultiLoggers{} +} + func (fl fakeLogger) Info(_ string, _ ...interface{}) { } diff --git a/pkg/plugins/manager/manager.go b/pkg/plugins/manager/manager.go index 1582905d276..d795e7806c9 100644 --- a/pkg/plugins/manager/manager.go +++ b/pkg/plugins/manager/manager.go @@ -4,22 +4,17 @@ import ( "context" "errors" "fmt" - "net/http" - "os" "path/filepath" "sync" "time" "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/backendplugin" "github.com/grafana/grafana/pkg/plugins/backendplugin/instrumentation" "github.com/grafana/grafana/pkg/plugins/manager/installer" - "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util/errutil" ) @@ -35,101 +30,50 @@ var _ plugins.StaticRouteResolver = (*PluginManager)(nil) var _ plugins.RendererManager = (*PluginManager)(nil) type PluginManager struct { - cfg *setting.Cfg - requestValidator models.PluginRequestValidator - sqlStore *sqlstore.SQLStore - store map[string]*plugins.Plugin - pluginInstaller plugins.Installer - pluginLoader plugins.Loader - pluginsMu sync.RWMutex - log log.Logger + cfg *plugins.Cfg + store map[string]*plugins.Plugin + pluginInstaller plugins.Installer + pluginLoader plugins.Loader + pluginsMu sync.RWMutex + pluginPaths map[plugins.Class][]string + log log.Logger } -func ProvideService(cfg *setting.Cfg, requestValidator models.PluginRequestValidator, pluginLoader plugins.Loader, - sqlStore *sqlstore.SQLStore) (*PluginManager, error) { - pm := newManager(cfg, requestValidator, pluginLoader, sqlStore) - if err := pm.init(); err != nil { +func ProvideService(grafanaCfg *setting.Cfg, pluginLoader plugins.Loader) (*PluginManager, error) { + pm := New(plugins.FromGrafanaCfg(grafanaCfg), map[plugins.Class][]string{ + plugins.Core: corePluginPaths(grafanaCfg), + plugins.Bundled: {grafanaCfg.BundledPluginsPath}, + plugins.External: append([]string{grafanaCfg.PluginsPath}, pluginSettingPaths(grafanaCfg)...), + }, pluginLoader) + if err := pm.Init(); err != nil { return nil, err } return pm, nil } -func newManager(cfg *setting.Cfg, pluginRequestValidator models.PluginRequestValidator, pluginLoader plugins.Loader, - sqlStore *sqlstore.SQLStore) *PluginManager { +func New(cfg *plugins.Cfg, pluginPaths map[plugins.Class][]string, pluginLoader plugins.Loader) *PluginManager { return &PluginManager{ - cfg: cfg, - requestValidator: pluginRequestValidator, - sqlStore: sqlStore, - pluginLoader: pluginLoader, - store: map[string]*plugins.Plugin{}, - log: log.New("plugin.manager"), - pluginInstaller: installer.New(false, cfg.BuildVersion, newInstallerLogger("plugin.installer", true)), + cfg: cfg, + pluginLoader: pluginLoader, + pluginPaths: pluginPaths, + store: make(map[string]*plugins.Plugin), + log: log.New("plugin.manager"), + pluginInstaller: installer.New(false, cfg.BuildVersion, newInstallerLogger("plugin.installer", true)), } } -func (m *PluginManager) init() error { - // create external plugin's path if not exists - exists, err := fs.Exists(m.cfg.PluginsPath) - if err != nil { - return err - } - - if !exists { - if err = os.MkdirAll(m.cfg.PluginsPath, os.ModePerm); err != nil { - m.log.Error("Failed to create external plugins directory", "dir", m.cfg.PluginsPath, "error", err) - } else { - m.log.Debug("External plugins directory created", "dir", m.cfg.PluginsPath) +func (m *PluginManager) Init() error { + for class, paths := range m.pluginPaths { + err := m.loadPlugins(context.Background(), class, paths...) + if err != nil { + return err } } - m.log.Info("Initialising plugins") - - // install Core plugins - err = m.loadPlugins(m.corePluginPaths()...) - if err != nil { - return err - } - - // install Bundled plugins - err = m.loadPlugins(m.cfg.BundledPluginsPath) - if err != nil { - return err - } - - // install External plugins - err = m.loadPlugins(m.cfg.PluginsPath) - if err != nil { - return err - } - - // install plugins from cfg.PluginSettings - err = m.loadPlugins(m.pluginSettingPaths()...) - if err != nil { - return err - } - return nil } func (m *PluginManager) Run(ctx context.Context) error { - if m.cfg.CheckForUpdates { - go func() { - m.checkForUpdates(ctx) - - ticker := time.NewTicker(time.Minute * 10) - run := true - - for run { - select { - case <-ticker.C: - m.checkForUpdates(ctx) - case <-ctx.Done(): - run = false - } - } - }() - } - <-ctx.Done() m.shutdown(ctx) return ctx.Err() @@ -161,7 +105,7 @@ func (m *PluginManager) plugins() []*plugins.Plugin { return res } -func (m *PluginManager) loadPlugins(paths ...string) error { +func (m *PluginManager) loadPlugins(ctx context.Context, class plugins.Class, paths ...string) error { if len(paths) == 0 { return nil } @@ -173,7 +117,7 @@ func (m *PluginManager) loadPlugins(paths ...string) error { } } - loadedPlugins, err := m.pluginLoader.Load(pluginPaths, m.registeredPlugins()) + loadedPlugins, err := m.pluginLoader.Load(ctx, class, pluginPaths, m.registeredPlugins()) if err != nil { m.log.Error("Could not load plugins", "paths", pluginPaths, "err", err) return err @@ -282,26 +226,13 @@ func (m *PluginManager) CollectMetrics(ctx context.Context, pluginID string) (*b } func (m *PluginManager) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { - var dsURL string - if req.PluginContext.DataSourceInstanceSettings != nil { - dsURL = req.PluginContext.DataSourceInstanceSettings.URL - } - - err := m.requestValidator.Validate(dsURL, nil) - if err != nil { - return &backend.CheckHealthResult{ - Status: http.StatusForbidden, - Message: "Access denied", - }, nil - } - p, exists := m.plugin(req.PluginContext.PluginID) if !exists { return nil, backendplugin.ErrPluginNotRegistered } var resp *backend.CheckHealthResult - err = instrumentation.InstrumentCheckHealthRequest(p.PluginID(), func() (innerErr error) { + err := instrumentation.InstrumentCheckHealthRequest(p.PluginID(), func() (innerErr error) { resp, innerErr = p.CheckHealth(ctx, &backend.CheckHealthRequest{PluginContext: req.PluginContext}) return }) @@ -499,24 +430,17 @@ func (m *PluginManager) shutdown(ctx context.Context) { } // corePluginPaths provides a list of the Core plugin paths which need to be scanned on init() -func (m *PluginManager) corePluginPaths() []string { - datasourcePaths := []string{ - filepath.Join(m.cfg.StaticRootPath, "app/plugins/datasource/alertmanager"), - filepath.Join(m.cfg.StaticRootPath, "app/plugins/datasource/dashboard"), - filepath.Join(m.cfg.StaticRootPath, "app/plugins/datasource/jaeger"), - filepath.Join(m.cfg.StaticRootPath, "app/plugins/datasource/mixed"), - filepath.Join(m.cfg.StaticRootPath, "app/plugins/datasource/zipkin"), - } +func corePluginPaths(cfg *setting.Cfg) []string { + datasourcePaths := filepath.Join(cfg.StaticRootPath, "app/plugins/datasource") + panelsPath := filepath.Join(cfg.StaticRootPath, "app/plugins/panel") - panelsPath := filepath.Join(m.cfg.StaticRootPath, "app/plugins/panel") - - return append(datasourcePaths, panelsPath) + return []string{datasourcePaths, panelsPath} } // pluginSettingPaths provides a plugin paths defined in cfg.PluginSettings which need to be scanned on init() -func (m *PluginManager) pluginSettingPaths() []string { +func pluginSettingPaths(cfg *setting.Cfg) []string { var pluginSettingDirs []string - for _, settings := range m.cfg.PluginSettings { + for _, settings := range cfg.PluginSettings { path, exists := settings["path"] if !exists || path == "" { continue diff --git a/pkg/plugins/manager/manager_integration_test.go b/pkg/plugins/manager/manager_integration_test.go index d7283b7649d..7397fb1814e 100644 --- a/pkg/plugins/manager/manager_integration_test.go +++ b/pkg/plugins/manager/manager_integration_test.go @@ -2,15 +2,39 @@ package manager import ( "context" + "net/http" "path/filepath" "strings" "testing" + "github.com/grafana/grafana/pkg/infra/tracing" + "go.opentelemetry.io/otel/trace" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" + "github.com/grafana/grafana/pkg/plugins/backendplugin/provider" "github.com/grafana/grafana/pkg/plugins/manager/loader" "github.com/grafana/grafana/pkg/plugins/manager/signature" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb/azuremonitor" + "github.com/grafana/grafana/pkg/tsdb/cloudmonitoring" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch" + "github.com/grafana/grafana/pkg/tsdb/grafanads" + "github.com/grafana/grafana/pkg/tsdb/graphite" + "github.com/grafana/grafana/pkg/tsdb/influxdb" + "github.com/grafana/grafana/pkg/tsdb/loki" + "github.com/grafana/grafana/pkg/tsdb/mssql" + "github.com/grafana/grafana/pkg/tsdb/mysql" + "github.com/grafana/grafana/pkg/tsdb/opentsdb" + "github.com/grafana/grafana/pkg/tsdb/postgres" + "github.com/grafana/grafana/pkg/tsdb/prometheus" + "github.com/grafana/grafana/pkg/tsdb/tempo" + "github.com/grafana/grafana/pkg/tsdb/testdatasource" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,11 +51,13 @@ func TestPluginManager_int_init(t *testing.T) { bundledPluginsPath, err := filepath.Abs("../../../plugins-bundled/internal") require.NoError(t, err) + features := featuremgmt.WithFeatures() cfg := &setting.Cfg{ - Raw: ini.Empty(), - Env: setting.Prod, - StaticRootPath: staticRootPath, - BundledPluginsPath: bundledPluginsPath, + Raw: ini.Empty(), + Env: setting.Prod, + StaticRootPath: staticRootPath, + BundledPluginsPath: bundledPluginsPath, + IsFeatureToggleEnabled: features.IsEnabled, PluginSettings: map[string]map[string]string{ "plugin.datasource-id": { "path": "testdata/test-app", @@ -39,12 +65,34 @@ func TestPluginManager_int_init(t *testing.T) { }, } + tracer := &fakeTracer{} + license := &licensing.OSSLicensingService{ Cfg: cfg, } - pm := newManager(cfg, nil, loader.New(license, cfg, &signature.UnsignedPluginAuthorizer{Cfg: cfg}), nil) - err = pm.init() + hcp := httpclient.NewProvider() + am := azuremonitor.ProvideService(cfg, hcp, tracer) + cw := cloudwatch.ProvideService(cfg, hcp) + cm := cloudmonitoring.ProvideService(hcp, tracer) + es := elasticsearch.ProvideService(hcp) + grap := graphite.ProvideService(hcp, tracer) + idb := influxdb.ProvideService(hcp) + lk := loki.ProvideService(hcp, tracer) + otsdb := opentsdb.ProvideService(hcp) + pr := prometheus.ProvideService(hcp, tracer) + tmpo := tempo.ProvideService(hcp) + td := testdatasource.ProvideService(cfg, features) + pg := postgres.ProvideService(cfg) + my := mysql.ProvideService(cfg, hcp) + ms := mssql.ProvideService(cfg) + graf := grafanads.ProvideService(cfg) + + coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf) + + pmCfg := plugins.FromGrafanaCfg(cfg) + pm, err := ProvideService(cfg, loader.New(pmCfg, license, signature.NewUnsignedAuthorizer(pmCfg), + provider.ProvideService(coreRegistry))) require.NoError(t, err) verifyCorePluginCatalogue(t, pm) @@ -90,12 +138,27 @@ func verifyCorePluginCatalogue(t *testing.T, pm *PluginManager) { } expDataSources := map[string]struct{}{ - "alertmanager": {}, - "dashboard": {}, - "input": {}, - "jaeger": {}, - "mixed": {}, - "zipkin": {}, + "cloudwatch": {}, + "stackdriver": {}, + "grafana-azure-monitor-datasource": {}, + "elasticsearch": {}, + "graphite": {}, + "influxdb": {}, + "loki": {}, + "opentsdb": {}, + "prometheus": {}, + "tempo": {}, + "testdata": {}, + "postgres": {}, + "mysql": {}, + "mssql": {}, + "grafana": {}, + "alertmanager": {}, + "dashboard": {}, + "input": {}, + "jaeger": {}, + "mixed": {}, + "zipkin": {}, } expApps := map[string]struct{}{ @@ -175,3 +238,19 @@ func verifyPluginStaticRoutes(t *testing.T, pm *PluginManager) { assert.Contains(t, routes, "test-app") assert.Equal(t, routes["test-app"].Directory, testAppPlugin.PluginDir) } + +type fakeTracer struct { + tracing.Tracer +} + +func (ft *fakeTracer) Run(context.Context) error { + return nil +} + +func (ft *fakeTracer) Start(ctx context.Context, _ string, _ ...trace.SpanStartOption) (context.Context, tracing.Span) { + return ctx, nil +} + +func (ft *fakeTracer) Inject(context.Context, http.Header, tracing.Span) { + +} diff --git a/pkg/plugins/manager/manager_test.go b/pkg/plugins/manager/manager_test.go index 26409d397b6..e7d550d4165 100644 --- a/pkg/plugins/manager/manager_test.go +++ b/pkg/plugins/manager/manager_test.go @@ -3,56 +3,23 @@ package manager import ( "context" "net/http" - "os" - "path/filepath" "sync" "testing" "time" "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/backendplugin" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/setting" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/ini.v1" ) const ( testPluginID = "test-plugin" ) -func TestPluginManager_init(t *testing.T) { - t.Run("Plugin folder will be created if not exists", func(t *testing.T) { - testDir := "plugin-test-dir" - - exists, err := fs.Exists(testDir) - require.NoError(t, err) - assert.False(t, exists) - - pm := createManager(t, func(pm *PluginManager) { - pm.cfg.PluginsPath = testDir - }) - - err = pm.init() - require.NoError(t, err) - - exists, err = fs.Exists(testDir) - require.NoError(t, err) - assert.True(t, exists) - - t.Cleanup(func() { - err = os.Remove(testDir) - require.NoError(t, err) - }) - }) -} - func TestPluginManager_loadPlugins(t *testing.T) { t.Run("Managed backend plugin", func(t *testing.T) { p, pc := createPlugin(testPluginID, "", plugins.External, true, true) @@ -64,7 +31,7 @@ func TestPluginManager_loadPlugins(t *testing.T) { pm := createManager(t, func(pm *PluginManager) { pm.pluginLoader = loader }) - err := pm.loadPlugins("test/path") + err := pm.loadPlugins(context.Background(), plugins.External, "test/path") require.NoError(t, err) assert.Equal(t, 1, pc.startCount) @@ -90,7 +57,7 @@ func TestPluginManager_loadPlugins(t *testing.T) { pm := createManager(t, func(pm *PluginManager) { pm.pluginLoader = loader }) - err := pm.loadPlugins("test/path") + err := pm.loadPlugins(context.Background(), plugins.External, "test/path") require.NoError(t, err) assert.Equal(t, 0, pc.startCount) @@ -116,7 +83,7 @@ func TestPluginManager_loadPlugins(t *testing.T) { pm := createManager(t, func(pm *PluginManager) { pm.pluginLoader = loader }) - err := pm.loadPlugins("test/path") + err := pm.loadPlugins(context.Background(), plugins.External, "test/path") require.NoError(t, err) assert.Equal(t, 0, pc.startCount) @@ -142,7 +109,7 @@ func TestPluginManager_loadPlugins(t *testing.T) { pm := createManager(t, func(pm *PluginManager) { pm.pluginLoader = loader }) - err := pm.loadPlugins("test/path") + err := pm.loadPlugins(context.Background(), plugins.External, "test/path") require.NoError(t, err) assert.Equal(t, 0, pc.startCount) @@ -257,7 +224,7 @@ func TestPluginManager_Installer(t *testing.T) { pm := createManager(t, func(pm *PluginManager) { pm.pluginLoader = loader }) - err := pm.loadPlugins("test/path") + err := pm.loadPlugins(context.Background(), plugins.Core, "test/path") require.NoError(t, err) assert.Equal(t, 1, pc.startCount) @@ -291,7 +258,7 @@ func TestPluginManager_Installer(t *testing.T) { pm := createManager(t, func(pm *PluginManager) { pm.pluginLoader = loader }) - err := pm.loadPlugins("test/path") + err := pm.loadPlugins(context.Background(), plugins.Bundled, "test/path") require.NoError(t, err) assert.Equal(t, 1, pc.startCount) @@ -499,18 +466,7 @@ func TestPluginManager_lifecycle_unmanaged(t *testing.T) { func createManager(t *testing.T, cbs ...func(*PluginManager)) *PluginManager { t.Helper() - staticRootPath, err := filepath.Abs("../../../public/") - require.NoError(t, err) - - cfg := &setting.Cfg{ - Raw: ini.Empty(), - Env: setting.Prod, - StaticRootPath: staticRootPath, - } - - requestValidator := &testPluginRequestValidator{} - loader := &fakeLoader{} - pm := newManager(cfg, requestValidator, loader, &sqlstore.SQLStore{}) + pm := New(&plugins.Cfg{}, nil, &fakeLoader{}) for _, cb := range cbs { cb(pm) @@ -555,7 +511,7 @@ type managerScenarioCtx struct { func newScenario(t *testing.T, managed bool, fn func(t *testing.T, ctx *managerScenarioCtx)) { t.Helper() - cfg := setting.NewCfg() + cfg := &plugins.Cfg{} cfg.AWSAllowedAuthProviders = []string{"keys", "credentials"} cfg.AWSAssumeRoleEnabled = true @@ -563,13 +519,8 @@ func newScenario(t *testing.T, managed bool, fn func(t *testing.T, ctx *managerS cfg.Azure.Cloud = "AzureCloud" cfg.Azure.ManagedIdentityClientId = "client-id" - staticRootPath, err := filepath.Abs("../../../public") - require.NoError(t, err) - cfg.StaticRootPath = staticRootPath - - requestValidator := &testPluginRequestValidator{} loader := &fakeLoader{} - manager := newManager(cfg, requestValidator, loader, nil) + manager := New(cfg, nil, loader) manager.pluginLoader = loader ctx := &managerScenarioCtx{ manager: manager, @@ -616,13 +567,13 @@ type fakeLoader struct { plugins.Loader } -func (l *fakeLoader) Load(paths []string, _ map[string]struct{}) ([]*plugins.Plugin, error) { +func (l *fakeLoader) Load(_ context.Context, _ plugins.Class, paths []string, _ map[string]struct{}) ([]*plugins.Plugin, error) { l.loadedPaths = append(l.loadedPaths, paths...) return l.mockedLoadedPlugins, nil } -func (l *fakeLoader) LoadWithFactory(path string, _ backendplugin.PluginFactoryFunc) (*plugins.Plugin, error) { +func (l *fakeLoader) LoadWithFactory(_ context.Context, _ plugins.Class, path string, _ backendplugin.PluginFactoryFunc) (*plugins.Plugin, error) { l.loadedPaths = append(l.loadedPaths, path) return l.mockedFactoryLoadedPlugin, nil @@ -744,12 +695,6 @@ func (pc *fakePluginClient) RunStream(_ context.Context, _ *backend.RunStreamReq return backendplugin.ErrMethodNotImplemented } -type testPluginRequestValidator struct{} - -func (t *testPluginRequestValidator) Validate(string, *http.Request) error { - return nil -} - type fakeLogger struct { log.Logger } diff --git a/pkg/plugins/manager/signature/authorizer.go b/pkg/plugins/manager/signature/authorizer.go index ac4b4f19f9e..24a8c97596a 100644 --- a/pkg/plugins/manager/signature/authorizer.go +++ b/pkg/plugins/manager/signature/authorizer.go @@ -5,14 +5,18 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -func ProvideService(cfg *setting.Cfg) (*UnsignedPluginAuthorizer, error) { +func NewUnsignedAuthorizer(cfg *plugins.Cfg) *UnsignedPluginAuthorizer { return &UnsignedPluginAuthorizer{ - Cfg: cfg, - }, nil + cfg: cfg, + } +} + +func ProvideOSSAuthorizer(cfg *setting.Cfg) *UnsignedPluginAuthorizer { + return NewUnsignedAuthorizer(plugins.FromGrafanaCfg(cfg)) } type UnsignedPluginAuthorizer struct { - Cfg *setting.Cfg + cfg *plugins.Cfg } func (u *UnsignedPluginAuthorizer) CanLoadPlugin(p *plugins.Plugin) bool { @@ -20,11 +24,11 @@ func (u *UnsignedPluginAuthorizer) CanLoadPlugin(p *plugins.Plugin) bool { return true } - if u.Cfg.Env == setting.Dev { + if u.cfg.DevMode { return true } - for _, pID := range u.Cfg.PluginsAllowUnsigned { + for _, pID := range u.cfg.PluginsAllowUnsigned { if pID == p.ID { return true } diff --git a/pkg/plugins/manager/store.go b/pkg/plugins/manager/store.go index 931b93e5df4..ebf50c48755 100644 --- a/pkg/plugins/manager/store.go +++ b/pkg/plugins/manager/store.go @@ -2,12 +2,10 @@ package manager import ( "context" - "fmt" "path/filepath" "strings" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin" ) func (m *PluginManager) Plugin(_ context.Context, pluginID string) (plugins.PluginDTO, bool) { @@ -75,7 +73,7 @@ func (m *PluginManager) Add(ctx context.Context, pluginID, version string) error return err } - err = m.loadPlugins(m.cfg.PluginsPath) + err = m.loadPlugins(context.Background(), plugins.External, m.cfg.PluginsPath) if err != nil { return err } @@ -83,28 +81,6 @@ func (m *PluginManager) Add(ctx context.Context, pluginID, version string) error return nil } -func (m *PluginManager) AddWithFactory(_ context.Context, pluginID string, factory backendplugin.PluginFactoryFunc, - pathResolver plugins.PluginPathResolver) error { - if m.isRegistered(pluginID) { - return fmt.Errorf("plugin %s is already registered", pluginID) - } - - path, err := pathResolver() - if err != nil { - return err - } - - p, err := m.pluginLoader.LoadWithFactory(path, factory) - if err != nil { - return err - } - err = m.register(p) - if err != nil { - return err - } - return nil -} - func (m *PluginManager) Remove(ctx context.Context, pluginID string) error { plugin, exists := m.plugin(pluginID) if !exists { diff --git a/pkg/plugins/manager/testdata/app-with-child/dist/MANIFEST.txt b/pkg/plugins/manager/testdata/app-with-child/dist/MANIFEST.txt new file mode 100644 index 00000000000..08f657aff0c --- /dev/null +++ b/pkg/plugins/manager/testdata/app-with-child/dist/MANIFEST.txt @@ -0,0 +1,29 @@ + +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +{ + "manifestVersion": "2.0.0", + "signatureType": "grafana", + "signedByOrg": "grafana", + "signedByOrgName": "Grafana Labs", + "rootUrls": [], + "plugin": "myorgid-simple-app", + "version": "%VERSION%", + "time": 1642614241713, + "keyId": "7e4d0c6a708866e7", + "files": { + "plugin.json": "1abecfd0229814f6c284ff3c8dd744548f8d676ab3250cd7902c99dabf11480e", + "child/plugin.json": "66ba0dffaf3b1bfa17eb9a8672918fc66d1001f465b1061f4fc19c2f2c100f51" + } +} +-----BEGIN PGP SIGNATURE----- +Version: OpenPGP.js v4.10.1 +Comment: https://openpgpjs.org + +wqEEARMKAAYFAmHoTeEACgkQfk0ManCIZudvLwIJAQcy8NwEZZb58SuD9fhS +kaWb9mxvhdAL6Zau+AZKQ3w2ZKzynwgKK/xwDSnX8BNI5UjcCh+0WxkpiAHL +bm6/dhzdAgi7Mo/W5IyXqMt9/0AyyFNA1wVlLCj47C+4op4hIw054ZJ9u2j9 +KCVjajZ9bGaV6ucBnG5/l4MLFhUK1M0slI2qvw== +=atro +-----END PGP SIGNATURE----- diff --git a/pkg/plugins/manager/testdata/app-with-child/dist/child/plugin.json b/pkg/plugins/manager/testdata/app-with-child/dist/child/plugin.json new file mode 100644 index 00000000000..34403080172 --- /dev/null +++ b/pkg/plugins/manager/testdata/app-with-child/dist/child/plugin.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://raw.githubusercontent.com/grafana/grafana/master/docs/sources/developers/plugins/plugin.schema.json", + "type": "panel", + "name": "Grafana Panel Plugin Template", + "id": "myorgid-simple-panel", + + "info": { + "description": "Grafana Panel Plugin Template", + "author": { + "name": "Your Name" + }, + "keywords": ["panel", "template"], + "logos": { + "small": "img/logo.svg", + "large": "img/logo.svg" + }, + "links": [ + {"name": "Website", "url": "https://github.com/grafana/grafana-starter-panel"}, + {"name": "License", "url": "https://github.com/grafana/grafana-starter-panel/blob/master/LICENSE"} + ], + "screenshots": [], + "version": "%VERSION%", + "updated": "%TODAY%" + }, + + "dependencies": { + "grafanaDependency": ">=7.0.0", + "plugins": [] + } +} diff --git a/pkg/plugins/manager/testdata/app-with-child/dist/plugin.json b/pkg/plugins/manager/testdata/app-with-child/dist/plugin.json new file mode 100644 index 00000000000..5f8b4d6e4e4 --- /dev/null +++ b/pkg/plugins/manager/testdata/app-with-child/dist/plugin.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://raw.githubusercontent.com/grafana/grafana/master/docs/sources/developers/plugins/plugin.schema.json", + "type": "app", + "name": "Simple App", + "id": "myorgid-simple-app", + + "info": { + "description": "Grafana App Plugin Template", + "author": { + "name": "Your Name" + }, + "keywords": ["panel", "template"], + "logos": { + "small": "img/logo.svg", + "large": "img/logo.svg" + }, + "links": [ + { "name": "Website", "url": "https://github.com/grafana/grafana-starter-app" }, + { "name": "License", "url": "https://github.com/grafana/grafana-starter-app/blob/master/LICENSE" } + ], + "screenshots": [], + "version": "%VERSION%", + "updated": "%TODAY%" + }, + + "includes": [ + { + "type": "page", + "name": "Root Page (react)", + "path": "/a/myorgid-simple-app", + "role": "Viewer", + "addToNav": true, + "defaultNav": true + }, + { + "type": "page", + "name": "Root Page (Tab B)", + "path": "/a/myorgid-simple-app/?tab=b", + "role": "Viewer", + "addToNav": true + }, + { + "type": "page", + "name": "React Config", + "path": "/plugins/myorgid-simple-app/?page=page2", + "role": "Admin", + "addToNav": true + }, + { + "type": "dashboard", + "name": "Streaming Example", + "path": "dashboards/streaming.json" + }, + { + "type": "dashboard", + "name": "Lots of Stats", + "path": "dashboards/stats.json" + } + ], + + "dependencies": { + "grafanaDependency": ">=7.0.0", + "grafanaVersion": "7.0.0", + "plugins": [] + } +} diff --git a/pkg/plugins/manager/testdata/test-app-with-includes/dashboards/memory.json b/pkg/plugins/manager/testdata/test-app-with-includes/dashboards/memory.json new file mode 100644 index 00000000000..983b994b40e --- /dev/null +++ b/pkg/plugins/manager/testdata/test-app-with-includes/dashboards/memory.json @@ -0,0 +1,5 @@ +{ + "title": "Nginx Memory", + "revision": 2, + "schemaVersion": 11 +} diff --git a/pkg/plugins/manager/testdata/test-app-with-includes/plugin.json b/pkg/plugins/manager/testdata/test-app-with-includes/plugin.json new file mode 100644 index 00000000000..043e7544bf1 --- /dev/null +++ b/pkg/plugins/manager/testdata/test-app-with-includes/plugin.json @@ -0,0 +1,38 @@ +{ + "type": "app", + "name": "Test App", + "id": "test-app", + "info": { + "description": "Official Grafana Test App & Dashboard bundle", + "author": { + "name": "Test Inc.", + "url": "http://test.com" + }, + "keywords": ["test"], + "links": [ + {"name": "Project site", "url": "http://project.com"}, + {"name": "License & Terms", "url": "http://license.com"} + ], + "version": "1.0.0", + "updated": "2015-02-10" + }, + "includes": [ + { + "type": "dashboard", + "name": "Nginx Memory", + "path": "dashboards/memory.json", + "defaultNav": true + }, + { + "type": "page", + "name": "Root Page (react)", + "path": "/a/my-simple-app", + "role": "Viewer", + "addToNav": true, + "defaultNav": true + } + ], + "dependencies": { + "grafanaDependency": ">=8.0.0" + } +} diff --git a/pkg/plugins/manager/update_checker.go b/pkg/plugins/manager/update_checker.go deleted file mode 100644 index 577376c4a94..00000000000 --- a/pkg/plugins/manager/update_checker.go +++ /dev/null @@ -1,84 +0,0 @@ -package manager - -import ( - "context" - "encoding/json" - "io/ioutil" - "net/http" - "strings" - "time" - - "github.com/hashicorp/go-version" -) - -var ( - httpClient = http.Client{Timeout: 10 * time.Second} -) - -type gcomPlugin struct { - Slug string `json:"slug"` - Version string `json:"version"` -} - -func (m *PluginManager) checkForUpdates(ctx context.Context) { - if !m.cfg.CheckForUpdates { - return - } - - m.log.Debug("Checking for updates") - - pluginIDs := m.pluginsEligibleForVersionCheck() - resp, err := httpClient.Get("https://grafana.com/api/plugins/versioncheck?slugIn=" + strings.Join(pluginIDs, ",") + "&grafanaVersion=" + m.cfg.BuildVersion) - if err != nil { - m.log.Debug("Failed to get plugins repo from grafana.com", "error", err.Error()) - return - } - defer func() { - if err := resp.Body.Close(); err != nil { - m.log.Warn("Failed to close response body", "err", err) - } - }() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - m.log.Debug("Update check failed, reading response from grafana.com", "error", err.Error()) - return - } - - var gcomPlugins []gcomPlugin - err = json.Unmarshal(body, &gcomPlugins) - if err != nil { - m.log.Debug("Failed to unmarshal plugin repo, reading response from grafana.com", "error", err.Error()) - return - } - - for _, localP := range m.Plugins(ctx) { - for _, gcomP := range gcomPlugins { - if gcomP.Slug == localP.ID { - localP.GrafanaComVersion = gcomP.Version - - plugVersion, err1 := version.NewVersion(localP.Info.Version) - gplugVersion, err2 := version.NewVersion(gcomP.Version) - - if err1 != nil || err2 != nil { - localP.GrafanaComHasUpdate = localP.Info.Version != localP.GrafanaComVersion - } else { - localP.GrafanaComHasUpdate = plugVersion.LessThan(gplugVersion) - } - } - } - } -} - -func (m *PluginManager) pluginsEligibleForVersionCheck() []string { - var result []string - for _, p := range m.plugins() { - if p.IsCorePlugin() { - continue - } - - result = append(result, p.ID) - } - - return result -} diff --git a/pkg/plugins/plugins.go b/pkg/plugins/plugins.go index 86c44914eb6..b87ab09865f 100644 --- a/pkg/plugins/plugins.go +++ b/pkg/plugins/plugins.go @@ -32,10 +32,6 @@ type Plugin struct { SignedFiles PluginFiles SignatureError *SignatureError - // GCOM update checker fields - GrafanaComVersion string - GrafanaComHasUpdate bool - // SystemJS fields Module string BaseURL string @@ -63,10 +59,6 @@ type PluginDTO struct { SignedFiles PluginFiles SignatureError *SignatureError - // GCOM update checker fields - GrafanaComVersion string - GrafanaComHasUpdate bool - // SystemJS fields Module string BaseURL string @@ -326,22 +318,20 @@ func (p *Plugin) ToDTO() PluginDTO { c, _ := p.Client() return PluginDTO{ - JSONData: p.JSONData, - PluginDir: p.PluginDir, - Class: p.Class, - IncludedInAppID: p.IncludedInAppID, - DefaultNavURL: p.DefaultNavURL, - Pinned: p.Pinned, - Signature: p.Signature, - SignatureType: p.SignatureType, - SignatureOrg: p.SignatureOrg, - SignedFiles: p.SignedFiles, - SignatureError: p.SignatureError, - GrafanaComVersion: p.GrafanaComVersion, - GrafanaComHasUpdate: p.GrafanaComHasUpdate, - Module: p.Module, - BaseURL: p.BaseURL, - StreamHandler: c, + JSONData: p.JSONData, + PluginDir: p.PluginDir, + Class: p.Class, + IncludedInAppID: p.IncludedInAppID, + DefaultNavURL: p.DefaultNavURL, + Pinned: p.Pinned, + Signature: p.Signature, + SignatureType: p.SignatureType, + SignatureOrg: p.SignatureOrg, + SignedFiles: p.SignedFiles, + SignatureError: p.SignatureError, + Module: p.Module, + BaseURL: p.BaseURL, + StreamHandler: c, } } diff --git a/pkg/server/backgroundsvcs/background_services.go b/pkg/server/backgroundsvcs/background_services.go index ce3496a6f33..5f413a34fd0 100644 --- a/pkg/server/backgroundsvcs/background_services.go +++ b/pkg/server/backgroundsvcs/background_services.go @@ -8,7 +8,6 @@ import ( uss "github.com/grafana/grafana/pkg/infra/usagestats/service" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins/manager" - "github.com/grafana/grafana/pkg/plugins/plugindashboards" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/cleanup" @@ -17,42 +16,26 @@ import ( "github.com/grafana/grafana/pkg/services/live/pushhttp" "github.com/grafana/grafana/pkg/services/ngalert" "github.com/grafana/grafana/pkg/services/notifications" + "github.com/grafana/grafana/pkg/services/plugindashboards" "github.com/grafana/grafana/pkg/services/pluginsettings" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/rendering" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/updatechecker" - "github.com/grafana/grafana/pkg/tsdb/azuremonitor" - "github.com/grafana/grafana/pkg/tsdb/cloudmonitoring" - "github.com/grafana/grafana/pkg/tsdb/cloudwatch" - "github.com/grafana/grafana/pkg/tsdb/elasticsearch" - "github.com/grafana/grafana/pkg/tsdb/grafanads" - "github.com/grafana/grafana/pkg/tsdb/graphite" - "github.com/grafana/grafana/pkg/tsdb/influxdb" - "github.com/grafana/grafana/pkg/tsdb/loki" - "github.com/grafana/grafana/pkg/tsdb/mssql" - "github.com/grafana/grafana/pkg/tsdb/mysql" - "github.com/grafana/grafana/pkg/tsdb/opentsdb" - "github.com/grafana/grafana/pkg/tsdb/postgres" - "github.com/grafana/grafana/pkg/tsdb/prometheus" - "github.com/grafana/grafana/pkg/tsdb/tempo" - "github.com/grafana/grafana/pkg/tsdb/testdatasource" ) func ProvideBackgroundServiceRegistry( - httpServer *api.HTTPServer, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, - live *live.GrafanaLive, pushGateway *pushhttp.Gateway, notifications *notifications.NotificationService, - rendering *rendering.RenderingService, tokenService models.UserTokenBackgroundService, - provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, pm *manager.PluginManager, - metrics *metrics.InternalMetricsService, usageStats *uss.UsageStats, updateChecker *updatechecker.Service, - tracing tracing.Tracer, remoteCache *remotecache.RemoteCache, secretsService *secretsManager.SecretsService, + httpServer *api.HTTPServer, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, live *live.GrafanaLive, + pushGateway *pushhttp.Gateway, notifications *notifications.NotificationService, pm *manager.PluginManager, + rendering *rendering.RenderingService, tokenService models.UserTokenBackgroundService, tracing tracing.Tracer, + provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats, + grafanaUpdateChecker *updatechecker.GrafanaService, pluginsUpdateChecker *updatechecker.PluginsService, + metrics *metrics.InternalMetricsService, secretsService *secretsManager.SecretsService, + remoteCache *remotecache.RemoteCache, // Need to make sure these are initialized, is there a better place to put them? - _ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service, - _ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service, - _ *testdatasource.Service, _ *plugindashboards.Service, _ *dashboardsnapshots.Service, - _ *postgres.Service, _ *mysql.Service, _ *mssql.Service, _ *grafanads.Service, _ *cloudmonitoring.Service, - _ *pluginsettings.Service, _ *alerting.AlertNotificationService, _ serviceaccounts.Service, + _ *plugindashboards.Service, _ *dashboardsnapshots.Service, _ *pluginsettings.Service, + _ *alerting.AlertNotificationService, _ serviceaccounts.Service, ) *BackgroundServiceRegistry { return NewBackgroundServiceRegistry( httpServer, @@ -66,7 +49,8 @@ func ProvideBackgroundServiceRegistry( provisioning, alerting, pm, - updateChecker, + grafanaUpdateChecker, + pluginsUpdateChecker, metrics, usageStats, tracing, diff --git a/pkg/server/test_env.go b/pkg/server/test_env.go index 009770ef581..5256cab34cb 100644 --- a/pkg/server/test_env.go +++ b/pkg/server/test_env.go @@ -1,12 +1,16 @@ package server -import "github.com/grafana/grafana/pkg/services/sqlstore" +import ( + "github.com/grafana/grafana/pkg/services/notifications" + "github.com/grafana/grafana/pkg/services/sqlstore" +) -func ProvideTestEnv(server *Server, store *sqlstore.SQLStore) (*TestEnv, error) { - return &TestEnv{server, store}, nil +func ProvideTestEnv(server *Server, store *sqlstore.SQLStore, ns *notifications.NotificationServiceMock) (*TestEnv, error) { + return &TestEnv{server, store, ns}, nil } type TestEnv struct { - Server *Server - SQLStore *sqlstore.SQLStore + Server *Server + SQLStore *sqlstore.SQLStore + NotificationService *notifications.NotificationServiceMock } diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 340bc66ff17..cf171ac8874 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -23,17 +23,21 @@ import ( "github.com/grafana/grafana/pkg/login/social" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/plugins/manager" "github.com/grafana/grafana/pkg/plugins/manager/loader" "github.com/grafana/grafana/pkg/plugins/plugincontext" - "github.com/grafana/grafana/pkg/plugins/plugindashboards" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourceservices" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/auth/jwt" "github.com/grafana/grafana/pkg/services/cleanup" "github.com/grafana/grafana/pkg/services/contexthandler" + "github.com/grafana/grafana/pkg/services/dashboardimport" + dashboardimportservice "github.com/grafana/grafana/pkg/services/dashboardimport/service" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" "github.com/grafana/grafana/pkg/services/datasourceproxy" "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/hooks" "github.com/grafana/grafana/pkg/services/libraryelements" "github.com/grafana/grafana/pkg/services/librarypanels" @@ -46,8 +50,10 @@ import ( ngmetrics "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/services/oauthtoken" + "github.com/grafana/grafana/pkg/services/plugindashboards" "github.com/grafana/grafana/pkg/services/pluginsettings" "github.com/grafana/grafana/pkg/services/query" + "github.com/grafana/grafana/pkg/services/queryhistory" "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/services/schemaloader" @@ -103,7 +109,8 @@ var wireBasicSet = wire.NewSet( hooks.ProvideService, kvstore.ProvideService, localcache.ProvideService, - updatechecker.ProvideService, + updatechecker.ProvideGrafanaService, + updatechecker.ProvidePluginsService, uss.ProvideService, wire.Bind(new(usagestats.Service), new(*uss.UsageStats)), manager.ProvideService, @@ -112,11 +119,11 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(plugins.StaticRouteResolver), new(*manager.PluginManager)), wire.Bind(new(plugins.PluginDashboardManager), new(*manager.PluginManager)), wire.Bind(new(plugins.RendererManager), new(*manager.PluginManager)), + coreplugin.ProvideCoreRegistry, loader.ProvideService, wire.Bind(new(plugins.Loader), new(*loader.Loader)), wire.Bind(new(plugins.ErrorResolver), new(*loader.Loader)), cloudwatch.ProvideService, - cloudwatch.ProvideLogsService, cloudmonitoring.ProvideService, azuremonitor.ProvideService, postgres.ProvideService, @@ -128,6 +135,8 @@ var wireBasicSet = wire.NewSet( cleanup.ProvideService, shorturls.ProvideService, wire.Bind(new(shorturls.Service), new(*shorturls.ShortURLService)), + queryhistory.ProvideService, + wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), quota.ProvideService, remotecache.ProvideService, loginservice.ProvideService, @@ -142,7 +151,6 @@ var wireBasicSet = wire.NewSet( contexthandler.ProvideService, jwt.ProvideService, wire.Bind(new(models.JWTService), new(*jwt.AuthService)), - plugindashboards.ProvideService, schemaloader.ProvideService, ngalert.ProvideService, librarypanels.ProvideService, @@ -150,6 +158,7 @@ var wireBasicSet = wire.NewSet( libraryelements.ProvideService, wire.Bind(new(libraryelements.Service), new(*libraryelements.LibraryElementService)), notifications.ProvideService, + notifications.ProvideSmtpService, tracing.ProvideService, metrics.ProvideService, testdatasource.ProvideService, @@ -180,12 +189,22 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(teamguardian.Store), new(*teamguardianDatabase.TeamGuardianStoreImpl)), teamguardianManager.ProvideService, wire.Bind(new(teamguardian.TeamGuardian), new(*teamguardianManager.Service)), + wire.Bind(new(authinfoservice.Service), new(*authinfoservice.Implementation)), + featuremgmt.ProvideManagerService, + featuremgmt.ProvideToggles, + resourceservices.ProvideResourceServices, + dashboardimportservice.ProvideService, + wire.Bind(new(dashboardimport.Service), new(*dashboardimportservice.ImportDashboardService)), + plugindashboards.ProvideService, ) var wireSet = wire.NewSet( wireBasicSet, sqlstore.ProvideService, ngmetrics.ProvideService, + wire.Bind(new(notifications.Service), new(*notifications.NotificationService)), + wire.Bind(new(notifications.WebhookSender), new(*notifications.NotificationService)), + wire.Bind(new(notifications.EmailSender), new(*notifications.NotificationService)), ) var wireTestSet = wire.NewSet( @@ -193,6 +212,11 @@ var wireTestSet = wire.NewSet( ProvideTestEnv, sqlstore.ProvideServiceForTests, ngmetrics.ProvideServiceForTest, + + notifications.MockNotificationService, + wire.Bind(new(notifications.Service), new(*notifications.NotificationServiceMock)), + wire.Bind(new(notifications.WebhookSender), new(*notifications.NotificationServiceMock)), + wire.Bind(new(notifications.EmailSender), new(*notifications.NotificationServiceMock)), ) func Initialize(cla setting.CommandLineArgs, opts Options, apiOpts api.ServerOptions) (*Server, error) { diff --git a/pkg/server/wireexts_oss.go b/pkg/server/wireexts_oss.go index 9abecb69d7b..620c1b57107 100644 --- a/pkg/server/wireexts_oss.go +++ b/pkg/server/wireexts_oss.go @@ -7,18 +7,21 @@ import ( "github.com/google/wire" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin/provider" "github.com/grafana/grafana/pkg/plugins/manager/signature" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/server/backgroundsvcs" "github.com/grafana/grafana/pkg/services/accesscontrol" acdb "github.com/grafana/grafana/pkg/services/accesscontrol/database" "github.com/grafana/grafana/pkg/services/accesscontrol/ossaccesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/encryption" "github.com/grafana/grafana/pkg/services/encryption/ossencryption" "github.com/grafana/grafana/pkg/services/kmsproviders" "github.com/grafana/grafana/pkg/services/kmsproviders/osskmsproviders" + "github.com/grafana/grafana/pkg/services/ldap" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/services/login" "github.com/grafana/grafana/pkg/services/login/authinfoservice" @@ -59,13 +62,17 @@ var wireExtsBasicSet = wire.NewSet( wire.Bind(new(models.SearchUserFilter), new(*filters.OSSSearchUserFilter)), searchusers.ProvideUsersService, wire.Bind(new(searchusers.Service), new(*searchusers.OSSService)), - signature.ProvideService, + signature.ProvideOSSAuthorizer, wire.Bind(new(plugins.PluginLoaderAuthorizer), new(*signature.UnsignedPluginAuthorizer)), + provider.ProvideService, + wire.Bind(new(plugins.BackendFactoryProvider), new(*provider.Service)), acdb.ProvideService, - wire.Bind(new(accesscontrol.ResourcePermissionsStore), new(*acdb.AccessControlStore)), + wire.Bind(new(resourcepermissions.Store), new(*acdb.AccessControlStore)), wire.Bind(new(accesscontrol.PermissionsProvider), new(*acdb.AccessControlStore)), osskmsproviders.ProvideService, wire.Bind(new(kmsproviders.Service), new(osskmsproviders.Service)), + ldap.ProvideGroupsService, + wire.Bind(new(ldap.Groups), new(*ldap.OSSGroups)), ) var wireExtsSet = wire.NewSet( diff --git a/pkg/services/accesscontrol/accesscontrol.go b/pkg/services/accesscontrol/accesscontrol.go index 538b99e9078..83fbeeaf4d3 100644 --- a/pkg/services/accesscontrol/accesscontrol.go +++ b/pkg/services/accesscontrol/accesscontrol.go @@ -20,9 +20,13 @@ type AccessControl interface { //IsDisabled returns if access control is enabled or not IsDisabled() bool - // DeclareFixedRoles allow the caller to declare, to the service, fixed roles and their + // DeclareFixedRoles allows the caller to declare, to the service, fixed roles and their // assignments to organization roles ("Viewer", "Editor", "Admin") or "Grafana Admin" DeclareFixedRoles(...RoleRegistration) error + + // RegisterAttributeScopeResolver allows the caller to register a scope resolver for a + // specific scope prefix (ex: datasources:name:) + RegisterAttributeScopeResolver(scopePrefix string, resolver AttributeScopeResolveFunc) } type PermissionsProvider interface { @@ -33,26 +37,11 @@ type ResourcePermissionsService interface { // GetPermissions returns all permissions for given resourceID GetPermissions(ctx context.Context, orgID int64, resourceID string) ([]ResourcePermission, error) // SetUserPermission sets permission on resource for a user - SetUserPermission(ctx context.Context, orgID, userID int64, resourceID string, actions []string) (*ResourcePermission, error) + SetUserPermission(ctx context.Context, orgID, userID int64, resourceID, permission string) (*ResourcePermission, error) // SetTeamPermission sets permission on resource for a team - SetTeamPermission(ctx context.Context, orgID, teamID int64, resourceID string, actions []string) (*ResourcePermission, error) + SetTeamPermission(ctx context.Context, orgID, teamID int64, resourceID, permission string) (*ResourcePermission, error) // SetBuiltInRolePermission sets permission on resource for a built-in role (Admin, Editor, Viewer) - SetBuiltInRolePermission(ctx context.Context, orgID int64, builtInRole string, resourceID string, actions []string) (*ResourcePermission, error) - // MapActions will map actions for a ResourcePermissions to it's "friendly" name configured in PermissionsToActions map. - MapActions(permission ResourcePermission) string - // MapPermission will map a friendly named permission to it's corresponding actions configured in PermissionsToAction map. - MapPermission(permission string) []string -} - -type ResourcePermissionsStore interface { - // SetUserResourcePermission sets permission for managed user role on a resource - SetUserResourcePermission(ctx context.Context, orgID, userID int64, cmd SetResourcePermissionCommand) (*ResourcePermission, error) - // SetTeamResourcePermission sets permission for managed team role on a resource - SetTeamResourcePermission(ctx context.Context, orgID, teamID int64, cmd SetResourcePermissionCommand) (*ResourcePermission, error) - // SetBuiltinResourcePermission sets permissions for managed builtin role on a resource - SetBuiltInResourcePermission(ctx context.Context, orgID int64, builtinRole string, cmd SetResourcePermissionCommand) (*ResourcePermission, error) - // GetResourcesPermissions will return all permission for all supplied resource ids - GetResourcesPermissions(ctx context.Context, orgID int64, query GetResourcesPermissionsQuery) ([]ResourcePermission, error) + SetBuiltInRolePermission(ctx context.Context, orgID int64, builtInRole string, resourceID string, permission string) (*ResourcePermission, error) } // Metadata contains user accesses for a given resource diff --git a/pkg/services/accesscontrol/database/database.go b/pkg/services/accesscontrol/database/database.go index 03ef65993ba..85f025cfbd4 100644 --- a/pkg/services/accesscontrol/database/database.go +++ b/pkg/services/accesscontrol/database/database.go @@ -37,6 +37,17 @@ func (s *AccessControlStore) GetUserPermissions(ctx context.Context, query acces INNER JOIN role ON role.id = permission.role_id ` + filter + if query.Actions != nil { + q += " AND permission.action IN(" + if len(query.Actions) > 0 { + q += "?" + strings.Repeat(",?", len(query.Actions)-1) + } + q += ")" + for _, a := range query.Actions { + params = append(params, a) + } + } + if err := sess.SQL(q, params...).Find(&result); err != nil { return err } diff --git a/pkg/services/accesscontrol/database/database_test.go b/pkg/services/accesscontrol/database/database_test.go index 49d695e8d13..d4fa790178f 100644 --- a/pkg/services/accesscontrol/database/database_test.go +++ b/pkg/services/accesscontrol/database/database_test.go @@ -20,6 +20,7 @@ type getUserPermissionsTestCase struct { userPermissions []string teamPermissions []string builtinPermissions []string + actions []string expected int } @@ -52,6 +53,26 @@ func TestAccessControlStore_GetUserPermissions(t *testing.T) { builtinPermissions: []string{"5", "6"}, expected: 5, }, + { + desc: "Should filter on actions", + orgID: 1, + role: "", + userPermissions: []string{"1", "2", "10"}, + teamPermissions: []string{"100", "2"}, + builtinPermissions: []string{"5", "6"}, + expected: 3, + actions: []string{"dashboards:write"}, + }, + { + desc: "Should return no permission when passing empty slice of actions", + orgID: 1, + role: "Viewer", + userPermissions: []string{"1", "2", "10"}, + teamPermissions: []string{"100", "2"}, + builtinPermissions: []string{"5", "6"}, + expected: 0, + actions: []string{}, + }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { @@ -61,10 +82,10 @@ func TestAccessControlStore_GetUserPermissions(t *testing.T) { for _, id := range tt.userPermissions { _, err := store.SetUserResourcePermission(context.Background(), tt.orgID, user.Id, accesscontrol.SetResourcePermissionCommand{ - Actions: []string{"dashboards:read"}, + Actions: []string{"dashboards:write"}, Resource: "dashboards", ResourceID: id, - }) + }, nil) require.NoError(t, err) } @@ -73,7 +94,7 @@ func TestAccessControlStore_GetUserPermissions(t *testing.T) { Actions: []string{"dashboards:read"}, Resource: "dashboards", ResourceID: id, - }) + }, nil) require.NoError(t, err) } @@ -82,7 +103,7 @@ func TestAccessControlStore_GetUserPermissions(t *testing.T) { Actions: []string{"dashboards:read"}, Resource: "dashboards", ResourceID: id, - }) + }, nil) require.NoError(t, err) } @@ -97,9 +118,10 @@ func TestAccessControlStore_GetUserPermissions(t *testing.T) { } permissions, err := store.GetUserPermissions(context.Background(), accesscontrol.GetUserPermissionsQuery{ - OrgID: tt.orgID, - UserID: user.Id, - Roles: roles, + OrgID: tt.orgID, + UserID: user.Id, + Roles: roles, + Actions: tt.actions, }) require.NoError(t, err) diff --git a/pkg/services/accesscontrol/database/resource_permissions.go b/pkg/services/accesscontrol/database/resource_permissions.go index f8c138ae46e..436d6179df5 100644 --- a/pkg/services/accesscontrol/database/resource_permissions.go +++ b/pkg/services/accesscontrol/database/resource_permissions.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions/types" "github.com/grafana/grafana/pkg/services/sqlstore" ) @@ -32,7 +33,11 @@ func (p *flatResourcePermission) Managed() bool { return strings.HasPrefix(p.RoleName, "managed:") } -func (s *AccessControlStore) SetUserResourcePermission(ctx context.Context, orgID, userID int64, cmd accesscontrol.SetResourcePermissionCommand) (*accesscontrol.ResourcePermission, error) { +func (s *AccessControlStore) SetUserResourcePermission( + ctx context.Context, orgID, userID int64, + cmd accesscontrol.SetResourcePermissionCommand, + hook types.UserResourceHookFunc, +) (*accesscontrol.ResourcePermission, error) { if userID == 0 { return nil, models.ErrUserNotFound } @@ -41,10 +46,11 @@ func (s *AccessControlStore) SetUserResourcePermission(ctx context.Context, orgI var permission *accesscontrol.ResourcePermission err = s.sql.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error { permission, err = s.setResourcePermission(sess, orgID, managedUserRoleName(userID), s.userAdder(sess, orgID, userID), cmd) - if err != nil { - return err + if err == nil && hook != nil { + return hook(sess, orgID, userID, cmd.ResourceID, cmd.Permission) } - return nil + + return err }) if err != nil { @@ -54,7 +60,11 @@ func (s *AccessControlStore) SetUserResourcePermission(ctx context.Context, orgI return permission, nil } -func (s *AccessControlStore) SetTeamResourcePermission(ctx context.Context, orgID, teamID int64, cmd accesscontrol.SetResourcePermissionCommand) (*accesscontrol.ResourcePermission, error) { +func (s *AccessControlStore) SetTeamResourcePermission( + ctx context.Context, orgID, teamID int64, + cmd accesscontrol.SetResourcePermissionCommand, + hook types.TeamResourceHookFunc, +) (*accesscontrol.ResourcePermission, error) { if teamID == 0 { return nil, models.ErrTeamNotFound } @@ -64,10 +74,11 @@ func (s *AccessControlStore) SetTeamResourcePermission(ctx context.Context, orgI err = s.sql.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error { permission, err = s.setResourcePermission(sess, orgID, managedTeamRoleName(teamID), s.teamAdder(sess, orgID, teamID), cmd) - if err != nil { - return err + if err == nil && hook != nil { + return hook(sess, orgID, teamID, cmd.ResourceID, cmd.Permission) } - return nil + + return err }) if err != nil { @@ -77,7 +88,11 @@ func (s *AccessControlStore) SetTeamResourcePermission(ctx context.Context, orgI return permission, nil } -func (s *AccessControlStore) SetBuiltInResourcePermission(ctx context.Context, orgID int64, builtInRole string, cmd accesscontrol.SetResourcePermissionCommand) (*accesscontrol.ResourcePermission, error) { +func (s *AccessControlStore) SetBuiltInResourcePermission( + ctx context.Context, orgID int64, builtInRole string, + cmd accesscontrol.SetResourcePermissionCommand, + hook types.BuiltinResourceHookFunc, +) (*accesscontrol.ResourcePermission, error) { if !models.RoleType(builtInRole).IsValid() || builtInRole == accesscontrol.RoleGrafanaAdmin { return nil, fmt.Errorf("invalid role: %s", builtInRole) } @@ -87,6 +102,9 @@ func (s *AccessControlStore) SetBuiltInResourcePermission(ctx context.Context, o err = s.sql.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error { permission, err = s.setResourcePermission(sess, orgID, managedBuiltInRoleName(builtInRole), s.builtInRoleAdder(sess, orgID, builtInRole), cmd) + if err == nil && hook != nil { + return hook(sess, orgID, builtInRole, cmd.ResourceID, cmd.Permission) + } return err }) diff --git a/pkg/services/accesscontrol/database/resource_permissions_bench_test.go b/pkg/services/accesscontrol/database/resource_permissions_bench_test.go index f1cebca80f0..77a6a9fa398 100644 --- a/pkg/services/accesscontrol/database/resource_permissions_bench_test.go +++ b/pkg/services/accesscontrol/database/resource_permissions_bench_test.go @@ -51,7 +51,7 @@ func benchmarkDSPermissions(b *testing.B, dsNum, usersNum int) { } } -func getDSPermissions(b *testing.B, store accesscontrol.ResourcePermissionsStore, dataSources []int64) { +func getDSPermissions(b *testing.B, store *AccessControlStore, dataSources []int64) { dsId := dataSources[0] permissions, err := store.GetResourcesPermissions(context.Background(), accesscontrol.GlobalOrgID, accesscontrol.GetResourcesPermissionsQuery{ @@ -99,6 +99,7 @@ func GenerateDatasourcePermissions(b *testing.B, db *sqlstore.SQLStore, ac *Acce Resource: dsResource, ResourceID: strconv.Itoa(int(dsID)), }, + nil, ) require.NoError(b, err) } @@ -115,6 +116,7 @@ func GenerateDatasourcePermissions(b *testing.B, db *sqlstore.SQLStore, ac *Acce Resource: "datasources", ResourceID: strconv.Itoa(int(dsID)), }, + nil, ) require.NoError(b, err) } diff --git a/pkg/services/accesscontrol/database/resource_permissions_test.go b/pkg/services/accesscontrol/database/resource_permissions_test.go index 1499ad3d366..e029da2b1f5 100644 --- a/pkg/services/accesscontrol/database/resource_permissions_test.go +++ b/pkg/services/accesscontrol/database/resource_permissions_test.go @@ -70,7 +70,7 @@ func TestAccessControlStore_SetUserResourcePermission(t *testing.T) { store, _ := setupTestEnv(t) for _, s := range test.seeds { - _, err := store.SetUserResourcePermission(context.Background(), test.orgID, test.userID, s) + _, err := store.SetUserResourcePermission(context.Background(), test.orgID, test.userID, s, nil) require.NoError(t, err) } @@ -78,7 +78,7 @@ func TestAccessControlStore_SetUserResourcePermission(t *testing.T) { Actions: test.actions, Resource: test.resource, ResourceID: test.resourceID, - }) + }, nil) require.NoError(t, err) if len(test.actions) == 0 { @@ -148,7 +148,7 @@ func TestAccessControlStore_SetTeamResourcePermission(t *testing.T) { store, _ := setupTestEnv(t) for _, s := range test.seeds { - _, err := store.SetTeamResourcePermission(context.Background(), test.orgID, test.teamID, s) + _, err := store.SetTeamResourcePermission(context.Background(), test.orgID, test.teamID, s, nil) require.NoError(t, err) } @@ -156,7 +156,7 @@ func TestAccessControlStore_SetTeamResourcePermission(t *testing.T) { Actions: test.actions, Resource: test.resource, ResourceID: test.resourceID, - }) + }, nil) require.NoError(t, err) if len(test.actions) == 0 { @@ -226,7 +226,7 @@ func TestAccessControlStore_SetBuiltInResourcePermission(t *testing.T) { store, _ := setupTestEnv(t) for _, s := range test.seeds { - _, err := store.SetBuiltInResourcePermission(context.Background(), test.orgID, test.builtInRole, s) + _, err := store.SetBuiltInResourcePermission(context.Background(), test.orgID, test.builtInRole, s, nil) require.NoError(t, err) } @@ -234,7 +234,7 @@ func TestAccessControlStore_SetBuiltInResourcePermission(t *testing.T) { Actions: test.actions, Resource: test.resource, ResourceID: test.resourceID, - }) + }, nil) require.NoError(t, err) if len(test.actions) == 0 { @@ -356,7 +356,7 @@ func seedResourcePermissions(t *testing.T, store *AccessControlStore, sql *sqlst Actions: actions, Resource: resource, ResourceID: resourceID, - }) + }, nil) require.NoError(t, err) } } diff --git a/pkg/services/accesscontrol/errors.go b/pkg/services/accesscontrol/errors.go index 593c67de1a2..78ef9498ab0 100644 --- a/pkg/services/accesscontrol/errors.go +++ b/pkg/services/accesscontrol/errors.go @@ -5,4 +5,5 @@ import "errors" var ( ErrFixedRolePrefixMissing = errors.New("fixed role should be prefixed with '" + FixedRolePrefix + "'") ErrInvalidBuiltinRole = errors.New("built-in role is not valid") + ErrInvalidScope = errors.New("invalid scope") ) diff --git a/pkg/services/accesscontrol/evaluator.go b/pkg/services/accesscontrol/evaluator.go index 99dc601a5e7..52be2d80549 100644 --- a/pkg/services/accesscontrol/evaluator.go +++ b/pkg/services/accesscontrol/evaluator.go @@ -1,9 +1,8 @@ package accesscontrol import ( - "bytes" + "context" "fmt" - "html/template" "strings" "github.com/grafana/grafana/pkg/infra/log" @@ -14,10 +13,11 @@ var logger = log.New("accesscontrol.evaluator") type Evaluator interface { // Evaluate permissions that are grouped by action Evaluate(permissions map[string][]string) (bool, error) - // Inject params into the evaluator's templated scopes. e.g. "settings:" + eval.Parameters(":id") and returns a new Evaluator - Inject(params ScopeParams) (Evaluator, error) + // MutateScopes executes a sequence of ScopeModifier functions on all embedded scopes of an evaluator and returns a new Evaluator + MutateScopes(context.Context, ...ScopeMutator) (Evaluator, error) // String returns a string representation of permission required by the evaluator - String() string + fmt.Stringer + fmt.GoStringer } var _ Evaluator = new(permissionEvaluator) @@ -89,23 +89,31 @@ func match(scope, target string) (bool, error) { return scope == target, nil } -func (p permissionEvaluator) Inject(params ScopeParams) (Evaluator, error) { +func (p permissionEvaluator) MutateScopes(ctx context.Context, modifiers ...ScopeMutator) (Evaluator, error) { + var err error + if p.Scopes == nil { + return EvalPermission(p.Action), nil + } + scopes := make([]string, 0, len(p.Scopes)) for _, scope := range p.Scopes { - tmpl, err := template.New("scope").Parse(scope) - if err != nil { - return nil, err + modified := scope + for _, modifier := range modifiers { + modified, err = modifier(ctx, modified) + if err != nil { + return nil, err + } } - var buf bytes.Buffer - if err = tmpl.Execute(&buf, params); err != nil { - return nil, err - } - scopes = append(scopes, buf.String()) + scopes = append(scopes, modified) } return EvalPermission(p.Action, scopes...), nil } func (p permissionEvaluator) String() string { + return p.Action +} + +func (p permissionEvaluator) GoString() string { return fmt.Sprintf("action:%s scopes:%s", p.Action, strings.Join(p.Scopes, ", ")) } @@ -129,16 +137,16 @@ func (a allEvaluator) Evaluate(permissions map[string][]string) (bool, error) { return true, nil } -func (a allEvaluator) Inject(params ScopeParams) (Evaluator, error) { - var injected []Evaluator +func (a allEvaluator) MutateScopes(ctx context.Context, modifiers ...ScopeMutator) (Evaluator, error) { + var modified []Evaluator for _, e := range a.allOf { - i, err := e.Inject(params) + i, err := e.MutateScopes(ctx, modifiers...) if err != nil { return nil, err } - injected = append(injected, i) + modified = append(modified, i) } - return EvalAll(injected...), nil + return EvalAll(modified...), nil } func (a allEvaluator) String() string { @@ -146,6 +154,16 @@ func (a allEvaluator) String() string { for _, e := range a.allOf { permissions = append(permissions, e.String()) } + + return fmt.Sprintf("all of %s", strings.Join(permissions, ", ")) +} + +func (a allEvaluator) GoString() string { + permissions := make([]string, 0, len(a.allOf)) + for _, e := range a.allOf { + permissions = append(permissions, e.GoString()) + } + return fmt.Sprintf("all(%s)", strings.Join(permissions, " ")) } @@ -173,16 +191,16 @@ func (a anyEvaluator) Evaluate(permissions map[string][]string) (bool, error) { return false, nil } -func (a anyEvaluator) Inject(params ScopeParams) (Evaluator, error) { - var injected []Evaluator +func (a anyEvaluator) MutateScopes(ctx context.Context, modifiers ...ScopeMutator) (Evaluator, error) { + var modified []Evaluator for _, e := range a.anyOf { - i, err := e.Inject(params) + i, err := e.MutateScopes(ctx, modifiers...) if err != nil { return nil, err } - injected = append(injected, i) + modified = append(modified, i) } - return EvalAny(injected...), nil + return EvalAny(modified...), nil } func (a anyEvaluator) String() string { @@ -190,5 +208,15 @@ func (a anyEvaluator) String() string { for _, e := range a.anyOf { permissions = append(permissions, e.String()) } + + return fmt.Sprintf("any of %s", strings.Join(permissions, ", ")) +} + +func (a anyEvaluator) GoString() string { + permissions := make([]string, 0, len(a.anyOf)) + for _, e := range a.anyOf { + permissions = append(permissions, e.String()) + } + return fmt.Sprintf("any(%s)", strings.Join(permissions, " ")) } diff --git a/pkg/services/accesscontrol/evaluator_test.go b/pkg/services/accesscontrol/evaluator_test.go index 729d6095369..230a4c9b823 100644 --- a/pkg/services/accesscontrol/evaluator_test.go +++ b/pkg/services/accesscontrol/evaluator_test.go @@ -1,6 +1,7 @@ package accesscontrol import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -120,7 +121,7 @@ func TestPermission_Inject(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - injected, err := test.evaluator.Inject(test.params) + injected, err := test.evaluator.MutateScopes(context.TODO(), ScopeInjector(test.params)) assert.NoError(t, err) ok, err := injected.Evaluate(test.permissions) assert.NoError(t, err) @@ -233,7 +234,7 @@ func TestAll_Inject(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - injected, err := test.evaluator.Inject(test.params) + injected, err := test.evaluator.MutateScopes(context.TODO(), ScopeInjector(test.params)) assert.NoError(t, err) ok, err := injected.Evaluate(test.permissions) assert.NoError(t, err) @@ -344,7 +345,7 @@ func TestAny_Inject(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - injected, err := test.evaluator.Inject(test.params) + injected, err := test.evaluator.MutateScopes(context.TODO(), ScopeInjector(test.params)) assert.NoError(t, err) ok, err := injected.Evaluate(test.permissions) assert.NoError(t, err) diff --git a/pkg/services/accesscontrol/filter.go b/pkg/services/accesscontrol/filter.go index 298206a4bec..ae7fed5625c 100644 --- a/pkg/services/accesscontrol/filter.go +++ b/pkg/services/accesscontrol/filter.go @@ -3,97 +3,72 @@ package accesscontrol import ( "context" "errors" - "fmt" + "strconv" "strings" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" ) -var sqlIDAcceptList = map[string]struct{}{} - -type SQLDialect interface { - DriverName() string +var sqlIDAcceptList = map[string]struct{}{ + "org_user.user_id": {}, } +const denyQuery = " 1 = 0" +const allowAllQuery = " 1 = 1" + // Filter creates a where clause to restrict the view of a query based on a users permissions // Scopes for a certain action will be compared against prefix:id:sqlID where prefix is the scope prefix and sqlID // is the id to generate scope from e.g. user.id -func Filter(ctx context.Context, dialect SQLDialect, sqlID, prefix, action string, user *models.SignedInUser) (string, []interface{}, error) { +func Filter(ctx context.Context, sqlID, prefix, action string, user *models.SignedInUser) (string, []interface{}, error) { if _, ok := sqlIDAcceptList[sqlID]; !ok { - return "", nil, errors.New("sqlID is not in the accept list") + return denyQuery, nil, errors.New("sqlID is not in the accept list") } - if user.Permissions == nil || user.Permissions[user.OrgId] == nil { - return "", nil, errors.New("missing permissions") + return denyQuery, nil, errors.New("missing permissions") } - scopes := user.Permissions[user.OrgId][action] - if len(scopes) == 0 { - return " 1 = 0", nil, nil + var hasWildcard bool + var ids []interface{} + for _, scope := range user.Permissions[user.OrgId][action] { + if strings.HasPrefix(scope, prefix) || scope == "*" { + if id := strings.TrimPrefix(scope, prefix); id == "*" || id == ":*" || id == ":id:*" { + hasWildcard = true + break + } + if id, err := parseScopeID(scope); err == nil { + ids = append(ids, id) + } + } } - var sql string - var args []interface{} - - switch { - case strings.Contains(dialect.DriverName(), migrator.SQLite): - sql, args = sqliteQuery(scopes, sqlID, prefix) - case strings.Contains(dialect.DriverName(), migrator.MySQL): - sql, args = mysqlQuery(scopes, sqlID, prefix) - case strings.Contains(dialect.DriverName(), migrator.Postgres): - sql, args = postgresQuery(scopes, sqlID, prefix) - default: - return "", nil, fmt.Errorf("unknown database: %s", dialect.DriverName()) + if hasWildcard { + return allowAllQuery, nil, nil } - return sql, args, nil + if len(ids) == 0 { + return denyQuery, nil, nil + } + + query := strings.Builder{} + query.WriteRune(' ') + query.WriteString(sqlID) + query.WriteString(" IN ") + query.WriteString("(?") + query.WriteString(strings.Repeat(",?", len(ids)-1)) + query.WriteRune(')') + + return query.String(), ids, nil } -func sqliteQuery(scopes []string, sqlID, prefix string) (string, []interface{}) { - args := []interface{}{prefix} - for _, s := range scopes { - args = append(args, s) - } - args = append(args, prefix, prefix, prefix) - - return fmt.Sprintf(` - ? || ':id:' || %s IN ( - WITH t(scope) AS ( - VALUES (?)`+strings.Repeat(`, (?)`, len(scopes)-1)+` - ) - SELECT IIF(t.scope = '*' OR t.scope = ? || ':*' OR t.scope = ? || ':id:*', ? || ':id:' || %s, t.scope) FROM t - ) - `, sqlID, sqlID), args +func parseScopeID(scope string) (int64, error) { + return strconv.ParseInt(scope[strings.LastIndex(scope, ":")+1:], 10, 64) } -func mysqlQuery(scopes []string, sqlID, prefix string) (string, []interface{}) { - args := []interface{}{prefix, prefix, prefix, prefix} - for _, s := range scopes { - args = append(args, s) +// SetAcceptListForTest allow us to mutate the list for blackbox testing +func SetAcceptListForTest(list map[string]struct{}) func() { + original := sqlIDAcceptList + sqlIDAcceptList = list + return func() { + sqlIDAcceptList = original } - - return fmt.Sprintf(` - CONCAT(?, ':id:', %s) IN ( - SELECT IF(t.scope = '*' OR t.scope = CONCAT(?, ':*') OR t.scope = CONCAT(?, ':id:*'), CONCAT(?, ':id:', %s), t.scope) FROM - (SELECT ? AS scope`+strings.Repeat(" UNION ALL SELECT ?", len(scopes)-1)+`) AS t - ) - `, sqlID, sqlID), args -} - -func postgresQuery(scopes []string, sqlID, prefix string) (string, []interface{}) { - args := []interface{}{prefix, prefix, prefix, prefix} - for _, s := range scopes { - args = append(args, s) - } - - return fmt.Sprintf(` - CONCAT(?, ':id:', %s) IN ( - SELECT - CASE WHEN p.scope = '*' OR p.scope = CONCAT(?, ':*') OR p.scope = CONCAT(?, ':id:*') THEN CONCAT(?, ':id:', %s) - ELSE p.scope - END - FROM (VALUES (?)`+strings.Repeat(", (?)", len(scopes)-1)+`) as p(scope) - ) - `, sqlID, sqlID), args } diff --git a/pkg/services/accesscontrol/filter_bench_test.go b/pkg/services/accesscontrol/filter_bench_test.go index 26602e6043e..7ef51af5251 100644 --- a/pkg/services/accesscontrol/filter_bench_test.go +++ b/pkg/services/accesscontrol/filter_bench_test.go @@ -1,4 +1,4 @@ -package accesscontrol +package accesscontrol_test import ( "context" @@ -9,31 +9,34 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/sqlstore" ) -func BenchmarkFilter10_10(b *testing.B) { benchmarkFilter(b, 10, 10) } -func BenchmarkFilter100_10(b *testing.B) { benchmarkFilter(b, 100, 10) } -func BenchmarkFilter100_100(b *testing.B) { benchmarkFilter(b, 100, 100) } +func BenchmarkFilter10_10(b *testing.B) { benchmarkFilter(b, 10, 10) } +func BenchmarkFilter100_10(b *testing.B) { benchmarkFilter(b, 100, 10) } +func BenchmarkFilter100_100(b *testing.B) { benchmarkFilter(b, 100, 100) } +func BenchmarkFilter1000_100(b *testing.B) { benchmarkFilter(b, 1000, 100) } +func BenchmarkFilter1000_1000(b *testing.B) { benchmarkFilter(b, 1000, 100) } func benchmarkFilter(b *testing.B, numDs, numPermissions int) { store, permissions := setupFilterBenchmark(b, numDs, numPermissions) b.ResetTimer() // set sqlIDAcceptList before running tests - sqlIDAcceptList = map[string]struct{}{ + restore := accesscontrol.SetAcceptListForTest(map[string]struct{}{ "data_source.id": {}, - } + }) + defer restore() for i := 0; i < b.N; i++ { baseSql := `SELECT data_source.* FROM data_source WHERE` - query, args, err := Filter( + query, args, err := accesscontrol.Filter( context.Background(), - &FakeDriver{name: "sqlite3"}, "data_source.id", "datasources", "datasources:read", - &models.SignedInUser{OrgId: 1, Permissions: map[int64]map[string][]string{1: GroupScopesByAction(permissions)}}, + &models.SignedInUser{OrgId: 1, Permissions: map[int64]map[string][]string{1: accesscontrol.GroupScopesByAction(permissions)}}, ) require.NoError(b, err) @@ -46,7 +49,7 @@ func benchmarkFilter(b *testing.B, numDs, numPermissions int) { } } -func setupFilterBenchmark(b *testing.B, numDs, numPermissions int) (*sqlstore.SQLStore, []*Permission) { +func setupFilterBenchmark(b *testing.B, numDs, numPermissions int) (*sqlstore.SQLStore, []*accesscontrol.Permission) { b.Helper() store := sqlstore.InitTestDB(b) @@ -62,11 +65,11 @@ func setupFilterBenchmark(b *testing.B, numDs, numPermissions int) (*sqlstore.SQ numPermissions = numDs } - permissions := make([]*Permission, 0, numPermissions) + permissions := make([]*accesscontrol.Permission, 0, numPermissions) for i := 1; i <= numPermissions; i++ { - permissions = append(permissions, &Permission{ + permissions = append(permissions, &accesscontrol.Permission{ Action: "datasources:read", - Scope: Scope("datasources", "id", strconv.Itoa(i)), + Scope: accesscontrol.Scope("datasources", "id", strconv.Itoa(i)), }) } diff --git a/pkg/services/accesscontrol/filter_test.go b/pkg/services/accesscontrol/filter_test.go index d0125c56f61..aa71df99b0a 100644 --- a/pkg/services/accesscontrol/filter_test.go +++ b/pkg/services/accesscontrol/filter_test.go @@ -1,4 +1,4 @@ -package accesscontrol +package accesscontrol_test import ( "context" @@ -9,151 +9,14 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/sqlstore" ) -type filterTest struct { - desc string - driverName string - sqlID string - action string - prefix string - permissions []*Permission - expectedQuery string - expectedArgs []interface{} -} - -func TestFilter(t *testing.T) { - tests := []filterTest{ - { - desc: "should produce datasource filter with sqlite driver", - driverName: "sqlite3", - sqlID: "data_source.id", - prefix: "datasources", - action: "datasources:query", - permissions: []*Permission{ - {Action: "datasources:query", Scope: "datasources:id:1"}, - {Action: "datasources:query", Scope: "datasources:id:2"}, - {Action: "datasources:query", Scope: "datasources:id:3"}, - {Action: "datasources:query", Scope: "datasources:id:8"}, - // Other permissions - {Action: "datasources:write", Scope: "datasources:id:100"}, - {Action: "datasources:delete", Scope: "datasources:id:101"}, - }, - expectedQuery: ` - ? || ':id:' || data_source.id IN ( - WITH t(scope) AS ( - VALUES (?), (?), (?), (?) - ) - SELECT IIF(t.scope = '*' OR t.scope = ? || ':*' OR t.scope = ? || ':id:*', ? || ':id:' || data_source.id, t.scope) FROM t - ) - `, - expectedArgs: []interface{}{ - "datasources", - "datasources:id:1", - "datasources:id:2", - "datasources:id:3", - "datasources:id:8", - "datasources", - "datasources", - "datasources", - }, - }, - { - desc: "should produce dashboard filter with mysql driver", - driverName: "mysql", - sqlID: "dashboard.id", - prefix: "dashboards", - action: "dashboards:read", - permissions: []*Permission{ - {Action: "dashboards:read", Scope: "dashboards:id:1"}, - {Action: "dashboards:read", Scope: "dashboards:id:2"}, - {Action: "dashboards:read", Scope: "dashboards:id:5"}, - // Other permissions - {Action: "dashboards:write", Scope: "dashboards:id:100"}, - {Action: "dashboards:delete", Scope: "dashboards:id:101"}, - }, - expectedQuery: ` - CONCAT(?, ':id:', dashboard.id) IN ( - SELECT IF(t.scope = '*' OR t.scope = CONCAT(?, ':*') OR t.scope = CONCAT(?, ':id:*'), CONCAT(?, ':id:', dashboard.id), t.scope) FROM - (SELECT ? AS scope UNION ALL SELECT ? UNION ALL SELECT ?) AS t - ) - `, - expectedArgs: []interface{}{ - "dashboards", - "dashboards", - "dashboards", - "dashboards", - "dashboards:id:1", - "dashboards:id:2", - "dashboards:id:5", - }, - }, - { - desc: "should produce user filter with postgres driver", - driverName: "postgres", - sqlID: "user.id", - prefix: "users", - action: "users:read", - permissions: []*Permission{ - {Action: "users:read", Scope: "users:id:1"}, - {Action: "users:read", Scope: "users:id:100"}, - // Other permissions - {Action: "dashboards:write", Scope: "dashboards:id:100"}, - {Action: "dashboards:delete", Scope: "dashboards:id:101"}, - }, - expectedQuery: ` - CONCAT(?, ':id:', user.id) IN ( - SELECT - CASE WHEN p.scope = '*' OR p.scope = CONCAT(?, ':*') OR p.scope = CONCAT(?, ':id:*') THEN CONCAT(?, ':id:', user.id) - ELSE p.scope - END - FROM (VALUES (?), (?)) as p(scope) - ) - `, - expectedArgs: []interface{}{ - "users", - "users", - "users", - "users", - "users:id:1", - "users:id:100", - }, - }, - } - - // set sqlIDAcceptList before running tests - sqlIDAcceptList = map[string]struct{}{ - "user.id": {}, - "dashboard.id": {}, - "data_source.id": {}, - } - - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - query, args, err := Filter( - context.Background(), - FakeDriver{name: tt.driverName}, - tt.sqlID, - tt.prefix, - tt.action, - &models.SignedInUser{OrgId: 1, Permissions: map[int64]map[string][]string{1: GroupScopesByAction(tt.permissions)}}, - ) - require.NoError(t, err) - assert.Equal(t, tt.expectedQuery, query) - - require.Len(t, args, len(tt.expectedArgs)) - for i := range tt.expectedArgs { - assert.Equal(t, tt.expectedArgs[i], args[i]) - } - }) - } -} - type filterDatasourcesTestCase struct { desc string sqlID string - permissions []*Permission + permissions []*accesscontrol.Permission expectedDataSources []string expectErr bool } @@ -163,31 +26,55 @@ func TestFilter_Datasources(t *testing.T) { { desc: "expect all data sources to be returned", sqlID: "data_source.id", - permissions: []*Permission{ + permissions: []*accesscontrol.Permission{ {Action: "datasources:read", Scope: "datasources:*"}, }, expectedDataSources: []string{"ds:1", "ds:2", "ds:3", "ds:4", "ds:5", "ds:6", "ds:7", "ds:8", "ds:9", "ds:10"}, }, + { + desc: "expect all data sources for wildcard id scope to be returned", + sqlID: "data_source.id", + permissions: []*accesscontrol.Permission{ + {Action: "datasources:read", Scope: "datasources:id:*"}, + }, + expectedDataSources: []string{"ds:1", "ds:2", "ds:3", "ds:4", "ds:5", "ds:6", "ds:7", "ds:8", "ds:9", "ds:10"}, + }, + { + desc: "expect all data sources for wildcard scope to be returned", + sqlID: "data_source.id", + permissions: []*accesscontrol.Permission{ + {Action: "datasources:read", Scope: "*"}, + }, + expectedDataSources: []string{"ds:1", "ds:2", "ds:3", "ds:4", "ds:5", "ds:6", "ds:7", "ds:8", "ds:9", "ds:10"}, + }, { desc: "expect no data sources to be returned", sqlID: "data_source.id", - permissions: []*Permission{}, + permissions: []*accesscontrol.Permission{}, expectedDataSources: []string{}, }, { desc: "expect data sources with id 3, 7 and 8 to be returned", sqlID: "data_source.id", - permissions: []*Permission{ + permissions: []*accesscontrol.Permission{ {Action: "datasources:read", Scope: "datasources:id:3"}, {Action: "datasources:read", Scope: "datasources:id:7"}, {Action: "datasources:read", Scope: "datasources:id:8"}, }, expectedDataSources: []string{"ds:3", "ds:7", "ds:8"}, }, + { + desc: "expect no data sources to be returned for malformed scope", + sqlID: "data_source.id", + permissions: []*accesscontrol.Permission{ + {Action: "datasources:read", Scope: "datasources:id:1*"}, + }, + expectedDataSources: []string{}, + }, { desc: "expect error if sqlID is not in the accept list", sqlID: "other.id", - permissions: []*Permission{ + permissions: []*accesscontrol.Permission{ {Action: "datasources:read", Scope: "datasources:id:3"}, {Action: "datasources:read", Scope: "datasources:id:7"}, {Action: "datasources:read", Scope: "datasources:id:8"}, @@ -198,9 +85,10 @@ func TestFilter_Datasources(t *testing.T) { } // set sqlIDAcceptList before running tests - sqlIDAcceptList = map[string]struct{}{ + restore := accesscontrol.SetAcceptListForTest(map[string]struct{}{ "data_source.id": {}, - } + }) + defer restore() for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { @@ -216,13 +104,12 @@ func TestFilter_Datasources(t *testing.T) { } baseSql := `SELECT data_source.* FROM data_source WHERE` - query, args, err := Filter( + query, args, err := accesscontrol.Filter( context.Background(), - &FakeDriver{name: "sqlite3"}, tt.sqlID, "datasources", "datasources:read", - &models.SignedInUser{OrgId: 1, Permissions: map[int64]map[string][]string{1: GroupScopesByAction(tt.permissions)}}, + &models.SignedInUser{OrgId: 1, Permissions: map[int64]map[string][]string{1: accesscontrol.GroupScopesByAction(tt.permissions)}}, ) if !tt.expectErr { @@ -241,11 +128,3 @@ func TestFilter_Datasources(t *testing.T) { }) } } - -type FakeDriver struct { - name string -} - -func (f FakeDriver) DriverName() string { - return f.name -} diff --git a/pkg/services/accesscontrol/middleware/middleware.go b/pkg/services/accesscontrol/middleware/middleware.go index 12c7a7044cb..ca5427b2c3b 100644 --- a/pkg/services/accesscontrol/middleware/middleware.go +++ b/pkg/services/accesscontrol/middleware/middleware.go @@ -3,6 +3,7 @@ package middleware import ( "fmt" "net/http" + "strconv" "time" "github.com/grafana/grafana/pkg/models" @@ -14,7 +15,7 @@ import ( ) func authorize(c *models.ReqContext, ac accesscontrol.AccessControl, user *models.SignedInUser, evaluator accesscontrol.Evaluator) { - injected, err := evaluator.Inject(buildScopeParams(c)) + injected, err := evaluator.MutateScopes(c.Req.Context(), accesscontrol.ScopeInjector(buildScopeParams(c))) if err != nil { c.JsonApiErr(http.StatusInternalServerError, "Internal server error", err) return @@ -48,7 +49,7 @@ func Deny(c *models.ReqContext, evaluator accesscontrol.Evaluator, err error) { "Access denied", "userID", c.UserId, "accessErrorID", id, - "permissions", evaluator.String(), + "permissions", evaluator.GoString(), ) } @@ -64,7 +65,7 @@ func Deny(c *models.ReqContext, evaluator accesscontrol.Evaluator, err error) { // internal server error or access denied. c.JSON(http.StatusForbidden, map[string]string{ "title": "Access denied", // the component needs to pick this up - "message": fmt.Sprintf("You'll need additional permissions to perform this action. Refer your administrator to a Grafana log with the reference %s to identify which permissions to add.", id), + "message": fmt.Sprintf("You'll need additional permissions to perform this action. Permissions needed: %s", evaluator.String()), "accessErrorId": id, }) } @@ -125,9 +126,10 @@ func AuthorizeInOrgMiddleware(ac accesscontrol.AccessControl, db *sqlstore.SQLSt } func UseOrgFromContextParams(c *models.ReqContext) (int64, error) { - orgID := c.ParamsInt64(":orgId") + orgID, err := strconv.ParseInt(web.Params(c.Req)[":orgId"], 10, 64) + // Special case of macaron handling invalid params - if orgID == 0 { + if orgID == 0 || err != nil { return 0, models.ErrOrgNotFound } diff --git a/pkg/services/accesscontrol/mock/mock.go b/pkg/services/accesscontrol/mock/mock.go index 10d0a79c0d2..5460749f9b9 100644 --- a/pkg/services/accesscontrol/mock/mock.go +++ b/pkg/services/accesscontrol/mock/mock.go @@ -14,13 +14,14 @@ type fullAccessControl interface { } type Calls struct { - Evaluate []interface{} - GetUserPermissions []interface{} - GetUserRoles []interface{} - IsDisabled []interface{} - DeclareFixedRoles []interface{} - GetUserBuiltInRoles []interface{} - RegisterFixedRoles []interface{} + Evaluate []interface{} + GetUserPermissions []interface{} + GetUserRoles []interface{} + IsDisabled []interface{} + DeclareFixedRoles []interface{} + GetUserBuiltInRoles []interface{} + RegisterFixedRoles []interface{} + RegisterAttributeScopeResolver []interface{} } type Mock struct { @@ -37,13 +38,14 @@ type Mock struct { Calls Calls // Override functions - EvaluateFunc func(context.Context, *models.SignedInUser, accesscontrol.Evaluator) (bool, error) - GetUserPermissionsFunc func(context.Context, *models.SignedInUser) ([]*accesscontrol.Permission, error) - GetUserRolesFunc func(context.Context, *models.SignedInUser) ([]*accesscontrol.RoleDTO, error) - IsDisabledFunc func() bool - DeclareFixedRolesFunc func(...accesscontrol.RoleRegistration) error - GetUserBuiltInRolesFunc func(user *models.SignedInUser) []string - RegisterFixedRolesFunc func() error + EvaluateFunc func(context.Context, *models.SignedInUser, accesscontrol.Evaluator) (bool, error) + GetUserPermissionsFunc func(context.Context, *models.SignedInUser) ([]*accesscontrol.Permission, error) + GetUserRolesFunc func(context.Context, *models.SignedInUser) ([]*accesscontrol.RoleDTO, error) + IsDisabledFunc func() bool + DeclareFixedRolesFunc func(...accesscontrol.RoleRegistration) error + GetUserBuiltInRolesFunc func(user *models.SignedInUser) []string + RegisterFixedRolesFunc func() error + RegisterAttributeScopeResolverFunc func(string, accesscontrol.AttributeScopeResolveFunc) } // Ensure the mock stays in line with the interface @@ -162,3 +164,13 @@ func (m *Mock) RegisterFixedRoles() error { } return nil } + +// RegisterAttributeScopeResolver allows the caller to register a scope resolver for a +// specific scope prefix (ex: datasources:name:) +func (m *Mock) RegisterAttributeScopeResolver(scopePrefix string, resolver accesscontrol.AttributeScopeResolveFunc) { + m.Calls.RegisterAttributeScopeResolver = append(m.Calls.RegisterAttributeScopeResolver, []struct{}{}) + // Use override if provided + if m.RegisterAttributeScopeResolverFunc != nil { + m.RegisterAttributeScopeResolverFunc(scopePrefix, resolver) + } +} diff --git a/pkg/services/accesscontrol/models.go b/pkg/services/accesscontrol/models.go index 5c54ffa9db4..a1d7c62e9fa 100644 --- a/pkg/services/accesscontrol/models.go +++ b/pkg/services/accesscontrol/models.go @@ -176,9 +176,10 @@ func (p Permission) OSSPermission() Permission { } type GetUserPermissionsQuery struct { - OrgID int64 `json:"-"` - UserID int64 `json:"userId"` - Roles []string + OrgID int64 `json:"-"` + UserID int64 `json:"userId"` + Roles []string + Actions []string } // ScopeParams holds the parameters used to fill in scope templates @@ -237,6 +238,7 @@ type SetResourcePermissionCommand struct { Actions []string Resource string ResourceID string + Permission string } type GetResourcesPermissionsQuery struct { @@ -311,8 +313,21 @@ const ( ActionLicensingDelete = "licensing:delete" ActionLicensingReportsRead = "licensing.reports:read" - // Team actions - ActionTeamsCreate = "teams:create" + // Team related actions + ActionTeamsCreate = "teams:create" + ActionTeamsDelete = "teams:delete" + ActionTeamsRead = "teams:read" + ActionTeamsWrite = "teams:write" + ActionTeamsPermissionsRead = "teams.permissions:read" + ActionTeamsPermissionsWrite = "teams.permissions:write" + + // Team related scopes + ScopeTeamsAll = "teams:*" +) + +var ( + // Team scope + ScopeTeamsID = Scope("teams", "id", Parameter(":teamId")) ) const RoleGrafanaAdmin = "Grafana Admin" diff --git a/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol.go b/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol.go index 4ae9742e767..0f343ebcb58 100644 --- a/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol.go +++ b/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol.go @@ -9,39 +9,46 @@ import ( "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/prometheus/client_golang/prometheus" ) -func ProvideService(cfg *setting.Cfg, usageStats usagestats.Service) *OSSAccessControlService { - s := &OSSAccessControlService{ - Cfg: cfg, - UsageStats: usageStats, - Log: log.New("accesscontrol"), - scopeResolver: accesscontrol.NewScopeResolver(), - } +func ProvideService(features featuremgmt.FeatureToggles, usageStats usagestats.Service, provider accesscontrol.PermissionsProvider) *OSSAccessControlService { + s := ProvideOSSAccessControl(features, usageStats, provider) s.registerUsageMetrics() return s } +// ProvideOSSAccessControl creates an oss implementation of access control without usage stats registration +func ProvideOSSAccessControl(features featuremgmt.FeatureToggles, usageStats usagestats.Service, provider accesscontrol.PermissionsProvider) *OSSAccessControlService { + return &OSSAccessControlService{ + features: features, + provider: provider, + usageStats: usageStats, + log: log.New("accesscontrol"), + scopeResolver: accesscontrol.NewScopeResolver(), + } +} + // OSSAccessControlService is the service implementing role based access control. type OSSAccessControlService struct { - Cfg *setting.Cfg - UsageStats usagestats.Service - Log log.Logger - registrations accesscontrol.RegistrationList + log log.Logger + usageStats usagestats.Service + features featuremgmt.FeatureToggles scopeResolver accesscontrol.ScopeResolver + provider accesscontrol.PermissionsProvider + registrations accesscontrol.RegistrationList } func (ac *OSSAccessControlService) IsDisabled() bool { - if ac.Cfg == nil { + if ac.features == nil { return true } - return !ac.Cfg.FeatureToggles["accesscontrol"] + return !ac.features.IsEnabled(featuremgmt.FlagAccesscontrol) } func (ac *OSSAccessControlService) registerUsageMetrics() { - ac.UsageStats.RegisterMetricsFunc(func(context.Context) (map[string]interface{}, error) { + ac.usageStats.RegisterMetricsFunc(func(context.Context) (map[string]interface{}, error) { return map[string]interface{}{ "stats.oss.accesscontrol.enabled.count": ac.getUsageMetrics(), }, nil @@ -74,7 +81,12 @@ func (ac *OSSAccessControlService) Evaluate(ctx context.Context, user *models.Si user.Permissions[user.OrgId] = accesscontrol.GroupScopesByAction(permissions) } - return evaluator.Evaluate(user.Permissions[user.OrgId]) + attributeMutator := ac.scopeResolver.GetResolveAttributeScopeMutator(user.OrgId) + resolvedEvaluator, err := evaluator.MutateScopes(ctx, attributeMutator) + if err != nil { + return false, err + } + return resolvedEvaluator.Evaluate(user.Permissions[user.OrgId]) } // GetUserRoles returns user permissions based on built-in roles @@ -87,28 +99,51 @@ func (ac *OSSAccessControlService) GetUserPermissions(ctx context.Context, user timer := prometheus.NewTimer(metrics.MAccessPermissionsSummary) defer timer.ObserveDuration() - builtinRoles := ac.GetUserBuiltInRoles(user) + permissions := ac.getFixedPermissions(ctx, user) + + dbPermissions, err := ac.provider.GetUserPermissions(ctx, accesscontrol.GetUserPermissionsQuery{ + OrgID: user.OrgId, + UserID: user.UserId, + Roles: ac.GetUserBuiltInRoles(user), + Actions: []string{}, + }) + if err != nil { + return nil, err + } + + permissions = append(permissions, dbPermissions...) + resolved := make([]*accesscontrol.Permission, 0, len(permissions)) + keywordMutator := ac.scopeResolver.GetResolveKeywordScopeMutator(user) + for _, p := range permissions { + // if the permission has a keyword in its scope it will be resolved + p.Scope, err = keywordMutator(ctx, p.Scope) + if err != nil { + return nil, err + } + resolved = append(resolved, p) + } + + return resolved, nil +} + +func (ac *OSSAccessControlService) getFixedPermissions(ctx context.Context, user *models.SignedInUser) []*accesscontrol.Permission { permissions := make([]*accesscontrol.Permission, 0) - for _, builtin := range builtinRoles { + + for _, builtin := range ac.GetUserBuiltInRoles(user) { if roleNames, ok := accesscontrol.FixedRoleGrants[builtin]; ok { for _, name := range roleNames { role, exists := accesscontrol.FixedRoles[name] if !exists { continue } - for _, p := range role.Permissions { - // if the permission has a keyword in its scope it will be resolved - permission, err := ac.scopeResolver.ResolveKeyword(user, p) - if err != nil { - return nil, err - } - permissions = append(permissions, permission) + for i := range role.Permissions { + permissions = append(permissions, &role.Permissions[i]) } } } } - return permissions, nil + return permissions } func (ac *OSSAccessControlService) GetUserBuiltInRoles(user *models.SignedInUser) []string { @@ -129,7 +164,7 @@ func (ac *OSSAccessControlService) saveFixedRole(role accesscontrol.RoleDTO) { // needs to be increased. Hence, we don't overwrite a role with a // greater version. if storedRole.Version >= role.Version { - ac.Log.Debug("the has already been stored in a greater version, skipping registration", "role", role.Name) + ac.log.Debug("the role has already been stored in a greater version, skipping registration", "role", role.Name) return } } @@ -145,7 +180,7 @@ func (ac *OSSAccessControlService) assignFixedRole(role accesscontrol.RoleDTO, b if ok { for _, assignedRole := range assignments { if assignedRole == role.Name { - ac.Log.Debug("the role has already been assigned", "rolename", role.Name, "build_in_role", builtInRole) + ac.log.Debug("the role has already been assigned", "rolename", role.Name, "build_in_role", builtInRole) alreadyAssigned = true } } @@ -201,3 +236,9 @@ func (ac *OSSAccessControlService) DeclareFixedRoles(registrations ...accesscont return nil } + +// RegisterAttributeScopeResolver allows the caller to register scope resolvers for a +// specific scope prefix (ex: datasources:name:) +func (ac *OSSAccessControlService) RegisterAttributeScopeResolver(scopePrefix string, resolver accesscontrol.AttributeScopeResolveFunc) { + ac.scopeResolver.AddAttributeResolver(scopePrefix, resolver) +} diff --git a/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol_test.go b/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol_test.go index d4c4fc1583f..103e86d498a 100644 --- a/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol_test.go +++ b/pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol_test.go @@ -12,21 +12,21 @@ import ( "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/services/accesscontrol/database" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/sqlstore" ) func setupTestEnv(t testing.TB) *OSSAccessControlService { t.Helper() - cfg := setting.NewCfg() - cfg.FeatureToggles = map[string]bool{"accesscontrol": true} - ac := &OSSAccessControlService{ - Cfg: cfg, - UsageStats: &usagestats.UsageStatsMock{T: t}, - Log: log.New("accesscontrol"), + features: featuremgmt.WithFeatures(featuremgmt.FlagAccesscontrol), + usageStats: &usagestats.UsageStatsMock{T: t}, + log: log.New("accesscontrol"), registrations: accesscontrol.RegistrationList{}, scopeResolver: accesscontrol.NewScopeResolver(), + provider: database.ProvideService(sqlstore.InitTestDB(t)), } return ac } @@ -148,13 +148,12 @@ func TestUsageMetrics(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfg := setting.NewCfg() - if tt.enabled { - cfg.FeatureToggles = map[string]bool{"accesscontrol": true} - } - - s := ProvideService(cfg, &usagestats.UsageStatsMock{T: t}) - report, err := s.UsageStats.GetUsageReport(context.Background()) + s := ProvideService( + featuremgmt.WithFeatures("accesscontrol", tt.enabled), + &usagestats.UsageStatsMock{T: t}, + database.ProvideService(sqlstore.InitTestDB(t)), + ) + report, err := s.usageStats.GetUsageReport(context.Background()) assert.Nil(t, err) assert.Equal(t, tt.expectedValue, report.Metrics["stats.oss.accesscontrol.enabled.count"]) @@ -267,9 +266,9 @@ func TestOSSAccessControlService_RegisterFixedRole(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ac := &OSSAccessControlService{ - Cfg: setting.NewCfg(), - UsageStats: &usagestats.UsageStatsMock{T: t}, - Log: log.New("accesscontrol-test"), + features: featuremgmt.WithFeatures(), + usageStats: &usagestats.UsageStatsMock{T: t}, + log: log.New("accesscontrol-test"), } for i, run := range tc.runs { @@ -385,13 +384,7 @@ func TestOSSAccessControlService_DeclareFixedRoles(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ac := &OSSAccessControlService{ - Cfg: setting.NewCfg(), - UsageStats: &usagestats.UsageStatsMock{T: t}, - Log: log.New("accesscontrol-test"), - registrations: accesscontrol.RegistrationList{}, - } - ac.Cfg.FeatureToggles = map[string]bool{"accesscontrol": true} + ac := setupTestEnv(t) // Test err := ac.DeclareFixedRoles(tt.registrations...) @@ -459,9 +452,6 @@ func TestOSSAccessControlService_RegisterFixedRoles(t *testing.T) { } for _, tt := range tests { - cfg := setting.NewCfg() - cfg.FeatureToggles = map[string]bool{"accesscontrol": true} - t.Run(tt.name, func(t *testing.T) { // Remove any inserted role after the test case has been run t.Cleanup(func() { @@ -469,15 +459,7 @@ func TestOSSAccessControlService_RegisterFixedRoles(t *testing.T) { removeRoleHelper(registration.Role.Name) } }) - - // Setup - ac := &OSSAccessControlService{ - Cfg: setting.NewCfg(), - UsageStats: &usagestats.UsageStatsMock{T: t}, - Log: log.New("accesscontrol-test"), - registrations: accesscontrol.RegistrationList{}, - } - ac.Cfg.FeatureToggles = map[string]bool{"accesscontrol": true} + ac := setupTestEnv(t) ac.registrations.Append(tt.registrations...) // Test @@ -552,7 +534,6 @@ func TestOSSAccessControlService_GetUserPermissions(t *testing.T) { // Setup ac := setupTestEnv(t) - ac.Cfg.FeatureToggles = map[string]bool{"accesscontrol": true} registration.Role.Permissions = []accesscontrol.Permission{tt.rawPerm} err := ac.DeclareFixedRoles(registration) @@ -576,3 +557,86 @@ func TestOSSAccessControlService_GetUserPermissions(t *testing.T) { }) } } + +func TestOSSAccessControlService_Evaluate(t *testing.T) { + testUser := models.SignedInUser{ + UserId: 2, + OrgId: 3, + OrgName: "TestOrg", + OrgRole: models.ROLE_VIEWER, + Login: "testUser", + Name: "Test User", + Email: "testuser@example.org", + } + registration := accesscontrol.RoleRegistration{ + Role: accesscontrol.RoleDTO{ + Version: 1, + UID: "fixed:test:test", + Name: "fixed:test:test", + Description: "Test role", + Permissions: []accesscontrol.Permission{}, + }, + Grants: []string{"Viewer"}, + } + userLoginScopeSolver := func(ctx context.Context, orgID int64, initialScope string) (string, error) { + if initialScope == "users:login:testUser" { + return "users:id:2", nil + } + return initialScope, nil + } + + tests := []struct { + name string + user models.SignedInUser + rawPerm accesscontrol.Permission + evaluator accesscontrol.Evaluator + wantAccess bool + wantErr bool + }{ + { + name: "Should translate users:self", + user: testUser, + rawPerm: accesscontrol.Permission{Action: "users:read", Scope: "users:self"}, + evaluator: accesscontrol.EvalPermission("users:read", "users:id:2"), + wantAccess: true, + wantErr: false, + }, + { + name: "Should translate users:login:testUser", + user: testUser, + rawPerm: accesscontrol.Permission{Action: "users:read", Scope: "users:id:2"}, + evaluator: accesscontrol.EvalPermission("users:read", "users:login:testUser"), + wantAccess: true, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Remove any inserted role after the test case has been run + t.Cleanup(func() { + removeRoleHelper(registration.Role.Name) + }) + + // Setup + ac := setupTestEnv(t) + ac.RegisterAttributeScopeResolver("users:login:", userLoginScopeSolver) + + registration.Role.Permissions = []accesscontrol.Permission{tt.rawPerm} + err := ac.DeclareFixedRoles(registration) + require.NoError(t, err) + + err = ac.RegisterFixedRoles() + require.NoError(t, err) + + // Test + hasAccess, err := ac.Evaluate(context.TODO(), &tt.user, tt.evaluator) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + + assert.Equal(t, tt.wantAccess, hasAccess) + }) + } +} diff --git a/pkg/services/accesscontrol/resourcepermissions/api.go b/pkg/services/accesscontrol/resourcepermissions/api.go index ed1b5337410..84a93834854 100644 --- a/pkg/services/accesscontrol/resourcepermissions/api.go +++ b/pkg/services/accesscontrol/resourcepermissions/api.go @@ -3,6 +3,7 @@ package resourcepermissions import ( "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" @@ -87,7 +88,7 @@ func (a *api) getPermissions(c *models.ReqContext) response.Response { dto := make([]resourcePermissionDTO, 0, len(permissions)) for _, p := range permissions { - if permission := a.service.MapActions(p); permission != "" { + if permission := a.service.mapActions(p); permission != "" { teamAvatarUrl := "" if p.TeamId != 0 { teamAvatarUrl = dtos.GetGravatarUrlWithDefault(p.TeamEmail, p.Team) @@ -119,7 +120,10 @@ type setPermissionCommand struct { } func (a *api) setUserPermission(c *models.ReqContext) response.Response { - userID := c.ParamsInt64(":userID") + userID, err := strconv.ParseInt(web.Params(c.Req)[":userID"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "userID is invalid", err) + } resourceID := web.Params(c.Req)[":resourceID"] var cmd setPermissionCommand @@ -127,7 +131,7 @@ func (a *api) setUserPermission(c *models.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "bad request data", err) } - _, err := a.service.SetUserPermission(c.Req.Context(), c.OrgId, userID, resourceID, a.service.MapPermission(cmd.Permission)) + _, err = a.service.SetUserPermission(c.Req.Context(), c.OrgId, userID, resourceID, cmd.Permission) if err != nil { return response.Error(http.StatusBadRequest, "failed to set user permission", err) } @@ -136,7 +140,10 @@ func (a *api) setUserPermission(c *models.ReqContext) response.Response { } func (a *api) setTeamPermission(c *models.ReqContext) response.Response { - teamID := c.ParamsInt64(":teamID") + teamID, err := strconv.ParseInt(web.Params(c.Req)[":teamID"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "teamID is invalid", err) + } resourceID := web.Params(c.Req)[":resourceID"] var cmd setPermissionCommand @@ -144,7 +151,7 @@ func (a *api) setTeamPermission(c *models.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "bad request data", err) } - _, err := a.service.SetTeamPermission(c.Req.Context(), c.OrgId, teamID, resourceID, a.service.MapPermission(cmd.Permission)) + _, err = a.service.SetTeamPermission(c.Req.Context(), c.OrgId, teamID, resourceID, cmd.Permission) if err != nil { return response.Error(http.StatusBadRequest, "failed to set team permission", err) } @@ -161,7 +168,7 @@ func (a *api) setBuiltinRolePermission(c *models.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "bad request data", err) } - _, err := a.service.SetBuiltInRolePermission(c.Req.Context(), c.OrgId, builtInRole, resourceID, a.service.MapPermission(cmd.Permission)) + _, err := a.service.SetBuiltInRolePermission(c.Req.Context(), c.OrgId, builtInRole, resourceID, cmd.Permission) if err != nil { return response.Error(http.StatusBadRequest, "failed to set role permission", err) } diff --git a/pkg/services/accesscontrol/resourcepermissions/api_test.go b/pkg/services/accesscontrol/resourcepermissions/api_test.go index b16e37361fc..6e596875cd4 100644 --- a/pkg/services/accesscontrol/resourcepermissions/api_test.go +++ b/pkg/services/accesscontrol/resourcepermissions/api_test.go @@ -155,16 +155,16 @@ func TestApi_getPermissions(t *testing.T) { // seed team 1 with "Edit" permission on dashboard 1 team, err := sql.CreateTeam("test", "test@test.com", 1) require.NoError(t, err) - _, err = service.SetTeamPermission(context.Background(), team.OrgId, team.Id, tt.resourceID, []string{"dashboards:read", "dashboards:write", "dashboards:delete"}) + _, err = service.SetTeamPermission(context.Background(), team.OrgId, team.Id, tt.resourceID, "Edit") require.NoError(t, err) // seed user 1 with "View" permission on dashboard 1 u, err := sql.CreateUser(context.Background(), models.CreateUserCommand{Login: "test", OrgId: 1}) require.NoError(t, err) - _, err = service.SetUserPermission(context.Background(), u.OrgId, u.Id, tt.resourceID, []string{"dashboards:read"}) + _, err = service.SetUserPermission(context.Background(), u.OrgId, u.Id, tt.resourceID, "View") require.NoError(t, err) - // seed built in role Admin with "View" permission on dashboard 1 - _, err = service.SetBuiltInRolePermission(context.Background(), 1, "Admin", tt.resourceID, []string{"dashboards:read", "dashboards:write", "dashboards:delete"}) + // seed built in role Admin with "Edit" permission on dashboard 1 + _, err = service.SetBuiltInRolePermission(context.Background(), 1, "Admin", tt.resourceID, "Edit") require.NoError(t, err) permissions, recorder := getPermission(t, server, testOptions.Resource, tt.resourceID) diff --git a/pkg/services/accesscontrol/resourcepermissions/error.go b/pkg/services/accesscontrol/resourcepermissions/error.go index 1c155f72f58..53a8164a926 100644 --- a/pkg/services/accesscontrol/resourcepermissions/error.go +++ b/pkg/services/accesscontrol/resourcepermissions/error.go @@ -3,6 +3,6 @@ package resourcepermissions import "errors" var ( - ErrInvalidActions = errors.New("invalid actions") + ErrInvalidPermission = errors.New("invalid permission") ErrInvalidAssignment = errors.New("invalid assignment") ) diff --git a/pkg/services/accesscontrol/resourcepermissions/options.go b/pkg/services/accesscontrol/resourcepermissions/options.go index fd6c93478f1..907a202ad5c 100644 --- a/pkg/services/accesscontrol/resourcepermissions/options.go +++ b/pkg/services/accesscontrol/resourcepermissions/options.go @@ -2,6 +2,8 @@ package resourcepermissions import ( "context" + + "github.com/grafana/grafana/pkg/services/sqlstore" ) type ResourceValidator func(ctx context.Context, orgID int64, resourceID string) error @@ -26,9 +28,9 @@ type Options struct { // RoleGroup is the group name for the generated fixed roles RoleGroup string // OnSetUser if configured will be called each time a permission is set for a user - OnSetUser func(ctx context.Context, orgID, userID int64, resourceID, permission string) error + OnSetUser func(session *sqlstore.DBSession, orgID, userID int64, resourceID, permission string) error // OnSetTeam if configured will be called each time a permission is set for a team - OnSetTeam func(ctx context.Context, orgID, teamID int64, resourceID, permission string) error + OnSetTeam func(session *sqlstore.DBSession, orgID, teamID int64, resourceID, permission string) error // OnSetBuiltInRole if configured will be called each time a permission is set for a built-in role - OnSetBuiltInRole func(ctx context.Context, orgID int64, builtInRole, resourceID, permission string) error + OnSetBuiltInRole func(session *sqlstore.DBSession, orgID int64, builtInRole, resourceID, permission string) error } diff --git a/pkg/services/accesscontrol/resourcepermissions/service.go b/pkg/services/accesscontrol/resourcepermissions/service.go index b06df263ee1..6ab670a313e 100644 --- a/pkg/services/accesscontrol/resourcepermissions/service.go +++ b/pkg/services/accesscontrol/resourcepermissions/service.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions/types" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/api/routing" @@ -12,13 +13,39 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol" ) -func New(options Options, router routing.RouteRegister, ac accesscontrol.AccessControl, store accesscontrol.ResourcePermissionsStore) (*Service, error) { +type Store interface { + // SetUserResourcePermission sets permission for managed user role on a resource + SetUserResourcePermission( + ctx context.Context, orgID, userID int64, + cmd accesscontrol.SetResourcePermissionCommand, + hook types.UserResourceHookFunc, + ) (*accesscontrol.ResourcePermission, error) + + // SetTeamResourcePermission sets permission for managed team role on a resource + SetTeamResourcePermission( + ctx context.Context, orgID, teamID int64, + cmd accesscontrol.SetResourcePermissionCommand, + hook types.TeamResourceHookFunc, + ) (*accesscontrol.ResourcePermission, error) + + // SetBuiltInResourcePermission sets permissions for managed builtin role on a resource + SetBuiltInResourcePermission( + ctx context.Context, orgID int64, builtinRole string, + cmd accesscontrol.SetResourcePermissionCommand, + hook types.BuiltinResourceHookFunc, + ) (*accesscontrol.ResourcePermission, error) + + // GetResourcesPermissions will return all permission for all supplied resource ids + GetResourcesPermissions(ctx context.Context, orgID int64, query accesscontrol.GetResourcesPermissionsQuery) ([]accesscontrol.ResourcePermission, error) +} + +func New(options Options, router routing.RouteRegister, ac accesscontrol.AccessControl, store Store, sqlStore *sqlstore.SQLStore) (*Service, error) { var permissions []string - validActions := make(map[string]struct{}) + actionSet := make(map[string]struct{}) for permission, actions := range options.PermissionsToActions { permissions = append(permissions, permission) for _, a := range actions { - validActions[a] = struct{}{} + actionSet[a] = struct{}{} } } @@ -27,18 +54,18 @@ func New(options Options, router routing.RouteRegister, ac accesscontrol.AccessC return len(options.PermissionsToActions[permissions[i]]) > len(options.PermissionsToActions[permissions[j]]) }) - actions := make([]string, 0, len(validActions)) - for action := range validActions { + actions := make([]string, 0, len(actionSet)) + for action := range actionSet { actions = append(actions, action) } s := &Service{ - ac: ac, - store: store, - options: options, - permissions: permissions, - actions: actions, - validActions: validActions, + ac: ac, + store: store, + options: options, + permissions: permissions, + actions: actions, + sqlStore: sqlStore, } s.api = newApi(ac, router, s) @@ -55,13 +82,13 @@ func New(options Options, router routing.RouteRegister, ac accesscontrol.AccessC // Service is used to create access control sub system including api / and service for managed resource permission type Service struct { ac accesscontrol.AccessControl - store accesscontrol.ResourcePermissionsStore + store Store api *api - options Options - permissions []string - actions []string - validActions map[string]struct{} + options Options + permissions []string + actions []string + sqlStore *sqlstore.SQLStore } func (s *Service) GetPermissions(ctx context.Context, orgID int64, resourceID string) ([]accesscontrol.ResourcePermission, error) { @@ -73,13 +100,14 @@ func (s *Service) GetPermissions(ctx context.Context, orgID int64, resourceID st }) } -func (s *Service) SetUserPermission(ctx context.Context, orgID, userID int64, resourceID string, actions []string) (*accesscontrol.ResourcePermission, error) { +func (s *Service) SetUserPermission(ctx context.Context, orgID, userID int64, resourceID, permission string) (*accesscontrol.ResourcePermission, error) { if !s.options.Assignments.Users { return nil, ErrInvalidAssignment } - if !s.validateActions(actions) { - return nil, ErrInvalidActions + actions, err := s.mapPermission(permission) + if err != nil { + return nil, err } if err := s.validateResource(ctx, orgID, resourceID); err != nil { @@ -90,29 +118,22 @@ func (s *Service) SetUserPermission(ctx context.Context, orgID, userID int64, re return nil, err } - permission, err := s.store.SetUserResourcePermission(ctx, orgID, userID, accesscontrol.SetResourcePermissionCommand{ + return s.store.SetUserResourcePermission(ctx, orgID, userID, accesscontrol.SetResourcePermissionCommand{ Actions: actions, + Permission: permission, ResourceID: resourceID, Resource: s.options.Resource, - }) - if err != nil { - return nil, err - } - - if s.options.OnSetUser != nil { - if err := s.options.OnSetUser(ctx, orgID, userID, resourceID, s.MapActions(*permission)); err != nil { - return nil, err - } - } - return permission, nil + }, s.options.OnSetUser) } -func (s *Service) SetTeamPermission(ctx context.Context, orgID, teamID int64, resourceID string, actions []string) (*accesscontrol.ResourcePermission, error) { +func (s *Service) SetTeamPermission(ctx context.Context, orgID, teamID int64, resourceID, permission string) (*accesscontrol.ResourcePermission, error) { if !s.options.Assignments.Teams { return nil, ErrInvalidAssignment } - if !s.validateActions(actions) { - return nil, ErrInvalidActions + + actions, err := s.mapPermission(permission) + if err != nil { + return nil, err } if err := s.validateTeam(ctx, orgID, teamID); err != nil { @@ -123,30 +144,22 @@ func (s *Service) SetTeamPermission(ctx context.Context, orgID, teamID int64, re return nil, err } - permission, err := s.store.SetTeamResourcePermission(ctx, orgID, teamID, accesscontrol.SetResourcePermissionCommand{ + return s.store.SetTeamResourcePermission(ctx, orgID, teamID, accesscontrol.SetResourcePermissionCommand{ Actions: actions, + Permission: permission, ResourceID: resourceID, Resource: s.options.Resource, - }) - if err != nil { - return nil, err - } - - if s.options.OnSetTeam != nil { - if err := s.options.OnSetTeam(ctx, orgID, teamID, resourceID, s.MapActions(*permission)); err != nil { - return nil, err - } - } - return permission, nil + }, s.options.OnSetTeam) } -func (s *Service) SetBuiltInRolePermission(ctx context.Context, orgID int64, builtInRole string, resourceID string, actions []string) (*accesscontrol.ResourcePermission, error) { +func (s *Service) SetBuiltInRolePermission(ctx context.Context, orgID int64, builtInRole, resourceID, permission string) (*accesscontrol.ResourcePermission, error) { if !s.options.Assignments.BuiltInRoles { return nil, ErrInvalidAssignment } - if !s.validateActions(actions) { - return nil, ErrInvalidActions + actions, err := s.mapPermission(permission) + if err != nil { + return nil, err } if err := s.validateBuiltinRole(ctx, builtInRole); err != nil { @@ -157,24 +170,15 @@ func (s *Service) SetBuiltInRolePermission(ctx context.Context, orgID int64, bui return nil, err } - permission, err := s.store.SetBuiltInResourcePermission(ctx, orgID, builtInRole, accesscontrol.SetResourcePermissionCommand{ + return s.store.SetBuiltInResourcePermission(ctx, orgID, builtInRole, accesscontrol.SetResourcePermissionCommand{ Actions: actions, + Permission: permission, ResourceID: resourceID, Resource: s.options.Resource, - }) - if err != nil { - return nil, err - } - - if s.options.OnSetBuiltInRole != nil { - if err := s.options.OnSetBuiltInRole(ctx, orgID, builtInRole, resourceID, s.MapActions(*permission)); err != nil { - return nil, err - } - } - return permission, nil + }, s.options.OnSetBuiltInRole) } -func (s *Service) MapActions(permission accesscontrol.ResourcePermission) string { +func (s *Service) mapActions(permission accesscontrol.ResourcePermission) string { for _, p := range s.permissions { if permission.Contains(s.options.PermissionsToActions[p]) { return p @@ -183,13 +187,17 @@ func (s *Service) MapActions(permission accesscontrol.ResourcePermission) string return "" } -func (s *Service) MapPermission(permission string) []string { +func (s *Service) mapPermission(permission string) ([]string, error) { + if permission == "" { + return []string{}, nil + } + for k, v := range s.options.PermissionsToActions { if permission == k { - return v + return v, nil } } - return []string{} + return nil, ErrInvalidPermission } func (s *Service) validateResource(ctx context.Context, orgID int64, resourceID string) error { @@ -199,24 +207,15 @@ func (s *Service) validateResource(ctx context.Context, orgID int64, resourceID return nil } -func (s *Service) validateActions(actions []string) bool { - for _, a := range actions { - if _, ok := s.validActions[a]; !ok { - return false - } - } - return true -} - func (s *Service) validateUser(ctx context.Context, orgID, userID int64) error { - if err := sqlstore.GetSignedInUser(ctx, &models.GetSignedInUserQuery{OrgId: orgID, UserId: userID}); err != nil { + if err := s.sqlStore.GetSignedInUser(ctx, &models.GetSignedInUserQuery{OrgId: orgID, UserId: userID}); err != nil { return err } return nil } func (s *Service) validateTeam(ctx context.Context, orgID, teamID int64) error { - if err := sqlstore.GetTeamById(ctx, &models.GetTeamByIdQuery{OrgId: orgID, Id: teamID}); err != nil { + if err := s.sqlStore.GetTeamById(ctx, &models.GetTeamByIdQuery{OrgId: orgID, Id: teamID}); err != nil { return err } return nil diff --git a/pkg/services/accesscontrol/resourcepermissions/service_test.go b/pkg/services/accesscontrol/resourcepermissions/service_test.go index 9adb896e827..dbdcd3e4b5e 100644 --- a/pkg/services/accesscontrol/resourcepermissions/service_test.go +++ b/pkg/services/accesscontrol/resourcepermissions/service_test.go @@ -46,13 +46,13 @@ func TestService_SetUserPermission(t *testing.T) { var hookCalled bool if tt.callHook { - service.options.OnSetUser = func(ctx context.Context, orgID, userID int64, resourceID, permission string) error { + service.options.OnSetUser = func(session *sqlstore.DBSession, orgID, userID int64, resourceID, permission string) error { hookCalled = true return nil } } - _, err = service.SetUserPermission(context.Background(), user.OrgId, user.Id, "1", []string{}) + _, err = service.SetUserPermission(context.Background(), user.OrgId, user.Id, "1", "") require.NoError(t, err) assert.Equal(t, tt.callHook, hookCalled) }) @@ -90,13 +90,13 @@ func TestService_SetTeamPermission(t *testing.T) { var hookCalled bool if tt.callHook { - service.options.OnSetTeam = func(ctx context.Context, orgID, teamID int64, resourceID, permission string) error { + service.options.OnSetTeam = func(session *sqlstore.DBSession, orgID, teamID int64, resourceID, permission string) error { hookCalled = true return nil } } - _, err = service.SetTeamPermission(context.Background(), team.OrgId, team.Id, "1", []string{}) + _, err = service.SetTeamPermission(context.Background(), team.OrgId, team.Id, "1", "") require.NoError(t, err) assert.Equal(t, tt.callHook, hookCalled) }) @@ -130,13 +130,13 @@ func TestService_SetBuiltInRolePermission(t *testing.T) { var hookCalled bool if tt.callHook { - service.options.OnSetBuiltInRole = func(ctx context.Context, orgID int64, builtInRole, resourceID, permission string) error { + service.options.OnSetBuiltInRole = func(session *sqlstore.DBSession, orgID int64, builtInRole, resourceID, permission string) error { hookCalled = true return nil } } - _, err := service.SetBuiltInRolePermission(context.Background(), 1, "Viewer", "1", []string{}) + _, err := service.SetBuiltInRolePermission(context.Background(), 1, "Viewer", "1", "") require.NoError(t, err) assert.Equal(t, tt.callHook, hookCalled) }) @@ -148,7 +148,7 @@ func setupTestEnvironment(t *testing.T, permissions []*accesscontrol.Permission, sql := sqlstore.InitTestDB(t) store := database.ProvideService(sql) - service, err := New(ops, routing.NewRouteRegister(), accesscontrolmock.New().WithPermissions(permissions), store) + service, err := New(ops, routing.NewRouteRegister(), accesscontrolmock.New().WithPermissions(permissions), store, sql) require.NoError(t, err) return service, sql diff --git a/pkg/services/accesscontrol/resourcepermissions/types/hook.go b/pkg/services/accesscontrol/resourcepermissions/types/hook.go new file mode 100644 index 00000000000..5e86cc3893c --- /dev/null +++ b/pkg/services/accesscontrol/resourcepermissions/types/hook.go @@ -0,0 +1,7 @@ +package types + +import "github.com/grafana/grafana/pkg/services/sqlstore" + +type UserResourceHookFunc func(session *sqlstore.DBSession, orgID, userID int64, resourceID, permission string) error +type TeamResourceHookFunc func(session *sqlstore.DBSession, orgID, teamID int64, resourceID, permission string) error +type BuiltinResourceHookFunc func(session *sqlstore.DBSession, orgID int64, builtInRole, resourceID, permission string) error diff --git a/pkg/services/accesscontrol/resourceservices/resource_services.go b/pkg/services/accesscontrol/resourceservices/resource_services.go new file mode 100644 index 00000000000..077e6cd93a5 --- /dev/null +++ b/pkg/services/accesscontrol/resourceservices/resource_services.go @@ -0,0 +1,103 @@ +package resourceservices + +import ( + "context" + "fmt" + "strconv" + + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func ProvideResourceServices(router routing.RouteRegister, sql *sqlstore.SQLStore, ac accesscontrol.AccessControl, store resourcepermissions.Store) (*ResourceServices, error) { + teamPermissions, err := ProvideTeamPermissions(router, sql, ac, store) + if err != nil { + return nil, err + } + + return &ResourceServices{services: map[string]*resourcepermissions.Service{ + "teams": teamPermissions, + }}, nil +} + +type ResourceServices struct { + services map[string]*resourcepermissions.Service +} + +func (s *ResourceServices) GetTeamService() *resourcepermissions.Service { + return s.services["teams"] +} + +var ( + TeamMemberActions = []string{ + accesscontrol.ActionTeamsRead, + } + + TeamAdminActions = []string{ + accesscontrol.ActionTeamsRead, + accesscontrol.ActionTeamsDelete, + accesscontrol.ActionTeamsWrite, + accesscontrol.ActionTeamsPermissionsRead, + accesscontrol.ActionTeamsPermissionsWrite, + } +) + +func ProvideTeamPermissions(router routing.RouteRegister, sql *sqlstore.SQLStore, ac accesscontrol.AccessControl, store resourcepermissions.Store) (*resourcepermissions.Service, error) { + options := resourcepermissions.Options{ + Resource: "teams", + OnlyManaged: true, + ResourceValidator: func(ctx context.Context, orgID int64, resourceID string) error { + id, err := strconv.ParseInt(resourceID, 10, 64) + if err != nil { + return err + } + + err = sql.GetTeamById(context.Background(), &models.GetTeamByIdQuery{ + OrgId: orgID, + Id: id, + }) + if err != nil { + return err + } + + return nil + }, + Assignments: resourcepermissions.Assignments{ + Users: true, + Teams: false, + BuiltInRoles: false, + }, + PermissionsToActions: map[string][]string{ + "Member": TeamMemberActions, + "Admin": TeamAdminActions, + }, + ReaderRoleName: "Team permission reader", + WriterRoleName: "Team permission writer", + RoleGroup: "Teams", + OnSetUser: func(session *sqlstore.DBSession, orgID, userID int64, resourceID, permission string) error { + teamId, err := strconv.ParseInt(resourceID, 10, 64) + if err != nil { + return err + } + switch permission { + case "Member": + return sqlstore.AddOrUpdateTeamMemberHook(session, userID, orgID, teamId, false, 0) + case "Admin": + return sqlstore.AddOrUpdateTeamMemberHook(session, userID, orgID, teamId, false, models.PERMISSION_ADMIN) + case "": + return sqlstore.RemoveTeamMemberHook(session, &models.RemoveTeamMemberCommand{ + OrgId: orgID, + UserId: userID, + TeamId: teamId, + }) + default: + return fmt.Errorf("invalid team permission type %s", permission) + } + }, + } + + return resourcepermissions.New(options, router, ac, store, sql) +} diff --git a/pkg/services/accesscontrol/roles.go b/pkg/services/accesscontrol/roles.go index 6e1107faf3d..5bbb6b8a633 100644 --- a/pkg/services/accesscontrol/roles.go +++ b/pkg/services/accesscontrol/roles.go @@ -15,19 +15,6 @@ type RoleRegistry interface { // Roles definition var ( - datasourcesExplorerRole = RoleDTO{ - Version: 3, - Name: datasourcesExplorer, - DisplayName: "Data source explorer", - Description: "Enable the Explore feature. Data source permissions still apply; you can only query data sources for which you have query permissions.", - Group: "Data sources", - Permissions: []Permission{ - { - Action: ActionDatasourcesExplore, - }, - }, - } - ldapReaderRole = RoleDTO{ Name: ldapReader, DisplayName: "LDAP reader", @@ -201,15 +188,14 @@ var ( // Role names definitions const ( - datasourcesExplorer = "fixed:datasources:explorer" - ldapReader = "fixed:ldap:reader" - ldapWriter = "fixed:ldap:writer" - orgUsersReader = "fixed:org.users:reader" - orgUsersWriter = "fixed:org.users:writer" - settingsReader = "fixed:settings:reader" - statsReader = "fixed:stats:reader" - usersReader = "fixed:users:reader" - usersWriter = "fixed:users:writer" + ldapReader = "fixed:ldap:reader" + ldapWriter = "fixed:ldap:writer" + orgUsersReader = "fixed:org.users:reader" + orgUsersWriter = "fixed:org.users:writer" + settingsReader = "fixed:settings:reader" + statsReader = "fixed:stats:reader" + usersReader = "fixed:users:reader" + usersWriter = "fixed:users:writer" ) var ( @@ -220,15 +206,14 @@ var ( // resource. FixedRoleGrants lists which built-in roles are // assigned which fixed roles in this list. FixedRoles = map[string]RoleDTO{ - datasourcesExplorer: datasourcesExplorerRole, - ldapReader: ldapReaderRole, - ldapWriter: ldapWriterRole, - orgUsersReader: orgUsersReaderRole, - orgUsersWriter: orgUsersWriterRole, - settingsReader: settingsReaderRole, - statsReader: statsReaderRole, - usersReader: usersReaderRole, - usersWriter: usersWriterRole, + ldapReader: ldapReaderRole, + ldapWriter: ldapWriterRole, + orgUsersReader: orgUsersReaderRole, + orgUsersWriter: orgUsersWriterRole, + settingsReader: settingsReaderRole, + statsReader: statsReaderRole, + usersReader: usersReaderRole, + usersWriter: usersWriterRole, } // FixedRoleGrants specifies which built-in roles are assigned @@ -248,9 +233,6 @@ var ( orgUsersReader, orgUsersWriter, }, - string(models.ROLE_EDITOR): { - datasourcesExplorer, - }, } ) diff --git a/pkg/services/accesscontrol/scope.go b/pkg/services/accesscontrol/scope.go index d5f87ee3660..eb8c5476c7b 100644 --- a/pkg/services/accesscontrol/scope.go +++ b/pkg/services/accesscontrol/scope.go @@ -1,12 +1,23 @@ package accesscontrol import ( + "bytes" + "context" "fmt" + "html/template" "strings" + "time" + "github.com/grafana/grafana/pkg/infra/localcache" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" ) +const ( + ttl = 30 * time.Second + cleanInterval = 2 * time.Minute +) + func GetResourceScope(resource string, resourceID string) string { return Scope(resource, "id", resourceID) } @@ -44,11 +55,17 @@ func Field(key string) string { return fmt.Sprintf(`{{ .%s }}`, key) } +// ScopeMutator alters a Scope to return a new modified Scope +type ScopeMutator func(context.Context, string) (string, error) + type KeywordScopeResolveFunc func(*models.SignedInUser) (string, error) -// ScopeResolver contains a map of functions to resolve scope keywords such as `self` or `current` into `id` based scopes +// ScopeResolver is used to resolve scope keywords such as `self` or `current` into `id` based scopes and scope attributes such as `name` or `uid` into `id` based scopes. type ScopeResolver struct { - keywordResolvers map[string]KeywordScopeResolveFunc + keywordResolvers map[string]KeywordScopeResolveFunc + attributeResolvers map[string]AttributeScopeResolveFunc + cache *localcache.CacheService + log log.Logger } func NewScopeResolver() ScopeResolver { @@ -56,21 +73,95 @@ func NewScopeResolver() ScopeResolver { keywordResolvers: map[string]KeywordScopeResolveFunc{ "users:self": resolveUserSelf, }, + attributeResolvers: map[string]AttributeScopeResolveFunc{}, + cache: localcache.New(ttl, cleanInterval), + log: log.New("accesscontrol.scoperesolution"), } } +func (s *ScopeResolver) AddKeywordResolver(keyword string, fn KeywordScopeResolveFunc) { + s.log.Debug("adding keyword resolution for '%v'", keyword) + s.keywordResolvers[keyword] = fn +} + +func (s *ScopeResolver) AddAttributeResolver(prefix string, fn AttributeScopeResolveFunc) { + s.log.Debug("adding attribute resolution for '%v'", prefix) + s.attributeResolvers[prefix] = fn +} + func resolveUserSelf(u *models.SignedInUser) (string, error) { return Scope("users", "id", fmt.Sprintf("%v", u.UserId)), nil } -// ResolveKeyword resolves scope with keywords such as `self` or `current` into `id` based scopes -func (s *ScopeResolver) ResolveKeyword(user *models.SignedInUser, permission Permission) (*Permission, error) { - if fn, ok := s.keywordResolvers[permission.Scope]; ok { - resolvedScope, err := fn(user) - if err != nil { - return nil, fmt.Errorf("could not resolve %v: %v", permission.Scope, err) +// GetResolveKeywordScopeMutator returns a function to resolve scope with keywords such as `self` or `current` into `id` based scopes +func (s *ScopeResolver) GetResolveKeywordScopeMutator(user *models.SignedInUser) ScopeMutator { + return func(_ context.Context, scope string) (string, error) { + var err error + // By default the scope remains unchanged + resolvedScope := scope + if fn, ok := s.keywordResolvers[scope]; ok { + resolvedScope, err = fn(user) + if err != nil { + return "", fmt.Errorf("could not resolve %v: %w", scope, err) + } + s.log.Debug("resolved '%v' to '%v'", scope, resolvedScope) } - permission.Scope = resolvedScope + return resolvedScope, nil + } +} + +type AttributeScopeResolveFunc func(ctx context.Context, orgID int64, initialScope string) (string, error) + +// getCacheKey creates an identifier to fetch and store resolution of scopes in the cache +func getCacheKey(orgID int64, scope string) string { + return fmt.Sprintf("%s-%v", scope, orgID) +} + +// GetResolveAttributeScopeMutator returns a function to resolve scopes with attributes such as `name` or `uid` into `id` based scopes +func (s *ScopeResolver) GetResolveAttributeScopeMutator(orgID int64) ScopeMutator { + return func(ctx context.Context, scope string) (string, error) { + // Check cache before computing the scope + if cachedScope, ok := s.cache.Get(getCacheKey(orgID, scope)); ok { + resolvedScope := cachedScope.(string) + s.log.Debug("used cache to resolve '%v' to '%v'", scope, resolvedScope) + return resolvedScope, nil + } + + var err error + // By default the scope remains unchanged + resolvedScope := scope + prefix := scopePrefix(scope) + if fn, ok := s.attributeResolvers[prefix]; ok { + resolvedScope, err = fn(ctx, orgID, scope) + if err != nil { + return "", fmt.Errorf("could not resolve %v: %w", scope, err) + } + // Cache result + s.cache.Set(getCacheKey(orgID, scope), resolvedScope, ttl) + s.log.Debug("resolved '%v' to '%v'", scope, resolvedScope) + } + return resolvedScope, nil + } +} + +func scopePrefix(scope string) string { + parts := strings.Split(scope, ":") + n := len(parts) - 1 + parts[n] = "" + return strings.Join(parts, ":") +} + +//Inject params into the evaluator's templated scopes. e.g. "settings:" + eval.Parameters(":id") +func ScopeInjector(params ScopeParams) ScopeMutator { + return func(_ context.Context, scope string) (string, error) { + tmpl, err := template.New("scope").Parse(scope) + if err != nil { + return "", err + } + var buf bytes.Buffer + if err = tmpl.Execute(&buf, params); err != nil { + return "", err + } + return buf.String(), nil } - return &permission, nil } diff --git a/pkg/services/accesscontrol/scope_test.go b/pkg/services/accesscontrol/scope_test.go index b03e1ed7ea7..a1bedaed3a6 100644 --- a/pkg/services/accesscontrol/scope_test.go +++ b/pkg/services/accesscontrol/scope_test.go @@ -1,6 +1,7 @@ package accesscontrol import ( + "context" "testing" "github.com/grafana/grafana/pkg/models" @@ -22,34 +23,156 @@ func TestResolveKeywordedScope(t *testing.T) { name string user *models.SignedInUser permission Permission - want *Permission + want Permission wantErr bool }{ { name: "no scope", user: testUser, permission: Permission{Action: "users:read"}, - want: &Permission{Action: "users:read"}, + want: Permission{Action: "users:read"}, wantErr: false, }, { name: "user if resolution", user: testUser, permission: Permission{Action: "users:read", Scope: "users:self"}, - want: &Permission{Action: "users:read", Scope: "users:id:2"}, + want: Permission{Action: "users:read", Scope: "users:id:2"}, wantErr: false, }, } for _, tt := range tests { + var err error t.Run(tt.name, func(t *testing.T) { resolver := NewScopeResolver() - resolved, err := resolver.ResolveKeyword(tt.user, tt.permission) + scopeModifier := resolver.GetResolveKeywordScopeMutator(tt.user) + tt.permission.Scope, err = scopeModifier(context.TODO(), tt.permission.Scope) if tt.wantErr { assert.Error(t, err, "expected an error during the resolution of the scope") return } assert.NoError(t, err) - assert.EqualValues(t, tt.want, resolved, "permission did not match expected resolution") + assert.EqualValues(t, tt.want, tt.permission, "permission did not match expected resolution") + }) + } +} + +func TestScopeResolver_ResolveAttribute(t *testing.T) { + // Calls allow us to see how many times the fakeDataSourceResolution has been called + calls := 0 + fakeDataSourceResolution := func(ctx context.Context, orgID int64, initialScope string) (string, error) { + calls++ + if initialScope == "datasources:name:testds" { + return Scope("datasources", "id", "1"), nil + } else if initialScope == "datasources:name:testds2" { + return Scope("datasources", "id", "2"), nil + } else { + return "", models.ErrDataSourceNotFound + } + } + + tests := []struct { + name string + orgID int64 + evaluator Evaluator + wantEvaluator Evaluator + wantCalls int + wantErr error + }{ + { + name: "should work with scope less permissions", + evaluator: EvalPermission("datasources:read"), + wantEvaluator: EvalPermission("datasources:read"), + wantCalls: 0, + }, + { + name: "should handle an error", + orgID: 1, + evaluator: EvalPermission("datasources:read", Scope("datasources", "name", "testds3")), + wantErr: models.ErrDataSourceNotFound, + wantCalls: 1, + }, + { + name: "should resolve a scope", + orgID: 1, + evaluator: EvalPermission("datasources:read", Scope("datasources", "name", "testds")), + wantEvaluator: EvalPermission("datasources:read", Scope("datasources", "id", "1")), + wantCalls: 1, + }, + { + name: "should resolve nested scopes with cache", + orgID: 1, + evaluator: EvalAll( + EvalPermission("datasources:read", Scope("datasources", "name", "testds")), + EvalAny( + EvalPermission("datasources:read", Scope("datasources", "name", "testds")), + EvalPermission("datasources:read", Scope("datasources", "name", "testds2")), + ), + ), + wantEvaluator: EvalAll( + EvalPermission("datasources:read", Scope("datasources", "id", "1")), + EvalAny( + EvalPermission("datasources:read", Scope("datasources", "id", "1")), + EvalPermission("datasources:read", Scope("datasources", "id", "2")), + ), + ), + wantCalls: 2, + }, + } + for _, tt := range tests { + resolver := NewScopeResolver() + + // Reset calls counter + calls = 0 + // Register a resolution method + resolver.AddAttributeResolver("datasources:name:", fakeDataSourceResolution) + + // Test + scopeModifier := resolver.GetResolveAttributeScopeMutator(tt.orgID) + resolvedEvaluator, err := tt.evaluator.MutateScopes(context.TODO(), scopeModifier) + if tt.wantErr != nil { + assert.ErrorAs(t, err, &tt.wantErr, "expected an error during the resolution of the scope") + return + } + assert.NoError(t, err) + assert.EqualValues(t, tt.wantEvaluator, resolvedEvaluator, "permission did not match expected resolution") + + assert.Equal(t, tt.wantCalls, calls, "cache has not been used") + } +} + +func Test_scopePrefix(t *testing.T) { + tests := []struct { + name string + scope string + want string + }{ + { + name: "empty", + scope: "", + want: "", + }, + { + name: "minimal", + scope: ":", + want: ":", + }, + { + name: "datasources", + scope: "datasources:", + want: "datasources:", + }, + { + name: "datasources name", + scope: "datasources:name:testds", + want: "datasources:name:", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefix := scopePrefix(tt.scope) + + assert.Equal(t, tt.want, prefix) }) } } diff --git a/pkg/services/alerting/conditions/query_interval_test.go b/pkg/services/alerting/conditions/query_interval_test.go index 88cebf3b4a4..2ed470c9906 100644 --- a/pkg/services/alerting/conditions/query_interval_test.go +++ b/pkg/services/alerting/conditions/query_interval_test.go @@ -176,7 +176,6 @@ func applyScenario(t *testing.T, timeRange string, dataSourceJsonData *simplejso }, verifier: verifier, } - _, err = condition.Eval(ctx.result, reqHandler) require.Nil(t, err) diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 51cefb8d72d..e89996d0b10 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -7,13 +7,12 @@ import ( "time" "github.com/benbjohnson/clock" - "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/ext" - tlog "github.com/opentracing/opentracing-go/log" + "go.opentelemetry.io/otel/attribute" "golang.org/x/sync/errgroup" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/encryption" @@ -40,6 +39,7 @@ type AlertEngine struct { log log.Logger resultHandler resultHandler usageStatsService usagestats.Service + tracer tracing.Tracer } // IsDisabled returns true if the alerting service is disabled for this instance. @@ -50,7 +50,7 @@ func (e *AlertEngine) IsDisabled() bool { // ProvideAlertEngine returns a new AlertEngine. func ProvideAlertEngine(renderer rendering.Service, bus bus.Bus, requestValidator models.PluginRequestValidator, dataService legacydata.RequestHandler, usageStatsService usagestats.Service, encryptionService encryption.Internal, - cfg *setting.Cfg) *AlertEngine { + cfg *setting.Cfg, tracer tracing.Tracer) *AlertEngine { e := &AlertEngine{ Cfg: cfg, RenderService: renderer, @@ -58,6 +58,7 @@ func ProvideAlertEngine(renderer rendering.Service, bus bus.Bus, requestValidato RequestValidator: requestValidator, DataService: dataService, usageStatsService: usageStatsService, + tracer: tracer, } e.ticker = NewTicker(time.Now(), time.Second*0, clock.New(), 1) e.execQueue = make(chan *Job, 1000) @@ -177,9 +178,7 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan alertCtx, cancelFn := context.WithTimeout(context.Background(), setting.AlertingEvaluationTimeout) cancelChan <- cancelFn - span := opentracing.StartSpan("alert execution") - alertCtx = opentracing.ContextWithSpan(alertCtx, span) - + alertCtx, span := e.tracer.Start(alertCtx, "alert execution") evalContext := NewEvalContext(alertCtx, job.Rule, e.RequestValidator) evalContext.Ctx = alertCtx @@ -187,32 +186,37 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan defer func() { if err := recover(); err != nil { e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1)) - ext.Error.Set(span, true) - span.LogFields( - tlog.Error(fmt.Errorf("%v", err)), - tlog.String("message", "failed to execute alert rule. panic was recovered."), - ) - span.Finish() + span.RecordError(fmt.Errorf("%v", err)) + span.AddEvents( + []string{"error", "message"}, + []tracing.EventValue{ + {Str: fmt.Sprintf("%v", err)}, + {Str: "failed to execute alert rule. panic was recovered."}, + }) + span.End() close(attemptChan) } }() e.evalHandler.Eval(evalContext) - span.SetTag("alertId", evalContext.Rule.ID) - span.SetTag("dashboardId", evalContext.Rule.DashboardID) - span.SetTag("firing", evalContext.Firing) - span.SetTag("nodatapoints", evalContext.NoDataFound) - span.SetTag("attemptID", attemptID) + span.SetAttributes("alertId", evalContext.Rule.ID, attribute.Key("alertId").Int64(evalContext.Rule.ID)) + span.SetAttributes("dashboardId", evalContext.Rule.DashboardID, attribute.Key("dashboardId").Int64(evalContext.Rule.DashboardID)) + span.SetAttributes("firing", evalContext.Firing, attribute.Key("firing").Bool(evalContext.Firing)) + span.SetAttributes("nodatapoints", evalContext.NoDataFound, attribute.Key("nodatapoints").Bool(evalContext.NoDataFound)) + span.SetAttributes("attemptID", attemptID, attribute.Key("attemptID").Int(attemptID)) if evalContext.Error != nil { - ext.Error.Set(span, true) - span.LogFields( - tlog.Error(evalContext.Error), - tlog.String("message", "alerting execution attempt failed"), - ) + span.RecordError(evalContext.Error) + span.AddEvents( + []string{"error", "message"}, + []tracing.EventValue{ + {Str: fmt.Sprintf("%v", evalContext.Error)}, + {Str: "alerting execution attempt failed"}, + }) + if attemptID < setting.AlertingMaxAttempts { - span.Finish() + span.End() e.log.Debug("Job Execution attempt triggered retry", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) attemptChan <- (attemptID + 1) return @@ -240,7 +244,7 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan } } - span.Finish() + span.End() e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.ID, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) close(attemptChan) }() diff --git a/pkg/services/alerting/engine_integration_test.go b/pkg/services/alerting/engine_integration_test.go index cbf9d9e35dd..29422ea5dc3 100644 --- a/pkg/services/alerting/engine_integration_test.go +++ b/pkg/services/alerting/engine_integration_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/services/encryption/ossencryption" "github.com/grafana/grafana/pkg/setting" @@ -21,7 +22,9 @@ import ( func TestEngineTimeouts(t *testing.T) { usMock := &usagestats.UsageStatsMock{T: t} - engine := ProvideAlertEngine(nil, nil, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg()) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + engine := ProvideAlertEngine(nil, nil, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg(), tracer) setting.AlertingNotificationTimeout = 30 * time.Second setting.AlertingMaxAttempts = 3 engine.resultHandler = &FakeResultHandler{} diff --git a/pkg/services/alerting/engine_test.go b/pkg/services/alerting/engine_test.go index 73ac09dc52b..da21e2dcdda 100644 --- a/pkg/services/alerting/engine_test.go +++ b/pkg/services/alerting/engine_test.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/encryption/ossencryption" @@ -45,7 +46,9 @@ func (handler *FakeResultHandler) handle(evalContext *EvalContext) error { func TestEngineProcessJob(t *testing.T) { bus := bus.New() usMock := &usagestats.UsageStatsMock{T: t} - engine := ProvideAlertEngine(nil, bus, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg()) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + engine := ProvideAlertEngine(nil, bus, nil, nil, usMock, ossencryption.ProvideService(), setting.NewCfg(), tracer) setting.AlertingEvaluationTimeout = 30 * time.Second setting.AlertingNotificationTimeout = 30 * time.Second setting.AlertingMaxAttempts = 3 diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 8874261fd1b..98e0b135822 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -200,11 +200,15 @@ func (n *notificationService) renderAndUploadImage(evalCtx *EvalContext, timeout } renderOpts := rendering.Opts{ + TimeoutOpts: rendering.TimeoutOpts{ + Timeout: timeout, + }, + AuthOpts: rendering.AuthOpts{ + OrgID: evalCtx.Rule.OrgID, + OrgRole: models.ROLE_ADMIN, + }, Width: 1000, Height: 500, - Timeout: timeout, - OrgID: evalCtx.Rule.OrgID, - OrgRole: models.ROLE_ADMIN, ConcurrentLimit: setting.AlertingRenderLimit, Theme: rendering.ThemeDark, } @@ -218,7 +222,7 @@ func (n *notificationService) renderAndUploadImage(evalCtx *EvalContext, timeout n.log.Debug("Rendering alert panel image", "ruleId", evalCtx.Rule.ID, "urlPath", renderOpts.Path) start := time.Now() - result, err := n.renderService.Render(evalCtx.Ctx, renderOpts) + result, err := n.renderService.Render(evalCtx.Ctx, renderOpts, nil) if err != nil { return err } diff --git a/pkg/services/alerting/notifier_test.go b/pkg/services/alerting/notifier_test.go index 4b158351d05..a1732eb89b1 100644 --- a/pkg/services/alerting/notifier_test.go +++ b/pkg/services/alerting/notifier_test.go @@ -345,6 +345,10 @@ type testRenderService struct { renderErrorImageProvider func(error error) (*rendering.RenderResult, error) } +func (s *testRenderService) HasCapability(feature rendering.CapabilityName) (rendering.CapabilitySupportRequestResult, error) { + return rendering.CapabilitySupportRequestResult{}, nil +} + func (s *testRenderService) IsAvailable() bool { if s.isAvailableProvider != nil { return s.isAvailableProvider() @@ -353,7 +357,7 @@ func (s *testRenderService) IsAvailable() bool { return true } -func (s *testRenderService) Render(ctx context.Context, opts rendering.Opts) (*rendering.RenderResult, error) { +func (s *testRenderService) Render(ctx context.Context, opts rendering.Opts, session rendering.Session) (*rendering.RenderResult, error) { if s.renderProvider != nil { return s.renderProvider(ctx, opts) } @@ -361,7 +365,7 @@ func (s *testRenderService) Render(ctx context.Context, opts rendering.Opts) (*r return &rendering.RenderResult{FilePath: "image.png"}, nil } -func (s *testRenderService) RenderCSV(ctx context.Context, opts rendering.CSVOpts) (*rendering.RenderCSVResult, error) { +func (s *testRenderService) RenderCSV(ctx context.Context, opts rendering.CSVOpts, session rendering.Session) (*rendering.RenderCSVResult, error) { return nil, nil } @@ -381,6 +385,10 @@ func (s *testRenderService) Version() string { return "" } +func (s *testRenderService) CreateRenderingSession(ctx context.Context, authOpts rendering.AuthOpts, sessionOpts rendering.SessionOpts) (rendering.Session, error) { + return nil, nil +} + var _ rendering.Service = &testRenderService{} type testImageUploader struct { diff --git a/pkg/services/contexthandler/auth_jwt.go b/pkg/services/contexthandler/auth_jwt.go index 7577b23c80a..748bd13c120 100644 --- a/pkg/services/contexthandler/auth_jwt.go +++ b/pkg/services/contexthandler/auth_jwt.go @@ -9,6 +9,7 @@ import ( ) const InvalidJWT = "Invalid JWT" +const UserNotFound = "User not found" func (h *ContextHandler) initContextWithJWT(ctx *models.ReqContext, orgId int64) bool { if !h.Cfg.JWTAuthEnabled || h.Cfg.JWTAuthHeaderName == "" { @@ -29,11 +30,29 @@ func (h *ContextHandler) initContextWithJWT(ctx *models.ReqContext, orgId int64) query := models.GetSignedInUserQuery{OrgId: orgId} + sub, _ := claims["sub"].(string) + + if sub == "" { + ctx.Logger.Warn("Got a JWT without the mandatory 'sub' claim", "error", err) + ctx.JsonApiErr(401, InvalidJWT, err) + return true + } + extUser := &models.ExternalUserInfo{ + AuthModule: "jwt", + AuthId: sub, + } + if key := h.Cfg.JWTAuthUsernameClaim; key != "" { query.Login, _ = claims[key].(string) + extUser.Login, _ = claims[key].(string) } if key := h.Cfg.JWTAuthEmailClaim; key != "" { query.Email, _ = claims[key].(string) + extUser.Email, _ = claims[key].(string) + } + + if name, _ := claims["name"].(string); name != "" { + extUser.Name = name } if query.Login == "" && query.Email == "" { @@ -42,6 +61,18 @@ func (h *ContextHandler) initContextWithJWT(ctx *models.ReqContext, orgId int64) return true } + if h.Cfg.JWTAuthAutoSignUp { + upsert := &models.UpsertUserCommand{ + ReqContext: ctx, + SignupAllowed: h.Cfg.JWTAuthAutoSignUp, + ExternalUser: extUser, + } + if err := bus.Dispatch(ctx.Req.Context(), upsert); err != nil { + ctx.Logger.Error("Failed to upsert JWT user", "error", err) + return false + } + } + if err := bus.Dispatch(ctx.Req.Context(), &query); err != nil { if errors.Is(err, models.ErrUserNotFound) { ctx.Logger.Debug( @@ -50,10 +81,11 @@ func (h *ContextHandler) initContextWithJWT(ctx *models.ReqContext, orgId int64) "username_claim", query.Login, ) err = login.ErrInvalidCredentials + ctx.JsonApiErr(401, UserNotFound, err) } else { ctx.Logger.Error("Failed to get signed in user", "error", err) + ctx.JsonApiErr(401, InvalidJWT, err) } - ctx.JsonApiErr(401, InvalidJWT, err) return true } diff --git a/pkg/services/contexthandler/auth_proxy_test.go b/pkg/services/contexthandler/auth_proxy_test.go index 712bcdac492..5eac50fce84 100644 --- a/pkg/services/contexthandler/auth_proxy_test.go +++ b/pkg/services/contexthandler/auth_proxy_test.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/remotecache" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/contexthandler/authproxy" @@ -102,6 +103,8 @@ func getContextHandler(t *testing.T) *ContextHandler { userAuthTokenSvc := auth.NewFakeUserAuthTokenService() renderSvc := &fakeRenderService{} authJWTSvc := models.NewFakeJWTService() + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) - return ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore) + return ProvideService(cfg, userAuthTokenSvc, authJWTSvc, remoteCacheSvc, renderSvc, sqlStore, tracer) } diff --git a/pkg/services/contexthandler/contexthandler.go b/pkg/services/contexthandler/contexthandler.go index 12659d75688..71757ab77d8 100644 --- a/pkg/services/contexthandler/contexthandler.go +++ b/pkg/services/contexthandler/contexthandler.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/network" "github.com/grafana/grafana/pkg/infra/remotecache" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/middleware/cookies" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/contexthandler/authproxy" @@ -23,8 +24,6 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" - "github.com/opentracing/opentracing-go" - ol "github.com/opentracing/opentracing-go/log" cw "github.com/weaveworks/common/tracing" ) @@ -36,7 +35,8 @@ const ( const ServiceName = "ContextHandler" func ProvideService(cfg *setting.Cfg, tokenService models.UserTokenService, jwtService models.JWTService, - remoteCache *remotecache.RemoteCache, renderService rendering.Service, sqlStore *sqlstore.SQLStore) *ContextHandler { + remoteCache *remotecache.RemoteCache, renderService rendering.Service, sqlStore *sqlstore.SQLStore, + tracer tracing.Tracer) *ContextHandler { return &ContextHandler{ Cfg: cfg, AuthTokenService: tokenService, @@ -44,6 +44,7 @@ func ProvideService(cfg *setting.Cfg, tokenService models.UserTokenService, jwtS RemoteCache: remoteCache, RenderService: renderService, SQLStore: sqlStore, + tracer: tracer, } } @@ -55,7 +56,7 @@ type ContextHandler struct { RemoteCache *remotecache.RemoteCache RenderService rendering.Service SQLStore *sqlstore.SQLStore - + tracer tracing.Tracer // GetTime returns the current time. // Stubbable by tests. GetTime func() time.Time @@ -73,8 +74,8 @@ func FromContext(c context.Context) *models.ReqContext { // Middleware provides a middleware to initialize the Macaron context. func (h *ContextHandler) Middleware(mContext *web.Context) { - span, _ := opentracing.StartSpanFromContext(mContext.Req.Context(), "Auth - Middleware") - defer span.Finish() + _, span := h.tracer.Start(mContext.Req.Context(), "Auth - Middleware") + defer span.End() reqContext := &models.ReqContext{ Context: mContext, @@ -135,11 +136,13 @@ func (h *ContextHandler) Middleware(mContext *web.Context) { } reqContext.Logger = log.New("context", "userId", reqContext.UserId, "orgId", reqContext.OrgId, "uname", reqContext.Login) - - span.LogFields( - ol.String("uname", reqContext.Login), - ol.Int64("orgId", reqContext.OrgId), - ol.Int64("userId", reqContext.UserId)) + span.AddEvents( + []string{"uname", "orgId", "userId"}, + []tracing.EventValue{ + {Str: reqContext.Login}, + {Num: reqContext.OrgId}, + {Num: reqContext.UserId}}, + ) mContext.Map(reqContext) @@ -157,8 +160,8 @@ func (h *ContextHandler) initContextWithAnonymousUser(reqContext *models.ReqCont return false } - span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithAnonymousUser") - defer span.Finish() + _, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithAnonymousUser") + defer span.End() org, err := h.SQLStore.GetOrgByName(h.Cfg.AnonymousOrgName) if err != nil { @@ -192,8 +195,8 @@ func (h *ContextHandler) initContextWithAPIKey(reqContext *models.ReqContext) bo return false } - span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithAPIKey") - defer span.Finish() + _, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithAPIKey") + defer span.End() // base64 decode key decoded, err := apikeygen.Decode(keyString) @@ -272,8 +275,8 @@ func (h *ContextHandler) initContextWithBasicAuth(reqContext *models.ReqContext, return false } - span, ctx := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithBasicAuth") - defer span.Finish() + ctx, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithBasicAuth") + defer span.End() username, password, err := util.DecodeBasicAuthHeader(header) if err != nil { @@ -328,8 +331,8 @@ func (h *ContextHandler) initContextWithToken(reqContext *models.ReqContext, org return false } - span, ctx := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithToken") - defer span.Finish() + ctx, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithToken") + defer span.End() token, err := h.AuthTokenService.LookupToken(ctx, rawToken) if err != nil { @@ -369,8 +372,8 @@ func (h *ContextHandler) rotateEndOfRequestFunc(reqContext *models.ReqContext, a return } - span, ctx := opentracing.StartSpanFromContext(reqContext.Req.Context(), "rotateEndOfRequestFunc") - defer span.Finish() + ctx, span := h.tracer.Start(reqContext.Req.Context(), "rotateEndOfRequestFunc") + defer span.End() addr := reqContext.RemoteAddr() ip, err := network.GetIPFromAddress(addr) @@ -396,8 +399,8 @@ func (h *ContextHandler) initContextWithRenderAuth(reqContext *models.ReqContext return false } - span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithRenderAuth") - defer span.Finish() + _, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithRenderAuth") + defer span.End() renderUser, exists := h.RenderService.GetRenderUser(reqContext.Req.Context(), key) if !exists { @@ -466,8 +469,8 @@ func (h *ContextHandler) initContextWithAuthProxy(reqContext *models.ReqContext, return false } - span, _ := opentracing.StartSpanFromContext(reqContext.Req.Context(), "initContextWithAuthProxy") - defer span.Finish() + _, span := h.tracer.Start(reqContext.Req.Context(), "initContextWithAuthProxy") + defer span.End() // Check if allowed to continue with this IP if err := auth.IsAllowedIP(); err != nil { diff --git a/pkg/services/dashboardimport/api/api.go b/pkg/services/dashboardimport/api/api.go new file mode 100644 index 00000000000..9a9249bae33 --- /dev/null +++ b/pkg/services/dashboardimport/api/api.go @@ -0,0 +1,89 @@ +package api + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/api/apierrors" + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/middleware" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/dashboardimport" + "github.com/grafana/grafana/pkg/web" +) + +type ImportDashboardAPI struct { + dashboardImportService dashboardimport.Service + quotaService QuotaService + schemaLoaderService SchemaLoaderService + pluginStore plugins.Store +} + +func New(dashboardImportService dashboardimport.Service, quotaService QuotaService, + schemaLoaderService SchemaLoaderService, pluginStore plugins.Store) *ImportDashboardAPI { + return &ImportDashboardAPI{ + dashboardImportService: dashboardImportService, + quotaService: quotaService, + schemaLoaderService: schemaLoaderService, + pluginStore: pluginStore, + } +} + +func (api *ImportDashboardAPI) RegisterAPIEndpoints(routeRegister routing.RouteRegister) { + routeRegister.Group("/api/dashboards", func(route routing.RouteRegister) { + route.Post("/import", routing.Wrap(api.ImportDashboard)) + }, middleware.ReqSignedIn) +} + +func (api *ImportDashboardAPI) ImportDashboard(c *models.ReqContext) response.Response { + req := dashboardimport.ImportDashboardRequest{} + if err := web.Bind(c.Req, &req); err != nil { + return response.Error(http.StatusBadRequest, "bad request data", err) + } + + if req.PluginId == "" && req.Dashboard == nil { + return response.Error(http.StatusUnprocessableEntity, "Dashboard must be set", nil) + } + + limitReached, err := api.quotaService.QuotaReached(c, "dashboard") + if err != nil { + return response.Error(500, "failed to get quota", err) + } + + if limitReached { + return response.Error(403, "Quota reached", nil) + } + + trimDefaults := c.QueryBoolWithDefault("trimdefaults", true) + if trimDefaults && !api.schemaLoaderService.IsDisabled() { + req.Dashboard, err = api.schemaLoaderService.DashboardApplyDefaults(req.Dashboard) + if err != nil { + return response.Error(http.StatusInternalServerError, "Error while applying default value to the dashboard json", err) + } + } + + req.User = c.SignedInUser + resp, err := api.dashboardImportService.ImportDashboard(c.Req.Context(), &req) + if err != nil { + return apierrors.ToDashboardErrorResponse(c.Req.Context(), api.pluginStore, err) + } + + return response.JSON(http.StatusOK, resp) +} + +type QuotaService interface { + QuotaReached(c *models.ReqContext, target string) (bool, error) +} + +type quotaServiceFunc func(c *models.ReqContext, target string) (bool, error) + +func (fn quotaServiceFunc) QuotaReached(c *models.ReqContext, target string) (bool, error) { + return fn(c, target) +} + +type SchemaLoaderService interface { + IsDisabled() bool + DashboardApplyDefaults(input *simplejson.Json) (*simplejson.Json, error) +} diff --git a/pkg/services/dashboardimport/api/api_test.go b/pkg/services/dashboardimport/api/api_test.go new file mode 100644 index 00000000000..29b24197933 --- /dev/null +++ b/pkg/services/dashboardimport/api/api_test.go @@ -0,0 +1,209 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/dashboardimport" + "github.com/grafana/grafana/pkg/web/webtest" + "github.com/stretchr/testify/require" +) + +func TestImportDashboardAPI(t *testing.T) { + t.Run("Quota not reached, schema loader service disabled", func(t *testing.T) { + importDashboardServiceCalled := false + service := &serviceMock{ + importDashboardFunc: func(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) { + importDashboardServiceCalled = true + return nil, nil + }, + } + + schemaLoaderServiceCalled := false + schemaLoaderService := &schemaLoaderServiceMock{ + dashboardApplyDefaultsFunc: func(input *simplejson.Json) (*simplejson.Json, error) { + schemaLoaderServiceCalled = true + return input, nil + }, + } + + importDashboardAPI := New(service, quotaServiceFunc(quotaNotReached), schemaLoaderService, nil) + routeRegister := routing.NewRouteRegister() + importDashboardAPI.RegisterAPIEndpoints(routeRegister) + s := webtest.NewServer(t, routeRegister) + + t.Run("Not signed in should return 404", func(t *testing.T) { + cmd := &dashboardimport.ImportDashboardRequest{} + jsonBytes, err := json.Marshal(cmd) + require.NoError(t, err) + req := s.NewRequest(http.MethodPost, "/api/dashboards/import", bytes.NewReader(jsonBytes)) + resp, err := s.Send(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + }) + + t.Run("Signed in, empty plugin id and dashboard model empty should return error", func(t *testing.T) { + cmd := &dashboardimport.ImportDashboardRequest{ + PluginId: "", + Dashboard: nil, + } + jsonBytes, err := json.Marshal(cmd) + require.NoError(t, err) + req := s.NewRequest(http.MethodPost, "/api/dashboards/import", bytes.NewReader(jsonBytes)) + webtest.RequestWithSignedInUser(req, &models.SignedInUser{ + UserId: 1, + }) + resp, err := s.Send(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) + }) + + t.Run("Signed in, dashboard model set should call import dashboard service", func(t *testing.T) { + cmd := &dashboardimport.ImportDashboardRequest{ + Dashboard: simplejson.New(), + } + jsonBytes, err := json.Marshal(cmd) + require.NoError(t, err) + req := s.NewRequest(http.MethodPost, "/api/dashboards/import", bytes.NewReader(jsonBytes)) + webtest.RequestWithSignedInUser(req, &models.SignedInUser{ + UserId: 1, + }) + resp, err := s.Send(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.True(t, importDashboardServiceCalled) + }) + + t.Run("Signed in, dashboard model set, trimdefaults enabled should not call schema loader service", func(t *testing.T) { + cmd := &dashboardimport.ImportDashboardRequest{ + Dashboard: simplejson.New(), + } + jsonBytes, err := json.Marshal(cmd) + require.NoError(t, err) + req := s.NewRequest(http.MethodPost, "/api/dashboards/import?trimdefaults=true", bytes.NewReader(jsonBytes)) + webtest.RequestWithSignedInUser(req, &models.SignedInUser{ + UserId: 1, + }) + resp, err := s.Send(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.False(t, schemaLoaderServiceCalled) + require.True(t, importDashboardServiceCalled) + }) + }) + + t.Run("Quota not reached, schema loader service enabled", func(t *testing.T) { + importDashboardServiceCalled := false + service := &serviceMock{ + importDashboardFunc: func(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) { + importDashboardServiceCalled = true + return nil, nil + }, + } + + schemaLoaderServiceCalled := false + schemaLoaderService := &schemaLoaderServiceMock{ + enabled: true, + dashboardApplyDefaultsFunc: func(input *simplejson.Json) (*simplejson.Json, error) { + schemaLoaderServiceCalled = true + return input, nil + }, + } + + importDashboardAPI := New(service, quotaServiceFunc(quotaNotReached), schemaLoaderService, nil) + routeRegister := routing.NewRouteRegister() + importDashboardAPI.RegisterAPIEndpoints(routeRegister) + s := webtest.NewServer(t, routeRegister) + + t.Run("Signed in, dashboard model set, trimdefaults enabled should call schema loader service", func(t *testing.T) { + cmd := &dashboardimport.ImportDashboardRequest{ + Dashboard: simplejson.New(), + } + jsonBytes, err := json.Marshal(cmd) + require.NoError(t, err) + req := s.NewRequest(http.MethodPost, "/api/dashboards/import?trimdefaults=true", bytes.NewReader(jsonBytes)) + webtest.RequestWithSignedInUser(req, &models.SignedInUser{ + UserId: 1, + }) + resp, err := s.Send(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.True(t, schemaLoaderServiceCalled) + require.True(t, importDashboardServiceCalled) + }) + }) + + t.Run("Quota reached", func(t *testing.T) { + service := &serviceMock{} + schemaLoaderService := &schemaLoaderServiceMock{} + importDashboardAPI := New(service, quotaServiceFunc(quotaReached), schemaLoaderService, nil) + + routeRegister := routing.NewRouteRegister() + importDashboardAPI.RegisterAPIEndpoints(routeRegister) + s := webtest.NewServer(t, routeRegister) + + t.Run("Signed in, dashboard model set, should return 403 forbidden/quota reached", func(t *testing.T) { + cmd := &dashboardimport.ImportDashboardRequest{ + Dashboard: simplejson.New(), + } + jsonBytes, err := json.Marshal(cmd) + require.NoError(t, err) + req := s.NewRequest(http.MethodPost, "/api/dashboards/import", bytes.NewReader(jsonBytes)) + webtest.RequestWithSignedInUser(req, &models.SignedInUser{ + UserId: 1, + }) + resp, err := s.Send(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusForbidden, resp.StatusCode) + }) + }) +} + +type serviceMock struct { + importDashboardFunc func(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) +} + +func (s *serviceMock) ImportDashboard(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) { + if s.importDashboardFunc != nil { + return s.importDashboardFunc(ctx, req) + } + + return nil, nil +} + +type schemaLoaderServiceMock struct { + enabled bool + dashboardApplyDefaultsFunc func(input *simplejson.Json) (*simplejson.Json, error) +} + +func (s *schemaLoaderServiceMock) IsDisabled() bool { + return !s.enabled +} + +func (s *schemaLoaderServiceMock) DashboardApplyDefaults(input *simplejson.Json) (*simplejson.Json, error) { + if s.dashboardApplyDefaultsFunc != nil { + return s.dashboardApplyDefaultsFunc(input) + } + + return input, nil +} + +func quotaReached(c *models.ReqContext, target string) (bool, error) { + return true, nil +} + +func quotaNotReached(c *models.ReqContext, target string) (bool, error) { + return false, nil +} diff --git a/pkg/services/dashboardimport/dashboardimport.go b/pkg/services/dashboardimport/dashboardimport.go new file mode 100644 index 00000000000..86d80064a15 --- /dev/null +++ b/pkg/services/dashboardimport/dashboardimport.go @@ -0,0 +1,52 @@ +package dashboardimport + +import ( + "context" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" +) + +// ImportDashboardInput definition of input parameters when importing a dashboard. +type ImportDashboardInput struct { + Type string `json:"type"` + PluginId string `json:"pluginId"` + Name string `json:"name"` + Value string `json:"value"` +} + +// ImportDashboardRequest request object for importing a dashboard. +type ImportDashboardRequest struct { + PluginId string `json:"pluginId"` + Path string `json:"path"` + Overwrite bool `json:"overwrite"` + Dashboard *simplejson.Json `json:"dashboard"` + Inputs []ImportDashboardInput `json:"inputs"` + FolderId int64 `json:"folderId"` + FolderUid string `json:"folderUid"` + + User *models.SignedInUser `json:"-"` +} + +// ImportDashboardResponse response object returned when importing a dashboard. +type ImportDashboardResponse struct { + UID string `json:"uid"` + PluginId string `json:"pluginId"` + Title string `json:"title"` + Imported bool `json:"imported"` + ImportedUri string `json:"importedUri"` + ImportedUrl string `json:"importedUrl"` + Slug string `json:"slug"` + DashboardId int64 `json:"dashboardId"` + FolderId int64 `json:"folderId"` + ImportedRevision int64 `json:"importedRevision"` + Revision int64 `json:"revision"` + Description string `json:"description"` + Path string `json:"path"` + Removed bool `json:"removed"` +} + +// Service service interface for importing dashboards. +type Service interface { + ImportDashboard(ctx context.Context, req *ImportDashboardRequest) (*ImportDashboardResponse, error) +} diff --git a/pkg/services/dashboardimport/service/service.go b/pkg/services/dashboardimport/service/service.go new file mode 100644 index 00000000000..1fcfeb3d836 --- /dev/null +++ b/pkg/services/dashboardimport/service/service.go @@ -0,0 +1,103 @@ +package service + +import ( + "context" + + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/dashboardimport" + "github.com/grafana/grafana/pkg/services/dashboardimport/api" + "github.com/grafana/grafana/pkg/services/dashboardimport/utils" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/librarypanels" + "github.com/grafana/grafana/pkg/services/quota" + "github.com/grafana/grafana/pkg/services/schemaloader" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func ProvideService(sqlStore *sqlstore.SQLStore, routeRegister routing.RouteRegister, + quotaService *quota.QuotaService, schemaLoaderService *schemaloader.SchemaLoaderService, + pluginDashboardManager plugins.PluginDashboardManager, pluginStore plugins.Store, + libraryPanelService librarypanels.Service) *ImportDashboardService { + s := &ImportDashboardService{ + pluginDashboardManager: pluginDashboardManager, + dashboardService: dashboards.NewService(sqlStore), + libraryPanelService: libraryPanelService, + } + + dashboardImportAPI := api.New(s, quotaService, schemaLoaderService, pluginStore) + dashboardImportAPI.RegisterAPIEndpoints(routeRegister) + + return s +} + +type ImportDashboardService struct { + pluginDashboardManager plugins.PluginDashboardManager + dashboardService dashboards.DashboardService + libraryPanelService librarypanels.Service +} + +func (s *ImportDashboardService) ImportDashboard(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) { + var dashboard *models.Dashboard + if req.PluginId != "" { + var err error + if dashboard, err = s.pluginDashboardManager.LoadPluginDashboard(ctx, req.PluginId, req.Path); err != nil { + return nil, err + } + } else { + dashboard = models.NewDashboardFromJson(req.Dashboard) + } + + evaluator := utils.NewDashTemplateEvaluator(dashboard.Data, req.Inputs) + generatedDash, err := evaluator.Eval() + if err != nil { + return nil, err + } + + saveCmd := models.SaveDashboardCommand{ + Dashboard: generatedDash, + OrgId: req.User.OrgId, + UserId: req.User.UserId, + Overwrite: req.Overwrite, + PluginId: req.PluginId, + FolderId: req.FolderId, + } + + dto := &dashboards.SaveDashboardDTO{ + OrgId: saveCmd.OrgId, + Dashboard: saveCmd.GetDashboardModel(), + Overwrite: saveCmd.Overwrite, + User: req.User, + } + + savedDash, err := s.dashboardService.ImportDashboard(ctx, dto) + if err != nil { + return nil, err + } + + err = s.libraryPanelService.ImportLibraryPanelsForDashboard(ctx, req.User, savedDash, req.FolderId) + if err != nil { + return nil, err + } + + err = s.libraryPanelService.ConnectLibraryPanelsForDashboard(ctx, req.User, dashboard) + if err != nil { + return nil, err + } + + return &dashboardimport.ImportDashboardResponse{ + UID: savedDash.Uid, + PluginId: req.PluginId, + Title: savedDash.Title, + Path: req.Path, + Revision: savedDash.Data.Get("revision").MustInt64(1), + FolderId: savedDash.FolderId, + ImportedUri: "db/" + savedDash.Slug, + ImportedUrl: savedDash.GetUrl(), + ImportedRevision: dashboard.Data.Get("revision").MustInt64(1), + Imported: true, + DashboardId: savedDash.Id, + Slug: savedDash.Slug, + }, nil +} diff --git a/pkg/services/dashboardimport/service/service_test.go b/pkg/services/dashboardimport/service/service_test.go new file mode 100644 index 00000000000..e72a80e682a --- /dev/null +++ b/pkg/services/dashboardimport/service/service_test.go @@ -0,0 +1,201 @@ +package service + +import ( + "context" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/dashboardimport" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/librarypanels" + "github.com/stretchr/testify/require" +) + +func TestImportDashboardService(t *testing.T) { + t.Run("When importing a plugin dashboard should save dashboard and sync library panels", func(t *testing.T) { + pluginDashboardManager := &pluginDashboardManagerMock{ + loadPluginDashboardFunc: loadTestDashboard, + } + + var importDashboardArg *dashboards.SaveDashboardDTO + dashboardService := &dashboardServiceMock{ + importDashboardFunc: func(ctx context.Context, dto *dashboards.SaveDashboardDTO) (*models.Dashboard, error) { + importDashboardArg = dto + return &models.Dashboard{ + Id: 4, + Uid: dto.Dashboard.Uid, + Slug: dto.Dashboard.Slug, + OrgId: 3, + Version: dto.Dashboard.Version, + PluginId: "prometheus", + FolderId: dto.Dashboard.FolderId, + Title: dto.Dashboard.Title, + Data: dto.Dashboard.Data, + }, nil + }, + } + + importLibraryPanelsForDashboard := false + connectLibraryPanelsForDashboardCalled := false + libraryPanelService := &libraryPanelServiceMock{ + importLibraryPanelsForDashboardFunc: func(ctx context.Context, signedInUser *models.SignedInUser, dash *models.Dashboard, folderID int64) error { + importLibraryPanelsForDashboard = true + return nil + }, + connectLibraryPanelsForDashboardFunc: func(ctx context.Context, signedInUser *models.SignedInUser, dash *models.Dashboard) error { + connectLibraryPanelsForDashboardCalled = true + return nil + }, + } + s := &ImportDashboardService{ + pluginDashboardManager: pluginDashboardManager, + dashboardService: dashboardService, + libraryPanelService: libraryPanelService, + } + + req := &dashboardimport.ImportDashboardRequest{ + PluginId: "prometheus", + Path: "dashboard.json", + Inputs: []dashboardimport.ImportDashboardInput{ + {Name: "*", Type: "datasource", Value: "prom"}, + }, + User: &models.SignedInUser{UserId: 2, OrgRole: models.ROLE_ADMIN, OrgId: 3}, + FolderId: 5, + } + resp, err := s.ImportDashboard(context.Background(), req) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, "UDdpyzz7z", resp.UID) + + require.NotNil(t, importDashboardArg) + require.Equal(t, int64(3), importDashboardArg.OrgId) + require.Equal(t, int64(2), importDashboardArg.User.UserId) + require.Equal(t, "prometheus", importDashboardArg.Dashboard.PluginId) + require.Equal(t, int64(5), importDashboardArg.Dashboard.FolderId) + + panel := importDashboardArg.Dashboard.Data.Get("panels").GetIndex(0) + require.Equal(t, "prom", panel.Get("datasource").MustString()) + + require.True(t, importLibraryPanelsForDashboard) + require.True(t, connectLibraryPanelsForDashboardCalled) + }) + + t.Run("When importing a non-plugin dashboard should save dashboard and sync library panels", func(t *testing.T) { + var importDashboardArg *dashboards.SaveDashboardDTO + dashboardService := &dashboardServiceMock{ + importDashboardFunc: func(ctx context.Context, dto *dashboards.SaveDashboardDTO) (*models.Dashboard, error) { + importDashboardArg = dto + return &models.Dashboard{ + Id: 4, + Uid: dto.Dashboard.Uid, + Slug: dto.Dashboard.Slug, + OrgId: 3, + Version: dto.Dashboard.Version, + PluginId: "prometheus", + FolderId: dto.Dashboard.FolderId, + Title: dto.Dashboard.Title, + Data: dto.Dashboard.Data, + }, nil + }, + } + libraryPanelService := &libraryPanelServiceMock{} + s := &ImportDashboardService{ + dashboardService: dashboardService, + libraryPanelService: libraryPanelService, + } + + dash, err := loadTestDashboard(context.Background(), "", "dashboard.json") + require.NoError(t, err) + + req := &dashboardimport.ImportDashboardRequest{ + Dashboard: dash.Data, + Path: "plugin_dashboard.json", + Inputs: []dashboardimport.ImportDashboardInput{ + {Name: "*", Type: "datasource", Value: "prom"}, + }, + User: &models.SignedInUser{UserId: 2, OrgRole: models.ROLE_ADMIN, OrgId: 3}, + FolderId: 5, + } + resp, err := s.ImportDashboard(context.Background(), req) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, "UDdpyzz7z", resp.UID) + + require.NotNil(t, importDashboardArg) + require.Equal(t, int64(3), importDashboardArg.OrgId) + require.Equal(t, int64(2), importDashboardArg.User.UserId) + require.Equal(t, "", importDashboardArg.Dashboard.PluginId) + require.Equal(t, int64(5), importDashboardArg.Dashboard.FolderId) + + panel := importDashboardArg.Dashboard.Data.Get("panels").GetIndex(0) + require.Equal(t, "prom", panel.Get("datasource").MustString()) + }) +} + +func loadTestDashboard(ctx context.Context, pluginID, path string) (*models.Dashboard, error) { + // It's safe to ignore gosec warning G304 since this is a test and arguments comes from test configuration. + // nolint:gosec + bytes, err := ioutil.ReadFile(filepath.Join("testdata", path)) + if err != nil { + return nil, err + } + + dashboardJSON, err := simplejson.NewJson(bytes) + if err != nil { + return nil, err + } + + return models.NewDashboardFromJson(dashboardJSON), nil +} + +type pluginDashboardManagerMock struct { + plugins.PluginDashboardManager + loadPluginDashboardFunc func(ctx context.Context, pluginID, path string) (*models.Dashboard, error) +} + +func (m *pluginDashboardManagerMock) LoadPluginDashboard(ctx context.Context, pluginID, path string) (*models.Dashboard, error) { + if m.loadPluginDashboardFunc != nil { + return m.loadPluginDashboardFunc(ctx, pluginID, path) + } + + return nil, nil +} + +type dashboardServiceMock struct { + dashboards.DashboardService + importDashboardFunc func(ctx context.Context, dto *dashboards.SaveDashboardDTO) (*models.Dashboard, error) +} + +func (s *dashboardServiceMock) ImportDashboard(ctx context.Context, dto *dashboards.SaveDashboardDTO) (*models.Dashboard, error) { + if s.importDashboardFunc != nil { + return s.importDashboardFunc(ctx, dto) + } + + return nil, nil +} + +type libraryPanelServiceMock struct { + librarypanels.Service + connectLibraryPanelsForDashboardFunc func(ctx context.Context, signedInUser *models.SignedInUser, dash *models.Dashboard) error + importLibraryPanelsForDashboardFunc func(ctx context.Context, signedInUser *models.SignedInUser, dash *models.Dashboard, folderID int64) error +} + +func (s *libraryPanelServiceMock) ConnectLibraryPanelsForDashboard(ctx context.Context, signedInUser *models.SignedInUser, dash *models.Dashboard) error { + if s.connectLibraryPanelsForDashboardFunc != nil { + return s.connectLibraryPanelsForDashboardFunc(ctx, signedInUser, dash) + } + + return nil +} + +func (s *libraryPanelServiceMock) ImportLibraryPanelsForDashboard(ctx context.Context, signedInUser *models.SignedInUser, dash *models.Dashboard, folderID int64) error { + if s.importLibraryPanelsForDashboardFunc != nil { + return s.importLibraryPanelsForDashboardFunc(ctx, signedInUser, dash, folderID) + } + + return nil +} diff --git a/pkg/services/dashboardimport/service/testdata/dashboard.json b/pkg/services/dashboardimport/service/testdata/dashboard.json new file mode 100644 index 00000000000..6a8393dfbfa --- /dev/null +++ b/pkg/services/dashboardimport/service/testdata/dashboard.json @@ -0,0 +1,220 @@ +{ + "__inputs": [ + { + "name": "DS_GDEV-PROMETHEUS", + "label": "gdev-prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.1.0-pre" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "links": [ + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Grafana Docs", + "tooltip": "", + "type": "link", + "url": "https://grafana.com/docs/grafana/latest/" + }, + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Prometheus Docs", + "type": "link", + "url": "http://prometheus.io/docs/introduction/overview/" + } + ], + "panels": [ + { + "datasource": "${DS_GDEV-PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "prometheus" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "{instance=\"localhost:9090\",job=\"prometheus\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.0-pre", + "targets": [ + { + "expr": "sum(irate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\"}[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "samples", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Samples Appended", + "type": "timeseries" + } + ], + "refresh": "1m", + "revision": "1.0", + "schemaVersion": 30, + "style": "dark", + "tags": ["prometheus"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Prometheus 2.0 Stats", + "uid": "UDdpyzz7z", + "version": 1 +} diff --git a/pkg/plugins/manager/dashboard_import.go b/pkg/services/dashboardimport/utils/dash_template_evaluator.go similarity index 85% rename from pkg/plugins/manager/dashboard_import.go rename to pkg/services/dashboardimport/utils/dash_template_evaluator.go index 28600b731c6..7c02e033a59 100644 --- a/pkg/plugins/manager/dashboard_import.go +++ b/pkg/services/dashboardimport/utils/dash_template_evaluator.go @@ -1,4 +1,4 @@ -package manager +package utils import ( "encoding/json" @@ -7,7 +7,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/expr" - "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/dashboardimport" ) var varRegex = regexp.MustCompile(`(\$\{.+?\})`) @@ -22,12 +22,19 @@ func (e DashboardInputMissingError) Error() string { type DashTemplateEvaluator struct { template *simplejson.Json - inputs []plugins.ImportDashboardInput + inputs []dashboardimport.ImportDashboardInput variables map[string]string result *simplejson.Json } -func (e *DashTemplateEvaluator) findInput(varName string, varType string) *plugins.ImportDashboardInput { +func NewDashTemplateEvaluator(template *simplejson.Json, inputs []dashboardimport.ImportDashboardInput) *DashTemplateEvaluator { + return &DashTemplateEvaluator{ + template: template, + inputs: inputs, + } +} + +func (e *DashTemplateEvaluator) findInput(varName string, varType string) *dashboardimport.ImportDashboardInput { for _, input := range e.inputs { if varType == input.Type && (input.Name == varName || input.Name == "*") { return &input @@ -50,7 +57,7 @@ func (e *DashTemplateEvaluator) Eval() (*simplejson.Json, error) { // force expressions value to `__expr__` if inputDefJson.Get("pluginId").MustString() == expr.DatasourceType { - input = &plugins.ImportDashboardInput{ + input = &dashboardimport.ImportDashboardInput{ Value: expr.DatasourceType, } } diff --git a/pkg/services/dashboardimport/utils/dash_template_evaluator_test.go b/pkg/services/dashboardimport/utils/dash_template_evaluator_test.go new file mode 100644 index 00000000000..9d6053228a4 --- /dev/null +++ b/pkg/services/dashboardimport/utils/dash_template_evaluator_test.go @@ -0,0 +1,39 @@ +package utils + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/dashboardimport" + "github.com/stretchr/testify/require" +) + +func TestDashTemplateEvaluator(t *testing.T) { + template, err := simplejson.NewJson([]byte(`{ + "__inputs": [ + { + "name": "DS_NAME", + "type": "datasource" + } + ], + "test": { + "prop": "${DS_NAME}_${DS_NAME}" + } + }`)) + require.NoError(t, err) + + evaluator := &DashTemplateEvaluator{ + template: template, + inputs: []dashboardimport.ImportDashboardInput{ + {Name: "*", Type: "datasource", Value: "my-server"}, + }, + } + + res, err := evaluator.Eval() + require.NoError(t, err) + + require.Equal(t, "my-server_my-server", res.GetPath("test", "prop").MustString()) + + inputs := res.Get("__inputs") + require.Nil(t, inputs.Interface()) +} diff --git a/pkg/services/datasourceproxy/datasourceproxy.go b/pkg/services/datasourceproxy/datasourceproxy.go index 17c4c85cd51..d8aa1607724 100644 --- a/pkg/services/datasourceproxy/datasourceproxy.go +++ b/pkg/services/datasourceproxy/datasourceproxy.go @@ -5,21 +5,24 @@ import ( "fmt" "net/http" "regexp" + "strconv" "github.com/grafana/grafana/pkg/api/datasource" "github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/metrics" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/web" ) func ProvideService(dataSourceCache datasources.CacheService, plugReqValidator models.PluginRequestValidator, pluginStore plugins.Store, cfg *setting.Cfg, httpClientProvider httpclient.Provider, - oauthTokenService *oauthtoken.Service, dsService *datasources.Service) *DataSourceProxyService { + oauthTokenService *oauthtoken.Service, dsService *datasources.Service, tracer tracing.Tracer) *DataSourceProxyService { return &DataSourceProxyService{ DataSourceCache: dataSourceCache, PluginRequestValidator: plugReqValidator, @@ -28,6 +31,7 @@ func ProvideService(dataSourceCache datasources.CacheService, plugReqValidator m HTTPClientProvider: httpClientProvider, OAuthTokenService: oauthTokenService, DataSourcesService: dsService, + tracer: tracer, } } @@ -39,10 +43,16 @@ type DataSourceProxyService struct { HTTPClientProvider httpclient.Provider OAuthTokenService *oauthtoken.Service DataSourcesService *datasources.Service + tracer tracing.Tracer } func (p *DataSourceProxyService) ProxyDataSourceRequest(c *models.ReqContext) { - p.ProxyDatasourceRequestWithID(c, c.ParamsInt64(":id")) + id, err := strconv.ParseInt(web.Params(c.Req)[":id"], 10, 64) + if err != nil { + c.JsonApiErr(http.StatusBadRequest, "id is invalid", err) + return + } + p.ProxyDatasourceRequestWithID(c, id) } func (p *DataSourceProxyService) ProxyDatasourceRequestWithID(c *models.ReqContext, dsID int64) { @@ -77,7 +87,7 @@ func (p *DataSourceProxyService) ProxyDatasourceRequestWithID(c *models.ReqConte proxyPath := getProxyPath(c) proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider, - p.OAuthTokenService, p.DataSourcesService) + p.OAuthTokenService, p.DataSourcesService, p.tracer) if err != nil { if errors.Is(err, datasource.URLValidationError{}) { c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err) diff --git a/pkg/services/datasources/service.go b/pkg/services/datasources/service.go index fe9cf320411..ca38e501cef 100644 --- a/pkg/services/datasources/service.go +++ b/pkg/services/datasources/service.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "strconv" + "strings" "sync" "time" @@ -14,6 +15,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" @@ -49,7 +51,7 @@ type cachedDecryptedJSON struct { json map[string]string } -func ProvideService(bus bus.Bus, store *sqlstore.SQLStore, secretsService secrets.Service) *Service { +func ProvideService(bus bus.Bus, store *sqlstore.SQLStore, secretsService secrets.Service, ac accesscontrol.AccessControl) *Service { s := &Service{ Bus: bus, SQLStore: store, @@ -70,9 +72,41 @@ func ProvideService(bus bus.Bus, store *sqlstore.SQLStore, secretsService secret s.Bus.AddHandler(s.UpdateDataSource) s.Bus.AddHandler(s.GetDefaultDataSource) + ac.RegisterAttributeScopeResolver(NewNameScopeResolver(store)) + return s } +type DataSourceRetriever interface { + GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error +} + +// NewNameScopeResolver provides an AttributeScopeResolver able to +// translate a scope prefixed with "datasources:name:" into an id based scope. +func NewNameScopeResolver(db DataSourceRetriever) (string, accesscontrol.AttributeScopeResolveFunc) { + dsNameResolver := func(ctx context.Context, orgID int64, initialScope string) (string, error) { + dsNames := strings.Split(initialScope, ":") + if dsNames[0] != "datasources" || len(dsNames) != 3 { + return "", accesscontrol.ErrInvalidScope + } + + dsName := dsNames[2] + // Special wildcard case + if dsName == "*" { + return accesscontrol.Scope("datasources", "id", "*"), nil + } + + query := models.GetDataSourceQuery{Name: dsName, OrgId: orgID} + if err := db.GetDataSource(ctx, &query); err != nil { + return "", err + } + + return accesscontrol.Scope("datasources", "id", fmt.Sprintf("%v", query.Result.Id)), nil + } + + return "datasources:name:", dsNameResolver +} + func (s *Service) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error { return s.SQLStore.GetDataSource(ctx, query) } @@ -243,14 +277,13 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio } } - if ds.JsonData != nil && ds.JsonData.Get("azureAuth").MustBool() { + if ds.JsonData != nil { credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), s.DecryptedValues(ds)) if err != nil { err = fmt.Errorf("invalid Azure credentials: %s", err) return nil, err } - opts.CustomOptions["_azureAuth"] = true if credentials != nil { opts.CustomOptions["_azureCredentials"] = credentials } diff --git a/pkg/services/datasources/service_test.go b/pkg/services/datasources/service_test.go index d307d30b330..136540fd516 100644 --- a/pkg/services/datasources/service_test.go +++ b/pkg/services/datasources/service_test.go @@ -13,6 +13,8 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" + acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets/database" "github.com/grafana/grafana/pkg/services/secrets/fakes" @@ -34,7 +36,7 @@ func TestService(t *testing.T) { }) secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) - s := ProvideService(bus.New(), sqlStore, secretsService) + s := ProvideService(bus.New(), sqlStore, secretsService, &acmock.Mock{}) var ds *models.DataSource @@ -66,6 +68,72 @@ func TestService(t *testing.T) { }) } +type dataSourceMockRetriever struct { + res *models.DataSource +} + +func (d *dataSourceMockRetriever) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error { + if query.Name == d.res.Name { + query.Result = d.res + + return nil + } + return models.ErrDataSourceNotFound +} + +func TestService_NameScopeResolver(t *testing.T) { + type testCaseResolver struct { + desc string + given string + want string + wantErr error + } + + testCases := []testCaseResolver{ + { + desc: "correct", + given: "datasources:name:test-datasource", + want: "datasources:id:1", + wantErr: nil, + }, + { + desc: "correct", + given: "datasources:name:*", + want: "datasources:id:*", + wantErr: nil, + }, + { + desc: "unknown datasource", + given: "datasources:name:unknown-datasource", + want: "", + wantErr: models.ErrDataSourceNotFound, + }, + { + desc: "malformed scope", + given: "datasources:unknown-datasource", + want: "", + wantErr: accesscontrol.ErrInvalidScope, + }, + } + + testDataSource := &models.DataSource{Id: 1, Name: "test-datasource"} + prefix, resolver := NewNameScopeResolver(&dataSourceMockRetriever{testDataSource}) + require.Equal(t, "datasources:name:", prefix) + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + resolved, err := resolver(context.Background(), 1, tc.given) + if tc.wantErr != nil { + require.Error(t, err) + require.Equal(t, tc.wantErr, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.want, resolved) + } + }) + } +} + //nolint:goconst func TestService_GetHttpTransport(t *testing.T) { t.Run("Should use cached proxy", func(t *testing.T) { @@ -83,7 +151,7 @@ func TestService_GetHttpTransport(t *testing.T) { } secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) rt1, err := dsService.GetHTTPTransport(&ds, provider) require.NoError(t, err) @@ -116,7 +184,7 @@ func TestService_GetHttpTransport(t *testing.T) { json.Set("tlsAuthWithCACert", true) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope()) require.NoError(t, err) @@ -166,7 +234,7 @@ func TestService_GetHttpTransport(t *testing.T) { json.Set("tlsAuth", true) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) tlsClientCert, err := secretsService.Encrypt(context.Background(), []byte(clientCert), secrets.WithoutScope()) require.NoError(t, err) @@ -209,7 +277,7 @@ func TestService_GetHttpTransport(t *testing.T) { json.Set("serverName", "server-name") secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope()) require.NoError(t, err) @@ -246,7 +314,7 @@ func TestService_GetHttpTransport(t *testing.T) { json.Set("tlsSkipVerify", true) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) ds := models.DataSource{ Id: 1, @@ -277,7 +345,7 @@ func TestService_GetHttpTransport(t *testing.T) { }) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) encryptedData, err := secretsService.Encrypt(context.Background(), []byte(`Bearer xf5yhfkpsnmgo`), secrets.WithoutScope()) require.NoError(t, err) @@ -336,7 +404,7 @@ func TestService_GetHttpTransport(t *testing.T) { }) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) ds := models.DataSource{ Id: 1, @@ -369,7 +437,7 @@ func TestService_GetHttpTransport(t *testing.T) { require.NoError(t, err) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) ds := models.DataSource{ Type: models.DS_ES, @@ -403,7 +471,7 @@ func TestService_getTimeout(t *testing.T) { } secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) for _, tc := range testCases { ds := &models.DataSource{ @@ -416,7 +484,7 @@ func TestService_getTimeout(t *testing.T) { func TestService_DecryptedValue(t *testing.T) { t.Run("When datasource hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) { secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) encryptedJsonData, err := secretsService.EncryptJsonData( context.Background(), @@ -470,7 +538,7 @@ func TestService_DecryptedValue(t *testing.T) { SecureJsonData: encryptedJsonData, } - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) // Populate cache password, ok := dsService.DecryptedValue(&ds, "password") @@ -502,90 +570,48 @@ func TestService_HTTPClientOptions(t *testing.T) { } t.Run("Azure authentication", func(t *testing.T) { - t.Run("should be disabled if not enabled in JsonData", func(t *testing.T) { + t.Run("should be disabled if no Azure credentials configured", func(t *testing.T) { t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData }) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) opts, err := dsService.httpClientOptions(&ds) require.NoError(t, err) - assert.NotEqual(t, true, opts.CustomOptions["_azureAuth"]) assert.NotContains(t, opts.CustomOptions, "_azureCredentials") }) - t.Run("should be enabled if enabled in JsonData without credentials configured", func(t *testing.T) { + t.Run("should be enabled if Azure credentials configured", func(t *testing.T) { t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData }) ds.JsonData = simplejson.NewFromAny(map[string]interface{}{ - "azureAuth": true, - }) - - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) - - opts, err := dsService.httpClientOptions(&ds) - require.NoError(t, err) - - assert.Equal(t, true, opts.CustomOptions["_azureAuth"]) - assert.NotContains(t, opts.CustomOptions, "_azureCredentials") - }) - - t.Run("should be enabled if enabled in JsonData with credentials configured", func(t *testing.T) { - t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData }) - - ds.JsonData = simplejson.NewFromAny(map[string]interface{}{ - "azureAuth": true, "azureCredentials": map[string]interface{}{ "authType": "msi", }, }) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) opts, err := dsService.httpClientOptions(&ds) require.NoError(t, err) - assert.Equal(t, true, opts.CustomOptions["_azureAuth"]) - require.Contains(t, opts.CustomOptions, "_azureCredentials") credentials := opts.CustomOptions["_azureCredentials"] assert.IsType(t, &azcredentials.AzureManagedIdentityCredentials{}, credentials) }) - t.Run("should be disabled if disabled in JsonData even with credentials configured", func(t *testing.T) { - t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData }) - - ds.JsonData = simplejson.NewFromAny(map[string]interface{}{ - "azureAuth": false, - "azureCredentials": map[string]interface{}{ - "authType": "msi", - }, - }) - - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) - - opts, err := dsService.httpClientOptions(&ds) - require.NoError(t, err) - - assert.NotEqual(t, true, opts.CustomOptions["_azureAuth"]) - assert.NotContains(t, opts.CustomOptions, "_azureCredentials") - }) - t.Run("should fail if credentials are invalid", func(t *testing.T) { t.Cleanup(func() { ds.JsonData = emptyJsonData; ds.SecureJsonData = emptySecureJsonData }) ds.JsonData = simplejson.NewFromAny(map[string]interface{}{ - "azureAuth": true, "azureCredentials": "invalid", }) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) _, err := dsService.httpClientOptions(&ds) assert.Error(t, err) @@ -599,7 +625,7 @@ func TestService_HTTPClientOptions(t *testing.T) { }) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(bus.New(), nil, secretsService) + dsService := ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) opts, err := dsService.httpClientOptions(&ds) require.NoError(t, err) diff --git a/pkg/services/featuremgmt/features.go b/pkg/services/featuremgmt/features.go new file mode 100644 index 00000000000..8a19c794f3b --- /dev/null +++ b/pkg/services/featuremgmt/features.go @@ -0,0 +1,96 @@ +package featuremgmt + +import ( + "bytes" + "encoding/json" +) + +type FeatureToggles interface { + IsEnabled(flag string) bool +} + +// FeatureFlagState indicates the quality level +type FeatureFlagState int + +const ( + // FeatureStateUnknown indicates that no state is specified + FeatureStateUnknown FeatureFlagState = iota + + // FeatureStateAlpha the feature is in active development and may change at any time + FeatureStateAlpha + + // FeatureStateBeta the feature is still in development, but settings will have migrations + FeatureStateBeta + + // FeatureStateStable this is a stable feature + FeatureStateStable + + // FeatureStateDeprecated the feature will be removed in the future + FeatureStateDeprecated +) + +func (s FeatureFlagState) String() string { + switch s { + case FeatureStateAlpha: + return "alpha" + case FeatureStateBeta: + return "beta" + case FeatureStateStable: + return "stable" + case FeatureStateDeprecated: + return "deprecated" + case FeatureStateUnknown: + } + return "unknown" +} + +// MarshalJSON marshals the enum as a quoted json string +func (s FeatureFlagState) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(s.String()) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +// UnmarshalJSON unmarshals a quoted json string to the enum value +func (s *FeatureFlagState) UnmarshalJSON(b []byte) error { + var j string + err := json.Unmarshal(b, &j) + if err != nil { + return err + } + + switch j { + case "alpha": + *s = FeatureStateAlpha + + case "beta": + *s = FeatureStateBeta + + case "stable": + *s = FeatureStateStable + + case "deprecated": + *s = FeatureStateDeprecated + + default: + *s = FeatureStateUnknown + } + return nil +} + +type FeatureFlag struct { + Name string `json:"name" yaml:"name"` // Unique name + Description string `json:"description"` + State FeatureFlagState `json:"state,omitempty"` + DocsURL string `json:"docsURL,omitempty"` + + // CEL-GO expression. Using the value "true" will mean this is on by default + Expression string `json:"expression,omitempty"` + + // Special behavior flags + RequiresDevMode bool `json:"requiresDevMode,omitempty"` // can not be enabled in production + RequiresRestart bool `json:"requiresRestart,omitempty"` // The server must be initialized with the value + RequiresLicense bool `json:"requiresLicense,omitempty"` // Must be enabled in the license + FrontendOnly bool `json:"frontend,omitempty"` // change is only seen in the frontend +} diff --git a/pkg/services/featuremgmt/manager.go b/pkg/services/featuremgmt/manager.go new file mode 100644 index 00000000000..1b795d0348d --- /dev/null +++ b/pkg/services/featuremgmt/manager.go @@ -0,0 +1,185 @@ +package featuremgmt + +import ( + "context" + "fmt" + "reflect" + + "github.com/grafana/grafana/pkg/infra/log" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/models" +) + +var ( + _ FeatureToggles = (*FeatureManager)(nil) +) + +type FeatureManager struct { + isDevMod bool + licensing models.Licensing + flags map[string]*FeatureFlag + enabled map[string]bool // only the "on" values + config string // path to config file + vars map[string]interface{} + log log.Logger +} + +// This will merge the flags with the current configuration +func (fm *FeatureManager) registerFlags(flags ...FeatureFlag) { + for _, add := range flags { + if add.Name == "" { + continue // skip it with warning? + } + flag, ok := fm.flags[add.Name] + if !ok { + f := add // make a copy + fm.flags[add.Name] = &f + continue + } + + // Selectively update properties + if add.Description != "" { + flag.Description = add.Description + } + if add.DocsURL != "" { + flag.DocsURL = add.DocsURL + } + if add.Expression != "" { + flag.Expression = add.Expression + } + + // The most recently defined state + if add.State != FeatureStateUnknown { + flag.State = add.State + } + + // Only gets more restrictive + if add.RequiresDevMode { + flag.RequiresDevMode = true + } + + if add.RequiresLicense { + flag.RequiresLicense = true + } + + if add.RequiresRestart { + flag.RequiresRestart = true + } + } + + // This will evaluate all flags + fm.update() +} + +func (fm *FeatureManager) evaluate(ff *FeatureFlag) bool { + if ff.RequiresDevMode && !fm.isDevMod { + return false + } + + if ff.RequiresLicense && (fm.licensing == nil || !fm.licensing.FeatureEnabled(ff.Name)) { + return false + } + + // TODO: CEL - expression + return ff.Expression == "true" +} + +// Update +func (fm *FeatureManager) update() { + enabled := make(map[string]bool) + for _, flag := range fm.flags { + val := fm.evaluate(flag) + + // Update the registry + track := 0.0 + if val { + track = 1 + enabled[flag.Name] = true + } + + // Register value with prometheus metric + featureToggleInfo.WithLabelValues(flag.Name).Set(track) + } + fm.enabled = enabled +} + +// Run is called by background services +func (fm *FeatureManager) readFile() error { + if fm.config == "" { + return nil // not configured + } + + cfg, err := readConfigFile(fm.config) + if err != nil { + return err + } + + fm.registerFlags(cfg.Flags...) + fm.vars = cfg.Vars + + return nil +} + +// IsEnabled checks if a feature is enabled +func (fm *FeatureManager) IsEnabled(flag string) bool { + return fm.enabled[flag] +} + +// GetEnabled returns a map contaning only the features that are enabled +func (fm *FeatureManager) GetEnabled(ctx context.Context) map[string]bool { + enabled := make(map[string]bool, len(fm.enabled)) + for key, val := range fm.enabled { + if val { + enabled[key] = true + } + } + return enabled +} + +// GetFlags returns all flag definitions +func (fm *FeatureManager) GetFlags() []FeatureFlag { + v := make([]FeatureFlag, 0, len(fm.flags)) + for _, value := range fm.flags { + v = append(v, *value) + } + return v +} + +func (fm *FeatureManager) HandleGetSettings(c *models.ReqContext) { + res := make(map[string]interface{}, 3) + res["enabled"] = fm.GetEnabled(c.Req.Context()) + + vv := make([]*FeatureFlag, 0, len(fm.flags)) + for _, v := range fm.flags { + vv = append(vv, v) + } + + res["info"] = vv + + response.JSON(200, res).WriteTo(c) +} + +// WithFeatures is used to define feature toggles for testing. +// The arguments are a list of strings that are optionally followed by a boolean value +func WithFeatures(spec ...interface{}) *FeatureManager { + count := len(spec) + enabled := make(map[string]bool, count) + + idx := 0 + for idx < count { + key := fmt.Sprintf("%v", spec[idx]) + val := true + idx++ + if idx < count && reflect.TypeOf(spec[idx]).Kind() == reflect.Bool { + val = spec[idx].(bool) + idx++ + } + + if val { + enabled[key] = true + } + } + + return &FeatureManager{enabled: enabled} +} diff --git a/pkg/services/featuremgmt/manager_test.go b/pkg/services/featuremgmt/manager_test.go new file mode 100644 index 00000000000..68c31daf9a9 --- /dev/null +++ b/pkg/services/featuremgmt/manager_test.go @@ -0,0 +1,77 @@ +package featuremgmt + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFeatureManager(t *testing.T) { + t.Run("check testing stubs", func(t *testing.T) { + ft := WithFeatures("a", "b", "c") + require.True(t, ft.IsEnabled("a")) + require.True(t, ft.IsEnabled("b")) + require.True(t, ft.IsEnabled("c")) + require.False(t, ft.IsEnabled("d")) + + require.Equal(t, map[string]bool{"a": true, "b": true, "c": true}, ft.GetEnabled(context.Background())) + + // Explicit values + ft = WithFeatures("a", true, "b", false) + require.True(t, ft.IsEnabled("a")) + require.False(t, ft.IsEnabled("b")) + require.Equal(t, map[string]bool{"a": true}, ft.GetEnabled(context.Background())) + }) + + t.Run("check license validation", func(t *testing.T) { + ft := FeatureManager{ + flags: map[string]*FeatureFlag{}, + } + ft.registerFlags(FeatureFlag{ + Name: "a", + RequiresLicense: true, + RequiresDevMode: true, + Expression: "true", + }, FeatureFlag{ + Name: "b", + Expression: "true", + }) + require.False(t, ft.IsEnabled("a")) + require.True(t, ft.IsEnabled("b")) + require.False(t, ft.IsEnabled("c")) // uknown flag + + // Try changing "requires license" + ft.registerFlags(FeatureFlag{ + Name: "a", + RequiresLicense: false, // shuld still require license! + }, FeatureFlag{ + Name: "b", + RequiresLicense: true, // expression is still "true" + }) + require.False(t, ft.IsEnabled("a")) + require.False(t, ft.IsEnabled("b")) + require.False(t, ft.IsEnabled("c")) + }) + + t.Run("check description and docs configs", func(t *testing.T) { + ft := FeatureManager{ + flags: map[string]*FeatureFlag{}, + } + ft.registerFlags(FeatureFlag{ + Name: "a", + Description: "first", + }, FeatureFlag{ + Name: "a", + Description: "second", + }, FeatureFlag{ + Name: "a", + DocsURL: "http://something", + }, FeatureFlag{ + Name: "a", + }) + flag := ft.flags["a"] + require.Equal(t, "second", flag.Description) + require.Equal(t, "http://something", flag.DocsURL) + }) +} diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go new file mode 100644 index 00000000000..651ed0f3009 --- /dev/null +++ b/pkg/services/featuremgmt/registry.go @@ -0,0 +1,117 @@ +package featuremgmt + +var ( + // Register each toggle here + standardFeatureFlags = []FeatureFlag{ + { + Name: "trimDefaults", + Description: "Use cue schema to remove values that will be applied automatically", + State: FeatureStateBeta, + }, + { + Name: "envelopeEncryption", + Description: "encrypt secrets", + State: FeatureStateBeta, + }, + + { + Name: "httpclientprovider_azure_auth", + Description: "use http client for azure auth", + State: FeatureStateBeta, + }, + { + Name: "service-accounts", + Description: "support service accounts", + State: FeatureStateBeta, + RequiresLicense: true, + }, + + { + Name: "database_metrics", + Description: "Add prometheus metrics for database tables", + State: FeatureStateStable, + }, + { + Name: "dashboardPreviews", + Description: "Create and show thumbnails for dashboard search results", + State: FeatureStateAlpha, + }, + { + Name: "live-config", + Description: "Save grafana live configuration in SQL tables", + State: FeatureStateAlpha, + }, + { + Name: "live-pipeline", + Description: "enable a generic live processing pipeline", + State: FeatureStateAlpha, + }, + { + Name: "live-service-web-worker", + Description: "This will use a webworker thread to processes events rather than the main thread", + State: FeatureStateAlpha, + FrontendOnly: true, + }, + { + Name: "queryOverLive", + Description: "Use grafana live websocket to execute backend queries", + State: FeatureStateAlpha, + FrontendOnly: true, + }, + { + Name: "tempoSearch", + Description: "Enable searching in tempo datasources", + State: FeatureStateBeta, + FrontendOnly: true, + }, + { + Name: "tempoBackendSearch", + Description: "Use backend for tempo search", + State: FeatureStateBeta, + }, + { + Name: "tempoServiceGraph", + Description: "show service", + State: FeatureStateBeta, + FrontendOnly: true, + }, + { + Name: "fullRangeLogsVolume", + Description: "Show full range logs volume in explore", + State: FeatureStateBeta, + FrontendOnly: true, + }, + { + Name: "accesscontrol", + Description: "Support robust access control", + State: FeatureStateBeta, + }, + { + Name: "prometheus_azure_auth", + Description: "Use azure authentication for prometheus datasource", + State: FeatureStateBeta, + }, + { + Name: "newNavigation", + Description: "Try the next gen navigation model", + State: FeatureStateAlpha, + }, + { + Name: "showFeatureFlagsInUI", + Description: "Show feature flags in the settings UI", + State: FeatureStateAlpha, + RequiresDevMode: true, + }, + { + Name: "disable_http_request_histogram", + Description: "Do not create histograms for http requests", + State: FeatureStateAlpha, + }, + { + Name: "validatedQueries", + Description: "only execute the query saved in a panel", + State: FeatureStateAlpha, + RequiresDevMode: true, + }, + } +) diff --git a/pkg/services/featuremgmt/service.go b/pkg/services/featuremgmt/service.go new file mode 100644 index 00000000000..d3e5622f451 --- /dev/null +++ b/pkg/services/featuremgmt/service.go @@ -0,0 +1,76 @@ +package featuremgmt + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/grafana/grafana/pkg/infra/log" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // The values are updated each time + featureToggleInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "feature_toggles_info", + Help: "info metric that exposes what feature toggles are enabled or not", + Namespace: "grafana", + }, []string{"name"}) +) + +func ProvideManagerService(cfg *setting.Cfg, licensing models.Licensing) (*FeatureManager, error) { + mgmt := &FeatureManager{ + isDevMod: setting.Env != setting.Prod, + licensing: licensing, + flags: make(map[string]*FeatureFlag, 30), + enabled: make(map[string]bool), + log: log.New("featuremgmt"), + } + + // Register the standard flags + mgmt.registerFlags(standardFeatureFlags...) + + // Load the flags from `custom.ini` files + flags, err := setting.ReadFeatureTogglesFromInitFile(cfg.Raw.Section("feature_toggles")) + if err != nil { + return mgmt, err + } + for key, val := range flags { + flag, ok := mgmt.flags[key] + if !ok { + flag = &FeatureFlag{ + Name: key, + State: FeatureStateUnknown, + } + mgmt.flags[key] = flag + } + flag.Expression = fmt.Sprintf("%t", val) // true | false + } + + // Load config settings + configfile := filepath.Join(cfg.HomePath, "conf", "features.yaml") + if _, err := os.Stat(configfile); err == nil { + mgmt.log.Info("[experimental] loading features from config file", "path", configfile) + mgmt.config = configfile + err = mgmt.readFile() + if err != nil { + return mgmt, err + } + } + + // update the values + mgmt.update() + + // Minimum approach to avoid circular dependency + cfg.IsFeatureToggleEnabled = mgmt.IsEnabled + return mgmt, nil +} + +// ProvideToggles allows read-only access to the feature state +func ProvideToggles(mgmt *FeatureManager) FeatureToggles { + return mgmt +} diff --git a/pkg/services/featuremgmt/service_test.go b/pkg/services/featuremgmt/service_test.go new file mode 100644 index 00000000000..1085ea91f2c --- /dev/null +++ b/pkg/services/featuremgmt/service_test.go @@ -0,0 +1,84 @@ +package featuremgmt + +import ( + "testing" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/require" +) + +func TestFeatureService(t *testing.T) { + license := stubLicenseServier{ + flags: []FeatureFlag{ + { + Name: "a.yes.default", + RequiresLicense: true, + Expression: "true", + }, + { + Name: "a.yes", + RequiresLicense: true, + Expression: "", + }, + { + Name: "b.no", + RequiresLicense: true, + }, + }, + enabled: map[string]bool{ + "a.yes.default": true, + "a.yes": true, + }, + } + require.False(t, license.FeatureEnabled("unknown")) + require.False(t, license.FeatureEnabled("b.no")) + require.True(t, license.FeatureEnabled("a.yes")) + require.True(t, license.FeatureEnabled("a.yes.default")) + + cfg := setting.NewCfg() + mgmt, err := ProvideManagerService(cfg, license) + require.NoError(t, err) + require.NotNil(t, mgmt) + + // Enterprise features do not fall though automatically + require.False(t, mgmt.IsEnabled("a.yes.default")) + require.False(t, mgmt.IsEnabled("a.yes")) // licensed, but not enabled +} + +var ( + _ models.Licensing = (*stubLicenseServier)(nil) +) + +type stubLicenseServier struct { + flags []FeatureFlag + enabled map[string]bool +} + +func (s stubLicenseServier) Expiry() int64 { + return 100 +} + +func (s stubLicenseServier) Edition() string { + return "test" +} + +func (s stubLicenseServier) ContentDeliveryPrefix() string { + return "" +} + +func (s stubLicenseServier) LicenseURL(showAdminLicensingPage bool) string { + return "http://??" +} + +func (s stubLicenseServier) StateInfo() string { + return "ok" +} + +func (s stubLicenseServier) EnabledFeatures() map[string]bool { + return map[string]bool{} +} + +func (s stubLicenseServier) FeatureEnabled(feature string) bool { + return s.enabled[feature] +} diff --git a/pkg/services/featuremgmt/settings.go b/pkg/services/featuremgmt/settings.go new file mode 100644 index 00000000000..dc2137bf567 --- /dev/null +++ b/pkg/services/featuremgmt/settings.go @@ -0,0 +1,34 @@ +package featuremgmt + +import ( + "io/ioutil" + + "gopkg.in/yaml.v2" +) + +type configBody struct { + // define variables that can be used in expressions + Vars map[string]interface{} `yaml:"vars"` + + // Define and override feature flag properties + Flags []FeatureFlag `yaml:"flags"` + + // keep track of where the fie was loaded from + filename string +} + +// will read a single configfile +func readConfigFile(filename string) (*configBody, error) { + cfg := &configBody{} + + // Can ignore gosec G304 because the file path is forced within config subfolder + //nolint:gosec + yamlFile, err := ioutil.ReadFile(filename) + if err != nil { + return cfg, err + } + + err = yaml.Unmarshal(yamlFile, cfg) + cfg.filename = filename + return cfg, err +} diff --git a/pkg/services/featuremgmt/settings_test.go b/pkg/services/featuremgmt/settings_test.go new file mode 100644 index 00000000000..58683ad5971 --- /dev/null +++ b/pkg/services/featuremgmt/settings_test.go @@ -0,0 +1,25 @@ +package featuremgmt + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestReadingFeatureSettings(t *testing.T) { + config, err := readConfigFile("testdata/features.yaml") + require.NoError(t, err, "No error when reading feature configs") + + assert.Equal(t, map[string]interface{}{ + "level": "free", + "stack": "something", + "valA": "value from features.yaml", + }, config.Vars) + + out, err := yaml.Marshal(config) + require.NoError(t, err) + fmt.Printf("%s", string(out)) +} diff --git a/pkg/services/featuremgmt/testdata/features.yaml b/pkg/services/featuremgmt/testdata/features.yaml new file mode 100644 index 00000000000..dd737494580 --- /dev/null +++ b/pkg/services/featuremgmt/testdata/features.yaml @@ -0,0 +1,33 @@ +include: + - included.yaml # not yet supported + +vars: + stack: something + level: free + valA: value from features.yaml + +flags: + - name: feature1 + description: feature1 + expression: "false" + + - name: feature3 + description: feature3 + expression: "true" + + - name: feature3 + description: feature3 + expression: env.level == 'free' + + - name: displaySwedishTheme + description: enable swedish background theme + expression: | + // restrict to users allowing swedish language + req.locale.contains("sv") + - name: displayFrenchFlag + description: sho background theme + expression: | + // only admins + user.id == 1 + // show to users allowing french language + && req.locale.contains("fr") \ No newline at end of file diff --git a/pkg/services/featuremgmt/testdata/included.yaml b/pkg/services/featuremgmt/testdata/included.yaml new file mode 100644 index 00000000000..322b5c7972f --- /dev/null +++ b/pkg/services/featuremgmt/testdata/included.yaml @@ -0,0 +1,13 @@ +include: + - features.yaml # make sure we avoid recusion! + +# variables that can be used in expressions +vars: + stack: something + deep: 1 + valA: value from included.yaml + +flags: + - name: featureFromIncludedFile + description: an inlcuded file + expression: invalid expression string here diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go new file mode 100644 index 00000000000..430b9fbda56 --- /dev/null +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -0,0 +1,85 @@ +// NOTE: This file is autogenerated + +package featuremgmt + +const ( + // FlagTrimDefaults + // Use cue schema to remove values that will be applied automatically + FlagTrimDefaults = "trimDefaults" + + // FlagEnvelopeEncryption + // encrypt secrets + FlagEnvelopeEncryption = "envelopeEncryption" + + // FlagHttpclientproviderAzureAuth + // use http client for azure auth + FlagHttpclientproviderAzureAuth = "httpclientprovider_azure_auth" + + // FlagServiceAccounts + // support service accounts + FlagServiceAccounts = "service-accounts" + + // FlagDatabaseMetrics + // Add prometheus metrics for database tables + FlagDatabaseMetrics = "database_metrics" + + // FlagDashboardPreviews + // Create and show thumbnails for dashboard search results + FlagDashboardPreviews = "dashboardPreviews" + + // FlagLiveConfig + // Save grafana live configuration in SQL tables + FlagLiveConfig = "live-config" + + // FlagLivePipeline + // enable a generic live processing pipeline + FlagLivePipeline = "live-pipeline" + + // FlagLiveServiceWebWorker + // This will use a webworker thread to processes events rather than the main thread + FlagLiveServiceWebWorker = "live-service-web-worker" + + // FlagQueryOverLive + // Use grafana live websocket to execute backend queries + FlagQueryOverLive = "queryOverLive" + + // FlagTempoSearch + // Enable searching in tempo datasources + FlagTempoSearch = "tempoSearch" + + // FlagTempoBackendSearch + // Use backend for tempo search + FlagTempoBackendSearch = "tempoBackendSearch" + + // FlagTempoServiceGraph + // show service + FlagTempoServiceGraph = "tempoServiceGraph" + + // FlagFullRangeLogsVolume + // Show full range logs volume in explore + FlagFullRangeLogsVolume = "fullRangeLogsVolume" + + // FlagAccesscontrol + // Support robust access control + FlagAccesscontrol = "accesscontrol" + + // FlagPrometheusAzureAuth + // Use azure authentication for prometheus datasource + FlagPrometheusAzureAuth = "prometheus_azure_auth" + + // FlagNewNavigation + // Try the next gen navigation model + FlagNewNavigation = "newNavigation" + + // FlagShowFeatureFlagsInUI + // Show feature flags in the settings UI + FlagShowFeatureFlagsInUI = "showFeatureFlagsInUI" + + // FlagDisableHttpRequestHistogram + // Do not create histograms for http requests + FlagDisableHttpRequestHistogram = "disable_http_request_histogram" + + // FlagValidatedQueries + // only execute the query saved in a panel + FlagValidatedQueries = "validatedQueries" +) diff --git a/pkg/services/featuremgmt/toggles_gen_test.go b/pkg/services/featuremgmt/toggles_gen_test.go new file mode 100644 index 00000000000..28e371617d6 --- /dev/null +++ b/pkg/services/featuremgmt/toggles_gen_test.go @@ -0,0 +1,140 @@ +package featuremgmt + +import ( + "bytes" + "fmt" + "html/template" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "unicode" + + "github.com/google/go-cmp/cmp" +) + +func TestFeatureToggleFiles(t *testing.T) { + // Typescript files + verifyAndGenerateFile(t, + "../../../packages/grafana-data/src/types/featureToggles.gen.ts", + generateTypeScript(), + ) + + // Golang files + verifyAndGenerateFile(t, + "toggles_gen.go", + generateRegistry(t), + ) +} + +func verifyAndGenerateFile(t *testing.T, fpath string, gen string) { + // nolint:gosec + // We can ignore the gosec G304 warning since this is a test and the function is only called explicitly above + body, err := ioutil.ReadFile(fpath) + if err == nil { + if diff := cmp.Diff(gen, string(body)); diff != "" { + str := fmt.Sprintf("body mismatch (-want +got):\n%s\n", diff) + err = fmt.Errorf(str) + } + } + + if err != nil { + e2 := os.WriteFile(fpath, []byte(gen), 0644) + if e2 != nil { + t.Errorf("error writing file: %s", e2.Error()) + } + abs, _ := filepath.Abs(fpath) + t.Errorf("feature toggle do not match: %s (%s)", err.Error(), abs) + t.Fail() + } +} + +func generateTypeScript() string { + buf := `// NOTE: This file was auto generated. DO NOT EDIT DIRECTLY! +// To change feature flags, edit: +// pkg/services/featuremgmt/registry.go +// Then run tests in: +// pkg/services/featuremgmt/toggles_gen_test.go + +/** + * Describes available feature toggles in Grafana. These can be configured via + * conf/custom.ini to enable features under development or not yet available in + * stable version. + * + * Only enabled values will be returned in this interface + * + * @public + */ +export interface FeatureToggles { + [name: string]: boolean | undefined; // support any string value + +` + for _, flag := range standardFeatureFlags { + buf += " " + getTypeScriptKey(flag.Name) + "?: boolean;\n" + } + + buf += "}\n" + return buf +} + +func getTypeScriptKey(key string) string { + if strings.Contains(key, "-") || strings.Contains(key, ".") { + return "['" + key + "']" + } + return key +} + +func isLetterOrNumber(c rune) bool { + return !unicode.IsLetter(c) && !unicode.IsNumber(c) +} + +func asCamelCase(key string) string { + parts := strings.FieldsFunc(key, isLetterOrNumber) + for idx, part := range parts { + parts[idx] = strings.Title(part) + } + return strings.Join(parts, "") +} + +func generateRegistry(t *testing.T) string { + tmpl, err := template.New("fn").Parse(` +{{"\t"}}// Flag{{.CamelCase}}{{.Ext}} +{{"\t"}}Flag{{.CamelCase}} = "{{.Flag.Name}}" +`) + if err != nil { + t.Fatal("error reading template", "error", err.Error()) + return "" + } + + data := struct { + CamelCase string + Flag FeatureFlag + Ext string + }{ + CamelCase: "?", + } + + var buff bytes.Buffer + + buff.WriteString(`// NOTE: This file is autogenerated + +package featuremgmt + +const (`) + + for _, flag := range standardFeatureFlags { + data.CamelCase = asCamelCase(flag.Name) + data.Flag = flag + data.Ext = "" + + if flag.Description != "" { + data.Ext += "\n\t// " + flag.Description + } + + _ = tmpl.Execute(&buff, data) + } + buff.WriteString(")\n") + + return buff.String() +} diff --git a/pkg/services/kmsproviders/osskmsproviders/osskmsproviders.go b/pkg/services/kmsproviders/osskmsproviders/osskmsproviders.go index 2e01653f3fa..9568730fa5f 100644 --- a/pkg/services/kmsproviders/osskmsproviders/osskmsproviders.go +++ b/pkg/services/kmsproviders/osskmsproviders/osskmsproviders.go @@ -2,6 +2,7 @@ package osskmsproviders import ( "github.com/grafana/grafana/pkg/services/encryption" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/kmsproviders" grafana "github.com/grafana/grafana/pkg/services/kmsproviders/defaultprovider" "github.com/grafana/grafana/pkg/services/secrets" @@ -21,7 +22,7 @@ func ProvideService(enc encryption.Internal, settings setting.Provider) Service } func (s Service) Provide() (map[secrets.ProviderID]secrets.Provider, error) { - if !s.settings.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle) { + if !s.settings.IsFeatureToggleEnabled(featuremgmt.FlagEnvelopeEncryption) { return nil, nil } diff --git a/pkg/services/ldap/ldap_groups.go b/pkg/services/ldap/ldap_groups.go new file mode 100644 index 00000000000..3406947660b --- /dev/null +++ b/pkg/services/ldap/ldap_groups.go @@ -0,0 +1,17 @@ +package ldap + +import "github.com/grafana/grafana/pkg/models" + +type Groups interface { + GetTeams(groups []string) ([]models.TeamOrgGroupDTO, error) +} + +type OSSGroups struct{} + +func ProvideGroupsService() *OSSGroups { + return &OSSGroups{} +} + +func (*OSSGroups) GetTeams(_ []string) ([]models.TeamOrgGroupDTO, error) { + return nil, nil +} diff --git a/pkg/services/libraryelements/libraryelements_test.go b/pkg/services/libraryelements/libraryelements_test.go index 981a7d14084..2648496a108 100644 --- a/pkg/services/libraryelements/libraryelements_test.go +++ b/pkg/services/libraryelements/libraryelements_test.go @@ -316,6 +316,7 @@ func testScenario(t *testing.T, desc string, fn func(t *testing.T, sc scenarioCo Name: "User In DB", Login: userInDbName, } + _, err := sqlStore.CreateUser(context.Background(), cmd) require.NoError(t, err) diff --git a/pkg/services/librarypanels/librarypanels_test.go b/pkg/services/librarypanels/librarypanels_test.go index 612ffe07ff2..2a73e01e632 100644 --- a/pkg/services/librarypanels/librarypanels_test.go +++ b/pkg/services/librarypanels/librarypanels_test.go @@ -1547,6 +1547,7 @@ func testScenario(t *testing.T, desc string, fn func(t *testing.T, sc scenarioCo Name: "User In DB", Login: userInDbName, } + _, err := sqlStore.CreateUser(context.Background(), cmd) require.NoError(t, err) diff --git a/pkg/services/live/live.go b/pkg/services/live/live.go index 0e5a86ef65f..927f0871538 100644 --- a/pkg/services/live/live.go +++ b/pkg/services/live/live.go @@ -15,6 +15,7 @@ import ( jsoniter "github.com/json-iterator/go" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/query" "github.com/centrifugal/centrifuge" @@ -46,7 +47,6 @@ import ( "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/tsdb/cloudwatch" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" "golang.org/x/sync/errgroup" @@ -66,14 +66,14 @@ type CoreGrafanaScope struct { } func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, routeRegister routing.RouteRegister, - logsService *cloudwatch.LogsService, pluginStore plugins.Store, cacheService *localcache.CacheService, + pluginStore plugins.Store, cacheService *localcache.CacheService, dataSourceCache datasources.CacheService, sqlStore *sqlstore.SQLStore, secretsService secrets.Service, - usageStatsService usagestats.Service, queryDataService *query.Service) (*GrafanaLive, error) { + usageStatsService usagestats.Service, queryDataService *query.Service, toggles featuremgmt.FeatureToggles) (*GrafanaLive, error) { g := &GrafanaLive{ Cfg: cfg, + Features: toggles, PluginContextProvider: plugCtxProvider, RouteRegister: routeRegister, - LogsService: logsService, pluginStore: pluginStore, CacheService: cacheService, DataSourceCache: dataSourceCache, @@ -176,7 +176,7 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r } g.ManagedStreamRunner = managedStreamRunner - if enabled := g.Cfg.FeatureToggles["live-pipeline"]; enabled { + if g.Features.IsEnabled(featuremgmt.FlagLivePipeline) { var builder pipeline.RuleBuilder if os.Getenv("GF_LIVE_DEV_BUILDER") != "" { builder = &pipeline.DevRuleBuilder{ @@ -393,8 +393,8 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r type GrafanaLive struct { PluginContextProvider *plugincontext.Provider Cfg *setting.Cfg + Features featuremgmt.FeatureToggles RouteRegister routing.RouteRegister - LogsService *cloudwatch.LogsService CacheService *localcache.CacheService DataSourceCache datasources.CacheService SQLStore *sqlstore.SQLStore @@ -895,13 +895,6 @@ func (g *GrafanaLive) handleGrafanaScope(_ *models.SignedInUser, namespace strin } func (g *GrafanaLive) handlePluginScope(ctx context.Context, _ *models.SignedInUser, namespace string) (models.ChannelHandlerFactory, error) { - // Temporary hack until we have a more generic solution later on - if namespace == "cloudwatch" { - return &cloudwatch.LogQueryRunnerSupplier{ - Publisher: g.Publish, - Service: g.LogsService, - }, nil - } streamHandler, err := g.getStreamPlugin(ctx, namespace) if err != nil { return nil, fmt.Errorf("can't find stream plugin: %s", namespace) diff --git a/pkg/services/login/authinfoservice/database.go b/pkg/services/login/authinfoservice/database.go index da5db5f975d..230d22abe26 100644 --- a/pkg/services/login/authinfoservice/database.go +++ b/pkg/services/login/authinfoservice/database.go @@ -38,6 +38,10 @@ func (s *Implementation) GetExternalUserInfoByLogin(ctx context.Context, query * } func (s *Implementation) GetAuthInfo(ctx context.Context, query *models.GetAuthInfoQuery) error { + if query.UserId == 0 && query.AuthId == "" { + return models.ErrUserNotFound + } + userAuth := &models.UserAuth{ UserId: query.UserId, AuthModule: query.AuthModule, diff --git a/pkg/services/login/authinfoservice/service.go b/pkg/services/login/authinfoservice/service.go index c5817b9b210..e1a13cea8c2 100644 --- a/pkg/services/login/authinfoservice/service.go +++ b/pkg/services/login/authinfoservice/service.go @@ -22,6 +22,10 @@ type Implementation struct { logger log.Logger } +type Service interface { + GetAuthInfo(ctx context.Context, query *models.GetAuthInfoQuery) error +} + func ProvideAuthInfoService(bus bus.Bus, store *sqlstore.SQLStore, userProtectionService login.UserProtectionService, secretsService secrets.Service) *Implementation { s := &Implementation{ diff --git a/pkg/services/ngalert/api/api_alertmanager_test.go b/pkg/services/ngalert/api/api_alertmanager_test.go index 7e9537a37a1..479dfb75828 100644 --- a/pkg/services/ngalert/api/api_alertmanager_test.go +++ b/pkg/services/ngalert/api/api_alertmanager_test.go @@ -261,7 +261,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager { }, // do not poll in tests. } - mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), log.New("testlogger")) + mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) require.NoError(t, err) t.Cleanup(cleanOrgDirectories(tmpDir, t)) err = mam.LoadAndSyncAlertmanagersForOrgs(context.Background()) diff --git a/pkg/services/ngalert/api/lotex_am.go b/pkg/services/ngalert/api/lotex_am.go index 7e5d14504fe..f626cfdb43e 100644 --- a/pkg/services/ngalert/api/lotex_am.go +++ b/pkg/services/ngalert/api/lotex_am.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/infra/log" @@ -59,7 +60,12 @@ func (am *LotexAM) withAMReq( extractor func(*response.NormalResponse) (interface{}, error), headers map[string]string, ) response.Response { - ds, err := am.DataProxy.DataSourceCache.GetDatasource(ctx.Req.Context(), ctx.ParamsInt64(":Recipient"), ctx.SignedInUser, ctx.SkipCache) + recipient, err := strconv.ParseInt(web.Params(ctx.Req)[":Recipient"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "Recipient is invalid", err) + } + + ds, err := am.DataProxy.DataSourceCache.GetDatasource(ctx.Req.Context(), recipient, ctx.SignedInUser, ctx.SkipCache) if err != nil { if errors.Is(err, models.ErrDataSourceAccessDenied) { return ErrResp(http.StatusForbidden, err, "Access denied to datasource") diff --git a/pkg/services/ngalert/api/lotex_prom.go b/pkg/services/ngalert/api/lotex_prom.go index 1f5ea483e76..d2c2efbed3d 100644 --- a/pkg/services/ngalert/api/lotex_prom.go +++ b/pkg/services/ngalert/api/lotex_prom.go @@ -3,11 +3,13 @@ package api import ( "fmt" "net/http" + "strconv" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/web" ) type promEndpoints struct { @@ -76,7 +78,12 @@ func (p *LotexProm) RouteGetRuleStatuses(ctx *models.ReqContext) response.Respon } func (p *LotexProm) getEndpoints(ctx *models.ReqContext) (*promEndpoints, error) { - ds, err := p.DataProxy.DataSourceCache.GetDatasource(ctx.Req.Context(), ctx.ParamsInt64(":Recipient"), ctx.SignedInUser, ctx.SkipCache) + recipient, err := strconv.ParseInt(web.Params(ctx.Req)[":Recipient"], 10, 64) + if err != nil { + return nil, fmt.Errorf("recipient is invalid") + } + + ds, err := p.DataProxy.DataSourceCache.GetDatasource(ctx.Req.Context(), recipient, ctx.SignedInUser, ctx.SkipCache) if err != nil { return nil, err } diff --git a/pkg/services/ngalert/api/lotex_ruler.go b/pkg/services/ngalert/api/lotex_ruler.go index 3342b07da1b..9cee9799cbb 100644 --- a/pkg/services/ngalert/api/lotex_ruler.go +++ b/pkg/services/ngalert/api/lotex_ruler.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/web" @@ -152,7 +153,12 @@ func (r *LotexRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apimo } func (r *LotexRuler) validateAndGetPrefix(ctx *models.ReqContext) (string, error) { - ds, err := r.DataProxy.DataSourceCache.GetDatasource(ctx.Req.Context(), ctx.ParamsInt64(":Recipient"), ctx.SignedInUser, ctx.SkipCache) + recipient, err := strconv.ParseInt(web.Params(ctx.Req)[":Recipient"], 10, 64) + if err != nil { + return "", fmt.Errorf("recipient is invalid") + } + + ds, err := r.DataProxy.DataSourceCache.GetDatasource(ctx.Req.Context(), recipient, ctx.SignedInUser, ctx.SkipCache) if err != nil { return "", err } diff --git a/pkg/services/ngalert/api/util.go b/pkg/services/ngalert/api/util.go index 0d8dc78f2c7..c1b14645850 100644 --- a/pkg/services/ngalert/api/util.go +++ b/pkg/services/ngalert/api/util.go @@ -105,7 +105,13 @@ func (p *AlertingProxy) withReq( } newCtx, resp := replacedResponseWriter(ctx) newCtx.Req = req - p.DataProxy.ProxyDatasourceRequestWithID(newCtx, ctx.ParamsInt64(":Recipient")) + + recipient, err := strconv.ParseInt(web.Params(ctx.Req)[":Recipient"], 10, 64) + if err != nil { + return ErrResp(http.StatusBadRequest, err, "Recipient is invalid") + } + + p.DataProxy.ProxyDatasourceRequestWithID(newCtx, recipient) status := resp.Status() if status >= 400 { @@ -259,7 +265,7 @@ func ErrResp(status int, err error, msg string, args ...interface{}) *response.N if msg != "" { err = errors.WithMessagef(err, msg, args...) } - return response.Error(status, "API error", err) + return response.Error(status, err.Error(), err) } // accessForbiddenResp creates a response of forbidden access. diff --git a/pkg/services/ngalert/metrics/ngalert.go b/pkg/services/ngalert/metrics/ngalert.go index ae59bfca25d..bb89cfc2b2b 100644 --- a/pkg/services/ngalert/metrics/ngalert.go +++ b/pkg/services/ngalert/metrics/ngalert.go @@ -45,10 +45,13 @@ type NGAlert struct { } type Scheduler struct { - Registerer prometheus.Registerer - EvalTotal *prometheus.CounterVec - EvalFailures *prometheus.CounterVec - EvalDuration *prometheus.SummaryVec + Registerer prometheus.Registerer + BehindSeconds prometheus.Gauge + EvalTotal *prometheus.CounterVec + EvalFailures *prometheus.CounterVec + EvalDuration *prometheus.SummaryVec + GetAlertRulesDuration prometheus.Histogram + SchedulePeriodicDuration prometheus.Histogram } type MultiOrgAlertmanager struct { @@ -120,6 +123,12 @@ func (moa *MultiOrgAlertmanager) GetOrCreateOrgRegistry(id int64) prometheus.Reg func newSchedulerMetrics(r prometheus.Registerer) *Scheduler { return &Scheduler{ Registerer: r, + BehindSeconds: promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "scheduler_behind_seconds", + Help: "The total number of seconds the scheduler is behind.", + }), // TODO: once rule groups support multiple rules, consider partitioning // on rule group as well as tenant, similar to loki|cortex. EvalTotal: promauto.With(r).NewCounterVec( @@ -152,6 +161,24 @@ func newSchedulerMetrics(r prometheus.Registerer) *Scheduler { }, []string{"org"}, ), + GetAlertRulesDuration: promauto.With(r).NewHistogram( + prometheus.HistogramOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "get_alert_rules_duration_seconds", + Help: "The time taken to get all alert rules.", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, + ), + SchedulePeriodicDuration: promauto.With(r).NewHistogram( + prometheus.HistogramOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "schedule_periodic_duration_seconds", + Help: "The time taken to run the scheduler.", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, + ), } } diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 8b96a84bec1..cb614c36241 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -21,6 +21,7 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/schedule" "github.com/grafana/grafana/pkg/services/ngalert/state" "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/sqlstore" @@ -40,19 +41,20 @@ const ( func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, routeRegister routing.RouteRegister, sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, expressionService *expr.Service, dataProxy *datasourceproxy.DataSourceProxyService, - quotaService *quota.QuotaService, secretsService secrets.Service, m *metrics.NGAlert) (*AlertNG, error) { + quotaService *quota.QuotaService, secretsService secrets.Service, notificationService notifications.Service, m *metrics.NGAlert) (*AlertNG, error) { ng := &AlertNG{ - Cfg: cfg, - DataSourceCache: dataSourceCache, - RouteRegister: routeRegister, - SQLStore: sqlStore, - KVStore: kvStore, - ExpressionService: expressionService, - DataProxy: dataProxy, - QuotaService: quotaService, - SecretsService: secretsService, - Metrics: m, - Log: log.New("ngalert"), + Cfg: cfg, + DataSourceCache: dataSourceCache, + RouteRegister: routeRegister, + SQLStore: sqlStore, + KVStore: kvStore, + ExpressionService: expressionService, + DataProxy: dataProxy, + QuotaService: quotaService, + SecretsService: secretsService, + Metrics: m, + NotificationService: notificationService, + Log: log.New("ngalert"), } if ng.IsDisabled() { @@ -68,19 +70,20 @@ func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, // AlertNG is the service for evaluating the condition of an alert definition. type AlertNG struct { - Cfg *setting.Cfg - DataSourceCache datasources.CacheService - RouteRegister routing.RouteRegister - SQLStore *sqlstore.SQLStore - KVStore kvstore.KVStore - ExpressionService *expr.Service - DataProxy *datasourceproxy.DataSourceProxyService - QuotaService *quota.QuotaService - SecretsService secrets.Service - Metrics *metrics.NGAlert - Log log.Logger - schedule schedule.ScheduleService - stateManager *state.Manager + Cfg *setting.Cfg + DataSourceCache datasources.CacheService + RouteRegister routing.RouteRegister + SQLStore *sqlstore.SQLStore + KVStore kvstore.KVStore + ExpressionService *expr.Service + DataProxy *datasourceproxy.DataSourceProxyService + QuotaService *quota.QuotaService + SecretsService secrets.Service + Metrics *metrics.NGAlert + NotificationService notifications.Service + Log log.Logger + schedule schedule.ScheduleService + stateManager *state.Manager // Alerting notification services MultiOrgAlertmanager *notifier.MultiOrgAlertmanager @@ -104,7 +107,7 @@ func (ng *AlertNG) init() error { decryptFn := ng.SecretsService.GetDecryptedValue multiOrgMetrics := ng.Metrics.GetMultiOrgAlertmanagerMetrics() - ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, log.New("ngalert.multiorg.alertmanager")) + ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager")) if err != nil { return err } diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index c7311ca3e0f..ec74131a01b 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -40,6 +40,7 @@ import ( ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels" "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/setting" ) @@ -86,10 +87,11 @@ type ClusterPeer interface { type Alertmanager struct { logger log.Logger - Settings *setting.Cfg - Store store.AlertingStore - fileStore *FileStore - Metrics *metrics.Alertmanager + Settings *setting.Cfg + Store store.AlertingStore + fileStore *FileStore + Metrics *metrics.Alertmanager + NotificationService notifications.Service notificationLog *nflog.Log marker types.Marker @@ -125,20 +127,21 @@ type Alertmanager struct { } func newAlertmanager(ctx context.Context, orgID int64, cfg *setting.Cfg, store store.AlertingStore, kvStore kvstore.KVStore, - peer ClusterPeer, decryptFn channels.GetDecryptedValueFn, m *metrics.Alertmanager) (*Alertmanager, error) { + peer ClusterPeer, decryptFn channels.GetDecryptedValueFn, ns notifications.Service, m *metrics.Alertmanager) (*Alertmanager, error) { am := &Alertmanager{ - Settings: cfg, - stopc: make(chan struct{}), - logger: log.New("alertmanager", "org", orgID), - marker: types.NewMarker(m.Registerer), - stageMetrics: notify.NewMetrics(m.Registerer), - dispatcherMetrics: dispatch.NewDispatcherMetrics(false, m.Registerer), - Store: store, - peer: peer, - peerTimeout: cfg.UnifiedAlerting.HAPeerTimeout, - Metrics: m, - orgID: orgID, - decryptFn: decryptFn, + Settings: cfg, + stopc: make(chan struct{}), + logger: log.New("alertmanager", "org", orgID), + marker: types.NewMarker(m.Registerer), + stageMetrics: notify.NewMetrics(m.Registerer), + dispatcherMetrics: dispatch.NewDispatcherMetrics(false, m.Registerer), + Store: store, + peer: peer, + peerTimeout: cfg.UnifiedAlerting.HAPeerTimeout, + Metrics: m, + NotificationService: ns, + orgID: orgID, + decryptFn: decryptFn, } am.fileStore = NewFileStore(am.orgID, kvStore, am.WorkingDirPath()) @@ -496,39 +499,39 @@ func (am *Alertmanager) buildReceiverIntegration(r *apimodels.PostableGrafanaRec ) switch r.Type { case "email": - n, err = channels.NewEmailNotifier(cfg, tmpl) // Email notifier already has a default template. + n, err = channels.NewEmailNotifier(cfg, am.NotificationService, tmpl) // Email notifier already has a default template. case "pagerduty": - n, err = channels.NewPagerdutyNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewPagerdutyNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "pushover": - n, err = channels.NewPushoverNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewPushoverNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "slack": n, err = channels.NewSlackNotifier(cfg, tmpl, am.decryptFn) case "telegram": - n, err = channels.NewTelegramNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewTelegramNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "victorops": - n, err = channels.NewVictoropsNotifier(cfg, tmpl) + n, err = channels.NewVictoropsNotifier(cfg, am.NotificationService, tmpl) case "teams": - n, err = channels.NewTeamsNotifier(cfg, tmpl) + n, err = channels.NewTeamsNotifier(cfg, am.NotificationService, tmpl) case "dingding": - n, err = channels.NewDingDingNotifier(cfg, tmpl) + n, err = channels.NewDingDingNotifier(cfg, am.NotificationService, tmpl) case "kafka": - n, err = channels.NewKafkaNotifier(cfg, tmpl) + n, err = channels.NewKafkaNotifier(cfg, am.NotificationService, tmpl) case "webhook": - n, err = channels.NewWebHookNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewWebHookNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "wecom": - n, err = channels.NewWeComNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewWeComNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "sensugo": - n, err = channels.NewSensuGoNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewSensuGoNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "discord": - n, err = channels.NewDiscordNotifier(cfg, tmpl) + n, err = channels.NewDiscordNotifier(cfg, am.NotificationService, tmpl) case "googlechat": - n, err = channels.NewGoogleChatNotifier(cfg, tmpl) + n, err = channels.NewGoogleChatNotifier(cfg, am.NotificationService, tmpl) case "LINE": - n, err = channels.NewLineNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewLineNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "threema": - n, err = channels.NewThreemaNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewThreemaNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "opsgenie": - n, err = channels.NewOpsgenieNotifier(cfg, tmpl, am.decryptFn) + n, err = channels.NewOpsgenieNotifier(cfg, am.NotificationService, tmpl, am.decryptFn) case "prometheus-alertmanager": n, err = channels.NewAlertmanagerNotifier(cfg, tmpl, am.decryptFn) default: diff --git a/pkg/services/ngalert/notifier/alertmanager_test.go b/pkg/services/ngalert/notifier/alertmanager_test.go index cd567285146..d3fa2bc1ea8 100644 --- a/pkg/services/ngalert/notifier/alertmanager_test.go +++ b/pkg/services/ngalert/notifier/alertmanager_test.go @@ -50,7 +50,7 @@ func setupAMTest(t *testing.T) *Alertmanager { kvStore := NewFakeKVStore(t) secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) decryptFn := secretsService.GetDecryptedValue - am, err := newAlertmanager(context.Background(), 1, cfg, s, kvStore, &NilPeer{}, decryptFn, m) + am, err := newAlertmanager(context.Background(), 1, cfg, s, kvStore, &NilPeer{}, decryptFn, nil, m) require.NoError(t, err) return am } diff --git a/pkg/services/ngalert/notifier/channels/dingding.go b/pkg/services/ngalert/notifier/channels/dingding.go index a2ac90b45a7..bca261e756c 100644 --- a/pkg/services/ngalert/notifier/channels/dingding.go +++ b/pkg/services/ngalert/notifier/channels/dingding.go @@ -9,15 +9,15 @@ import ( "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" ) const defaultDingdingMsgType = "link" // NewDingDingNotifier is the constructor for the Dingding notifier -func NewDingDingNotifier(model *NotificationChannelConfig, t *template.Template) (*DingDingNotifier, error) { +func NewDingDingNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template) (*DingDingNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -42,6 +42,7 @@ func NewDingDingNotifier(model *NotificationChannelConfig, t *template.Template) Message: model.Settings.Get("message").MustString(`{{ template "default.message" .}}`), log: log.New("alerting.notifier.dingding"), tmpl: t, + ns: ns, }, nil } @@ -52,6 +53,7 @@ type DingDingNotifier struct { URL string Message string tmpl *template.Template + ns notifications.WebhookSender log log.Logger } @@ -115,7 +117,7 @@ func (dd *DingDingNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo Body: string(body), } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := dd.ns.SendWebhookSync(ctx, cmd); err != nil { return false, fmt.Errorf("send notification to dingding: %w", err) } diff --git a/pkg/services/ngalert/notifier/channels/dingding_test.go b/pkg/services/ngalert/notifier/channels/dingding_test.go index 1f2db672d02..a868eca502e 100644 --- a/pkg/services/ngalert/notifier/channels/dingding_test.go +++ b/pkg/services/ngalert/notifier/channels/dingding_test.go @@ -11,9 +11,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" ) func TestDingdingNotifier(t *testing.T) { @@ -99,19 +97,14 @@ func TestDingdingNotifier(t *testing.T) { Settings: settingsJSON, } - pn, err := NewDingDingNotifier(m, tmpl) + webhookSender := mockNotificationService() + pn, err := NewDingDingNotifier(m, webhookSender, tmpl) if c.expInitError != "" { require.Equal(t, c.expInitError, err.Error()) return } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -127,7 +120,7 @@ func TestDingdingNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), body) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/discord.go b/pkg/services/ngalert/notifier/channels/discord.go index f710581683c..22cb9bd23b3 100644 --- a/pkg/services/ngalert/notifier/channels/discord.go +++ b/pkg/services/ngalert/notifier/channels/discord.go @@ -9,16 +9,17 @@ import ( "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/setting" ) type DiscordNotifier struct { *Base log log.Logger + ns notifications.WebhookSender tmpl *template.Template Content string AvatarURL string @@ -26,7 +27,7 @@ type DiscordNotifier struct { UseDiscordUsername bool } -func NewDiscordNotifier(model *NotificationChannelConfig, t *template.Template) (*DiscordNotifier, error) { +func NewDiscordNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template) (*DiscordNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -55,6 +56,7 @@ func NewDiscordNotifier(model *NotificationChannelConfig, t *template.Template) AvatarURL: avatarURL, WebhookURL: discordURL, log: log.New("alerting.notifier.discord"), + ns: ns, tmpl: t, UseDiscordUsername: useDiscordUsername, }, nil @@ -114,7 +116,7 @@ func (d DiscordNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, Body: string(body), } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := d.ns.SendWebhookSync(ctx, cmd); err != nil { d.log.Error("Failed to send notification to Discord", "error", err) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/discord_test.go b/pkg/services/ngalert/notifier/channels/discord_test.go index 3236554fa12..3a7efd27c22 100644 --- a/pkg/services/ngalert/notifier/channels/discord_test.go +++ b/pkg/services/ngalert/notifier/channels/discord_test.go @@ -11,9 +11,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" ) @@ -143,19 +141,14 @@ func TestDiscordNotifier(t *testing.T) { Settings: settingsJson, } - dn, err := NewDiscordNotifier(m, tmpl) + webhookSender := mockNotificationService() + dn, err := NewDiscordNotifier(m, webhookSender, tmpl) if c.expInitError != "" { require.Equal(t, c.expInitError, err.Error()) return } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := dn.Notify(ctx, c.alerts...) @@ -171,7 +164,7 @@ func TestDiscordNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), body) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/email.go b/pkg/services/ngalert/notifier/channels/email.go index 8228c058b8e..47d23d67243 100644 --- a/pkg/services/ngalert/notifier/channels/email.go +++ b/pkg/services/ngalert/notifier/channels/email.go @@ -8,9 +8,9 @@ import ( "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/util" ) @@ -22,12 +22,13 @@ type EmailNotifier struct { SingleEmail bool Message string log log.Logger + ns notifications.EmailSender tmpl *template.Template } // NewEmailNotifier is the constructor function // for the EmailNotifier. -func NewEmailNotifier(model *NotificationChannelConfig, t *template.Template) (*EmailNotifier, error) { +func NewEmailNotifier(model *NotificationChannelConfig, ns notifications.EmailSender, t *template.Template) (*EmailNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -54,6 +55,7 @@ func NewEmailNotifier(model *NotificationChannelConfig, t *template.Template) (* SingleEmail: singleEmail, Message: model.Settings.Get("message").MustString(), log: log.New("alerting.notifier.email"), + ns: ns, tmpl: t, }, nil } @@ -103,7 +105,7 @@ func (en *EmailNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, en.log.Warn("failed to template email message", "err", tmplErr.Error()) } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := en.ns.SendEmailCommandHandlerSync(ctx, cmd); err != nil { return false, err } diff --git a/pkg/services/ngalert/notifier/channels/email_test.go b/pkg/services/ngalert/notifier/channels/email_test.go index 089668baa82..42a738ba196 100644 --- a/pkg/services/ngalert/notifier/channels/email_test.go +++ b/pkg/services/ngalert/notifier/channels/email_test.go @@ -10,9 +10,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" ) func TestEmailNotifier(t *testing.T) { @@ -32,7 +30,7 @@ func TestEmailNotifier(t *testing.T) { Settings: settingsJSON, } - _, err := NewEmailNotifier(model, tmpl) + _, err := NewEmailNotifier(model, nil, tmpl) require.Error(t, err) }) @@ -44,25 +42,16 @@ func TestEmailNotifier(t *testing.T) { settingsJSON, err := simplejson.NewJson([]byte(json)) require.NoError(t, err) + emailSender := mockNotificationService() + emailNotifier, err := NewEmailNotifier(&NotificationChannelConfig{ Name: "ops", Type: "email", Settings: settingsJSON, - }, tmpl) + }, emailSender, tmpl) require.NoError(t, err) - expected := map[string]interface{}{} - bus.AddHandler("test", func(ctx context.Context, cmd *models.SendEmailCommandSync) error { - expected["subject"] = cmd.SendEmailCommand.Subject - expected["to"] = cmd.SendEmailCommand.To - expected["single_email"] = cmd.SendEmailCommand.SingleEmail - expected["template"] = cmd.SendEmailCommand.Template - expected["data"] = cmd.SendEmailCommand.Data - - return nil - }) - alerts := []*types.Alert{ { Alert: model.Alert{ @@ -76,6 +65,13 @@ func TestEmailNotifier(t *testing.T) { require.NoError(t, err) require.True(t, ok) + expected := map[string]interface{}{ + "subject": emailSender.EmailSync.Subject, + "to": emailSender.EmailSync.To, + "single_email": emailSender.EmailSync.SingleEmail, + "template": emailSender.EmailSync.Template, + "data": emailSender.EmailSync.Data, + } require.Equal(t, map[string]interface{}{ "subject": "[FIRING:1] (AlwaysFiring warning)", "to": []string{"someops@example.com", "somedev@example.com"}, diff --git a/pkg/services/ngalert/notifier/channels/googlechat.go b/pkg/services/ngalert/notifier/channels/googlechat.go index 98c8c01ae5c..f14c1f373e2 100644 --- a/pkg/services/ngalert/notifier/channels/googlechat.go +++ b/pkg/services/ngalert/notifier/channels/googlechat.go @@ -9,9 +9,9 @@ import ( "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/setting" ) @@ -21,11 +21,12 @@ type GoogleChatNotifier struct { *Base URL string log log.Logger + ns notifications.WebhookSender tmpl *template.Template content string } -func NewGoogleChatNotifier(model *NotificationChannelConfig, t *template.Template) (*GoogleChatNotifier, error) { +func NewGoogleChatNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template) (*GoogleChatNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -47,6 +48,7 @@ func NewGoogleChatNotifier(model *NotificationChannelConfig, t *template.Templat }), URL: url, log: log.New("alerting.notifier.googlechat"), + ns: ns, tmpl: t, content: content, }, nil @@ -137,7 +139,7 @@ func (gcn *GoogleChatNotifier) Notify(ctx context.Context, as ...*types.Alert) ( Body: string(body), } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := gcn.ns.SendWebhookSync(ctx, cmd); err != nil { gcn.log.Error("Failed to send Google Hangouts Chat alert", "error", err, "webhook", gcn.Name) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/googlechat_test.go b/pkg/services/ngalert/notifier/channels/googlechat_test.go index c260d95c591..15a70034dfe 100644 --- a/pkg/services/ngalert/notifier/channels/googlechat_test.go +++ b/pkg/services/ngalert/notifier/channels/googlechat_test.go @@ -12,9 +12,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" ) @@ -274,7 +272,8 @@ func TestGoogleChatNotifier(t *testing.T) { Settings: settingsJSON, } - pn, err := NewGoogleChatNotifier(m, tmpl) + webhookSender := mockNotificationService() + pn, err := NewGoogleChatNotifier(m, webhookSender, tmpl) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -282,12 +281,6 @@ func TestGoogleChatNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -303,7 +296,7 @@ func TestGoogleChatNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), body) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/kafka.go b/pkg/services/ngalert/notifier/channels/kafka.go index ad68265a3b2..65983dbb366 100644 --- a/pkg/services/ngalert/notifier/channels/kafka.go +++ b/pkg/services/ngalert/notifier/channels/kafka.go @@ -9,10 +9,10 @@ import ( "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" ) // KafkaNotifier is responsible for sending @@ -22,11 +22,12 @@ type KafkaNotifier struct { Endpoint string Topic string log log.Logger + ns notifications.WebhookSender tmpl *template.Template } // NewKafkaNotifier is the constructor function for the Kafka notifier. -func NewKafkaNotifier(model *NotificationChannelConfig, t *template.Template) (*KafkaNotifier, error) { +func NewKafkaNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template) (*KafkaNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -51,6 +52,7 @@ func NewKafkaNotifier(model *NotificationChannelConfig, t *template.Template) (* Endpoint: endpoint, Topic: topic, log: log.New("alerting.notifier.kafka"), + ns: ns, tmpl: t, }, nil } @@ -112,7 +114,7 @@ func (kn *KafkaNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, }, } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := kn.ns.SendWebhookSync(ctx, cmd); err != nil { kn.log.Error("Failed to send notification to Kafka", "error", err, "body", string(body)) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/kafka_test.go b/pkg/services/ngalert/notifier/channels/kafka_test.go index 173a30677f2..9e30bfebdd1 100644 --- a/pkg/services/ngalert/notifier/channels/kafka_test.go +++ b/pkg/services/ngalert/notifier/channels/kafka_test.go @@ -10,9 +10,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" ) func TestKafkaNotifier(t *testing.T) { @@ -117,7 +115,8 @@ func TestKafkaNotifier(t *testing.T) { Settings: settingsJSON, } - pn, err := NewKafkaNotifier(m, tmpl) + webhookSender := mockNotificationService() + pn, err := NewKafkaNotifier(m, webhookSender, tmpl) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -125,14 +124,6 @@ func TestKafkaNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - actUrl := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - actUrl = webhook.Url - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -145,8 +136,8 @@ func TestKafkaNotifier(t *testing.T) { require.NoError(t, err) require.True(t, ok) - require.Equal(t, c.expUrl, actUrl) - require.JSONEq(t, c.expMsg, body) + require.Equal(t, c.expUrl, webhookSender.Webhook.Url) + require.JSONEq(t, c.expMsg, webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/line.go b/pkg/services/ngalert/notifier/channels/line.go index 7b67b7e19eb..79f27180b43 100644 --- a/pkg/services/ngalert/notifier/channels/line.go +++ b/pkg/services/ngalert/notifier/channels/line.go @@ -6,9 +6,9 @@ import ( "net/url" "path" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" ) @@ -18,7 +18,7 @@ var ( ) // NewLineNotifier is the constructor for the LINE notifier -func NewLineNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*LineNotifier, error) { +func NewLineNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*LineNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -41,6 +41,7 @@ func NewLineNotifier(model *NotificationChannelConfig, t *template.Template, fn }), Token: token, log: log.New("alerting.notifier.line"), + ns: ns, tmpl: t, }, nil } @@ -51,6 +52,7 @@ type LineNotifier struct { *Base Token string log log.Logger + ns notifications.WebhookSender tmpl *template.Template } @@ -86,7 +88,7 @@ func (ln *LineNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, e Body: form.Encode(), } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := ln.ns.SendWebhookSync(ctx, cmd); err != nil { ln.log.Error("Failed to send notification to LINE", "error", err, "body", body) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/line_test.go b/pkg/services/ngalert/notifier/channels/line_test.go index c7311aec5a1..7d6a50cfd81 100644 --- a/pkg/services/ngalert/notifier/channels/line_test.go +++ b/pkg/services/ngalert/notifier/channels/line_test.go @@ -10,9 +10,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" ) @@ -92,9 +90,10 @@ func TestLineNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewLineNotifier(m, tmpl, decryptFn) + pn, err := NewLineNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -102,14 +101,6 @@ func TestLineNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - var headers map[string]string - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - headers = webhook.HttpHeader - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -122,8 +113,8 @@ func TestLineNotifier(t *testing.T) { require.NoError(t, err) require.True(t, ok) - require.Equal(t, c.expHeaders, headers) - require.Equal(t, c.expMsg, body) + require.Equal(t, c.expHeaders, webhookSender.Webhook.HttpHeader) + require.Equal(t, c.expMsg, webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/opsgenie.go b/pkg/services/ngalert/notifier/channels/opsgenie.go index 4a5c5fd1a05..c9067561f92 100644 --- a/pkg/services/ngalert/notifier/channels/opsgenie.go +++ b/pkg/services/ngalert/notifier/channels/opsgenie.go @@ -7,10 +7,10 @@ import ( "net/http" "sort" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -38,10 +38,11 @@ type OpsgenieNotifier struct { SendTagsAs string tmpl *template.Template log log.Logger + ns notifications.WebhookSender } // NewOpsgenieNotifier is the constructor for the Opsgenie notifier -func NewOpsgenieNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*OpsgenieNotifier, error) { +func NewOpsgenieNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*OpsgenieNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -81,6 +82,7 @@ func NewOpsgenieNotifier(model *NotificationChannelConfig, t *template.Template, SendTagsAs: sendTagsAs, tmpl: t, log: log.New("alerting.notifier." + model.Name), + ns: ns, }, nil } @@ -120,7 +122,7 @@ func (on *OpsgenieNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo }, } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := on.ns.SendWebhookSync(ctx, cmd); err != nil { return false, fmt.Errorf("send notification to Opsgenie: %w", err) } diff --git a/pkg/services/ngalert/notifier/channels/opsgenie_test.go b/pkg/services/ngalert/notifier/channels/opsgenie_test.go index a36453d9249..c45ad66ff6b 100644 --- a/pkg/services/ngalert/notifier/channels/opsgenie_test.go +++ b/pkg/services/ngalert/notifier/channels/opsgenie_test.go @@ -6,9 +6,7 @@ import ( "testing" "time" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -172,9 +170,11 @@ func TestOpsgenieNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() + webhookSender.Webhook.Body = "" secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewOpsgenieNotifier(m, tmpl, decryptFn) + pn, err := NewOpsgenieNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -182,12 +182,6 @@ func TestOpsgenieNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -202,9 +196,9 @@ func TestOpsgenieNotifier(t *testing.T) { if c.expMsg == "" { // No notification was expected. - require.Equal(t, "", body) + require.Equal(t, "", webhookSender.Webhook.Body) } else { - require.JSONEq(t, c.expMsg, body) + require.JSONEq(t, c.expMsg, webhookSender.Webhook.Body) } }) } diff --git a/pkg/services/ngalert/notifier/channels/pagerduty.go b/pkg/services/ngalert/notifier/channels/pagerduty.go index 2258b2ad0d7..51e5fd51347 100644 --- a/pkg/services/ngalert/notifier/channels/pagerduty.go +++ b/pkg/services/ngalert/notifier/channels/pagerduty.go @@ -6,9 +6,9 @@ import ( "fmt" "os" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -37,10 +37,11 @@ type PagerdutyNotifier struct { Summary string tmpl *template.Template log log.Logger + ns notifications.WebhookSender } // NewPagerdutyNotifier is the constructor for the PagerDuty notifier -func NewPagerdutyNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*PagerdutyNotifier, error) { +func NewPagerdutyNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*PagerdutyNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -75,6 +76,7 @@ func NewPagerdutyNotifier(model *NotificationChannelConfig, t *template.Template Summary: model.Settings.Get("summary").MustString(DefaultMessageTitleEmbed), tmpl: t, log: log.New("alerting.notifier." + model.Name), + ns: ns, }, nil } @@ -105,7 +107,7 @@ func (pn *PagerdutyNotifier) Notify(ctx context.Context, as ...*types.Alert) (bo "Content-Type": "application/json", }, } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := pn.ns.SendWebhookSync(ctx, cmd); err != nil { return false, fmt.Errorf("send notification to Pagerduty: %w", err) } diff --git a/pkg/services/ngalert/notifier/channels/pagerduty_test.go b/pkg/services/ngalert/notifier/channels/pagerduty_test.go index 1cc899b9581..3703b83910f 100644 --- a/pkg/services/ngalert/notifier/channels/pagerduty_test.go +++ b/pkg/services/ngalert/notifier/channels/pagerduty_test.go @@ -7,9 +7,7 @@ import ( "os" "testing" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -138,9 +136,10 @@ func TestPagerdutyNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewPagerdutyNotifier(m, tmpl, decryptFn) + pn, err := NewPagerdutyNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -148,12 +147,6 @@ func TestPagerdutyNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -169,7 +162,7 @@ func TestPagerdutyNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), body) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/pushover.go b/pkg/services/ngalert/notifier/channels/pushover.go index 4c574604552..f684ba8ceaa 100644 --- a/pkg/services/ngalert/notifier/channels/pushover.go +++ b/pkg/services/ngalert/notifier/channels/pushover.go @@ -7,9 +7,9 @@ import ( "mime/multipart" "strconv" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" @@ -36,10 +36,11 @@ type PushoverNotifier struct { Message string tmpl *template.Template log log.Logger + ns notifications.WebhookSender } // NewSlackNotifier is the constructor for the Slack notifier -func NewPushoverNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*PushoverNotifier, error) { +func NewPushoverNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*PushoverNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -92,6 +93,7 @@ func NewPushoverNotifier(model *NotificationChannelConfig, t *template.Template, Message: model.Settings.Get("message").MustString(`{{ template "default.message" .}}`), tmpl: t, log: log.New("alerting.notifier.pushover"), + ns: ns, }, nil } @@ -110,7 +112,7 @@ func (pn *PushoverNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo Body: uploadBody.String(), } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := pn.ns.SendWebhookSync(ctx, cmd); err != nil { pn.log.Error("Failed to send pushover notification", "error", err, "webhook", pn.Name) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/pushover_test.go b/pkg/services/ngalert/notifier/channels/pushover_test.go index 2f5a0e4c019..2b81a43079d 100644 --- a/pkg/services/ngalert/notifier/channels/pushover_test.go +++ b/pkg/services/ngalert/notifier/channels/pushover_test.go @@ -11,9 +11,7 @@ import ( "strings" "testing" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -146,9 +144,10 @@ func TestPushoverNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewPushoverNotifier(m, tmpl, decryptFn) + pn, err := NewPushoverNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -156,12 +155,6 @@ func TestPushoverNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -174,7 +167,7 @@ func TestPushoverNotifier(t *testing.T) { require.NoError(t, err) require.True(t, ok) - bodyReader := multipart.NewReader(strings.NewReader(body), boundary) + bodyReader := multipart.NewReader(strings.NewReader(webhookSender.Webhook.Body), boundary) for { part, err := bodyReader.NextPart() if part == nil || errors.Is(err, io.EOF) { diff --git a/pkg/services/ngalert/notifier/channels/sensugo.go b/pkg/services/ngalert/notifier/channels/sensugo.go index 5bb061703d6..c6e6a4b05e2 100644 --- a/pkg/services/ngalert/notifier/channels/sensugo.go +++ b/pkg/services/ngalert/notifier/channels/sensugo.go @@ -6,9 +6,9 @@ import ( "fmt" "strings" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" @@ -17,6 +17,7 @@ import ( type SensuGoNotifier struct { *Base log log.Logger + ns notifications.WebhookSender tmpl *template.Template URL string @@ -29,7 +30,7 @@ type SensuGoNotifier struct { } // NewSensuGoNotifier is the constructor for the SensuGo notifier -func NewSensuGoNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*SensuGoNotifier, error) { +func NewSensuGoNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*SensuGoNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -63,6 +64,7 @@ func NewSensuGoNotifier(model *NotificationChannelConfig, t *template.Template, APIKey: apikey, Message: model.Settings.Get("message").MustString(`{{ template "default.message" .}}`), log: log.New("alerting.notifier.sensugo"), + ns: ns, tmpl: t, }, nil } @@ -145,7 +147,7 @@ func (sn *SensuGoNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool "Authorization": fmt.Sprintf("Key %s", sn.APIKey), }, } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := sn.ns.SendWebhookSync(ctx, cmd); err != nil { sn.log.Error("Failed to send Sensu Go event", "error", err, "sensugo", sn.Name) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/sensugo_test.go b/pkg/services/ngalert/notifier/channels/sensugo_test.go index 5c63e2c0fad..2277f09aed5 100644 --- a/pkg/services/ngalert/notifier/channels/sensugo_test.go +++ b/pkg/services/ngalert/notifier/channels/sensugo_test.go @@ -7,9 +7,7 @@ import ( "testing" "time" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -146,9 +144,10 @@ func TestSensuGoNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - sn, err := NewSensuGoNotifier(m, tmpl, decryptFn) + sn, err := NewSensuGoNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -156,12 +155,6 @@ func TestSensuGoNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := sn.Notify(ctx, c.alerts...) @@ -177,7 +170,7 @@ func TestSensuGoNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), body) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/teams.go b/pkg/services/ngalert/notifier/channels/teams.go index e3f0eec926e..509b6e10844 100644 --- a/pkg/services/ngalert/notifier/channels/teams.go +++ b/pkg/services/ngalert/notifier/channels/teams.go @@ -8,9 +8,9 @@ import ( "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" ) // TeamsNotifier is responsible for sending @@ -21,10 +21,11 @@ type TeamsNotifier struct { Message string tmpl *template.Template log log.Logger + ns notifications.WebhookSender } // NewTeamsNotifier is the constructor for Teams notifier. -func NewTeamsNotifier(model *NotificationChannelConfig, t *template.Template) (*TeamsNotifier, error) { +func NewTeamsNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template) (*TeamsNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -45,6 +46,7 @@ func NewTeamsNotifier(model *NotificationChannelConfig, t *template.Template) (* URL: u, Message: model.Settings.Get("message").MustString(`{{ template "teams.default.message" .}}`), log: log.New("alerting.notifier.teams"), + ns: ns, tmpl: t, }, nil } @@ -97,7 +99,7 @@ func (tn *TeamsNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, } cmd := &models.SendWebhookSync{Url: u, Body: string(b)} - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := tn.ns.SendWebhookSync(ctx, cmd); err != nil { return false, errors.Wrap(err, "send notification to Teams") } diff --git a/pkg/services/ngalert/notifier/channels/teams_test.go b/pkg/services/ngalert/notifier/channels/teams_test.go index b0e57f63d35..92bf8e15fdd 100644 --- a/pkg/services/ngalert/notifier/channels/teams_test.go +++ b/pkg/services/ngalert/notifier/channels/teams_test.go @@ -11,9 +11,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" ) func TestTeamsNotifier(t *testing.T) { @@ -123,7 +121,8 @@ func TestTeamsNotifier(t *testing.T) { Settings: settingsJSON, } - pn, err := NewTeamsNotifier(m, tmpl) + webhookSender := mockNotificationService() + pn, err := NewTeamsNotifier(m, webhookSender, tmpl) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -131,12 +130,6 @@ func TestTeamsNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -152,7 +145,7 @@ func TestTeamsNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), body) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/telegram.go b/pkg/services/ngalert/notifier/channels/telegram.go index 301bed12fa7..a9d6e75b1a4 100644 --- a/pkg/services/ngalert/notifier/channels/telegram.go +++ b/pkg/services/ngalert/notifier/channels/telegram.go @@ -6,9 +6,9 @@ import ( "fmt" "mime/multipart" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" ) @@ -25,11 +25,12 @@ type TelegramNotifier struct { ChatID string Message string log log.Logger + ns notifications.WebhookSender tmpl *template.Template } // NewTelegramNotifier is the constructor for the Telegram notifier -func NewTelegramNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*TelegramNotifier, error) { +func NewTelegramNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*TelegramNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -62,6 +63,7 @@ func NewTelegramNotifier(model *NotificationChannelConfig, t *template.Template, Message: message, tmpl: t, log: log.New("alerting.notifier.telegram"), + ns: ns, }, nil } @@ -109,7 +111,7 @@ func (tn *TelegramNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo }, } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := tn.ns.SendWebhookSync(ctx, cmd); err != nil { tn.log.Error("Failed to send webhook", "error", err, "webhook", tn.Name) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/telegram_test.go b/pkg/services/ngalert/notifier/channels/telegram_test.go index 72714e75093..1e7d5f24a20 100644 --- a/pkg/services/ngalert/notifier/channels/telegram_test.go +++ b/pkg/services/ngalert/notifier/channels/telegram_test.go @@ -98,9 +98,10 @@ func TestTelegramNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewTelegramNotifier(m, tmpl, decryptFn) + pn, err := NewTelegramNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) diff --git a/pkg/services/ngalert/notifier/channels/testing.go b/pkg/services/ngalert/notifier/channels/testing.go index 0f22f81ebbb..3fac1346bc6 100644 --- a/pkg/services/ngalert/notifier/channels/testing.go +++ b/pkg/services/ngalert/notifier/channels/testing.go @@ -1,6 +1,11 @@ package channels -import "time" +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/models" +) // mockTimeNow replaces function timeNow to return constant time. // It returns a function that resets the variable back to its original value. @@ -21,3 +26,25 @@ func mockTimeNow(constTime time.Time) func() { func resetTimeNow() { timeNow = time.Now } + +type notificationServiceMock struct { + Webhook models.SendWebhookSync + EmailSync models.SendEmailCommandSync + Emailx models.SendEmailCommand + ShouldError error +} + +func (ns *notificationServiceMock) SendWebhookSync(ctx context.Context, cmd *models.SendWebhookSync) error { + ns.Webhook = *cmd + return ns.ShouldError +} +func (ns *notificationServiceMock) SendEmailCommandHandlerSync(ctx context.Context, cmd *models.SendEmailCommandSync) error { + ns.EmailSync = *cmd + return ns.ShouldError +} +func (ns *notificationServiceMock) SendEmailCommandHandler(ctx context.Context, cmd *models.SendEmailCommand) error { + ns.Emailx = *cmd + return ns.ShouldError +} + +func mockNotificationService() *notificationServiceMock { return ¬ificationServiceMock{} } diff --git a/pkg/services/ngalert/notifier/channels/threema.go b/pkg/services/ngalert/notifier/channels/threema.go index 7bfdaa46816..8fd6bbc6b1b 100644 --- a/pkg/services/ngalert/notifier/channels/threema.go +++ b/pkg/services/ngalert/notifier/channels/threema.go @@ -7,9 +7,9 @@ import ( "path" "strings" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" @@ -27,11 +27,12 @@ type ThreemaNotifier struct { RecipientID string APISecret string log log.Logger + ns notifications.WebhookSender tmpl *template.Template } // NewThreemaNotifier is the constructor for the Threema notifier -func NewThreemaNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*ThreemaNotifier, error) { +func NewThreemaNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*ThreemaNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -74,6 +75,7 @@ func NewThreemaNotifier(model *NotificationChannelConfig, t *template.Template, RecipientID: recipientID, APISecret: apiSecret, log: log.New("alerting.notifier.threema"), + ns: ns, tmpl: t, }, nil } @@ -119,7 +121,7 @@ func (tn *ThreemaNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool "Content-Type": "application/x-www-form-urlencoded", }, } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := tn.ns.SendWebhookSync(ctx, cmd); err != nil { tn.log.Error("Failed to send threema notification", "error", err, "webhook", tn.Name) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/threema_test.go b/pkg/services/ngalert/notifier/channels/threema_test.go index 024db4f9fa7..01e4f530411 100644 --- a/pkg/services/ngalert/notifier/channels/threema_test.go +++ b/pkg/services/ngalert/notifier/channels/threema_test.go @@ -5,9 +5,7 @@ import ( "net/url" "testing" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -110,9 +108,10 @@ func TestThreemaNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewThreemaNotifier(m, tmpl, decryptFn) + pn, err := NewThreemaNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -120,12 +119,6 @@ func TestThreemaNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -138,7 +131,7 @@ func TestThreemaNotifier(t *testing.T) { require.NoError(t, err) require.True(t, ok) - require.Equal(t, c.expMsg, body) + require.Equal(t, c.expMsg, webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/channels/victorops.go b/pkg/services/ngalert/notifier/channels/victorops.go index 72e1f362833..aba0167965f 100644 --- a/pkg/services/ngalert/notifier/channels/victorops.go +++ b/pkg/services/ngalert/notifier/channels/victorops.go @@ -10,10 +10,10 @@ import ( "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/setting" ) @@ -27,7 +27,7 @@ const ( // NewVictoropsNotifier creates an instance of VictoropsNotifier that // handles posting notifications to Victorops REST API -func NewVictoropsNotifier(model *NotificationChannelConfig, t *template.Template) (*VictoropsNotifier, error) { +func NewVictoropsNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template) (*VictoropsNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -48,6 +48,7 @@ func NewVictoropsNotifier(model *NotificationChannelConfig, t *template.Template URL: url, MessageType: model.Settings.Get("messageType").MustString(), log: log.New("alerting.notifier.victorops"), + ns: ns, tmpl: t, }, nil } @@ -60,6 +61,7 @@ type VictoropsNotifier struct { URL string MessageType string log log.Logger + ns notifications.WebhookSender tmpl *template.Template } @@ -109,7 +111,7 @@ func (vn *VictoropsNotifier) Notify(ctx context.Context, as ...*types.Alert) (bo Body: string(b), } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := vn.ns.SendWebhookSync(ctx, cmd); err != nil { vn.log.Error("Failed to send Victorops notification", "error", err, "webhook", vn.Name) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/victorops_test.go b/pkg/services/ngalert/notifier/channels/victorops_test.go index 732307decc8..2db53055964 100644 --- a/pkg/services/ngalert/notifier/channels/victorops_test.go +++ b/pkg/services/ngalert/notifier/channels/victorops_test.go @@ -11,9 +11,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" ) @@ -95,7 +93,8 @@ func TestVictoropsNotifier(t *testing.T) { Settings: settingsJSON, } - pn, err := NewVictoropsNotifier(m, tmpl) + webhookSender := mockNotificationService() + pn, err := NewVictoropsNotifier(m, webhookSender, tmpl) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -103,12 +102,6 @@ func TestVictoropsNotifier(t *testing.T) { } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -122,12 +115,12 @@ func TestVictoropsNotifier(t *testing.T) { require.True(t, ok) // Remove the non-constant timestamp - j, err := simplejson.NewJson([]byte(body)) + j, err := simplejson.NewJson([]byte(webhookSender.Webhook.Body)) require.NoError(t, err) j.Del("timestamp") b, err := j.MarshalJSON() require.NoError(t, err) - body = string(b) + body := string(b) expJson, err := json.Marshal(c.expMsg) require.NoError(t, err) diff --git a/pkg/services/ngalert/notifier/channels/webhook.go b/pkg/services/ngalert/notifier/channels/webhook.go index 1969b21c098..5422816d7b8 100644 --- a/pkg/services/ngalert/notifier/channels/webhook.go +++ b/pkg/services/ngalert/notifier/channels/webhook.go @@ -4,9 +4,9 @@ import ( "context" "encoding/json" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" @@ -23,13 +23,14 @@ type WebhookNotifier struct { HTTPMethod string MaxAlerts int log log.Logger + ns notifications.WebhookSender tmpl *template.Template orgID int64 } // NewWebHookNotifier is the constructor for // the WebHook notifier. -func NewWebHookNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*WebhookNotifier, error) { +func NewWebHookNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*WebhookNotifier, error) { if model.Settings == nil { return nil, receiverInitError{Cfg: *model, Reason: "no settings supplied"} } @@ -55,6 +56,7 @@ func NewWebHookNotifier(model *NotificationChannelConfig, t *template.Template, HTTPMethod: model.Settings.Get("httpMethod").MustString("POST"), MaxAlerts: model.Settings.Get("maxAlerts").MustInt(0), log: log.New("alerting.notifier.webhook"), + ns: ns, tmpl: t, }, nil } @@ -118,7 +120,7 @@ func (wn *WebhookNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool HttpMethod: wn.HTTPMethod, } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := wn.ns.SendWebhookSync(ctx, cmd); err != nil { return false, err } diff --git a/pkg/services/ngalert/notifier/channels/webhook_test.go b/pkg/services/ngalert/notifier/channels/webhook_test.go index 564959e1168..e043abc4757 100644 --- a/pkg/services/ngalert/notifier/channels/webhook_test.go +++ b/pkg/services/ngalert/notifier/channels/webhook_test.go @@ -6,9 +6,7 @@ import ( "net/url" "testing" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -192,9 +190,10 @@ func TestWebhookNotifier(t *testing.T) { SecureSettings: secureSettings, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewWebHookNotifier(m, tmpl, decryptFn) + pn, err := NewWebHookNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Error(t, err) require.Equal(t, c.expInitError, err.Error()) @@ -202,12 +201,6 @@ func TestWebhookNotifier(t *testing.T) { } require.NoError(t, err) - var payload *models.SendWebhookSync - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - payload = webhook - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ctx = notify.WithReceiverName(ctx, "my_receiver") @@ -224,11 +217,11 @@ func TestWebhookNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), payload.Body) - require.Equal(t, c.expUrl, payload.Url) - require.Equal(t, c.expUsername, payload.User) - require.Equal(t, c.expPassword, payload.Password) - require.Equal(t, c.expHttpMethod, payload.HttpMethod) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) + require.Equal(t, c.expUrl, webhookSender.Webhook.Url) + require.Equal(t, c.expUsername, webhookSender.Webhook.User) + require.Equal(t, c.expPassword, webhookSender.Webhook.Password) + require.Equal(t, c.expHttpMethod, webhookSender.Webhook.HttpMethod) }) } } diff --git a/pkg/services/ngalert/notifier/channels/wecom.go b/pkg/services/ngalert/notifier/channels/wecom.go index 37ee36f1a2d..6d135f9ddc5 100644 --- a/pkg/services/ngalert/notifier/channels/wecom.go +++ b/pkg/services/ngalert/notifier/channels/wecom.go @@ -7,14 +7,14 @@ import ( "github.com/prometheus/alertmanager/types" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/template" ) // NewWeComNotifier is the constructor for WeCom notifier. -func NewWeComNotifier(model *NotificationChannelConfig, t *template.Template, fn GetDecryptedValueFn) (*WeComNotifier, error) { +func NewWeComNotifier(model *NotificationChannelConfig, ns notifications.WebhookSender, t *template.Template, fn GetDecryptedValueFn) (*WeComNotifier, error) { url := fn(context.Background(), model.SecureSettings, "url", model.Settings.Get("url").MustString()) if url == "" { @@ -31,6 +31,7 @@ func NewWeComNotifier(model *NotificationChannelConfig, t *template.Template, fn }), URL: url, log: log.New("alerting.notifier.wecom"), + ns: ns, Message: model.Settings.Get("message").MustString(`{{ template "default.message" .}}`), tmpl: t, }, nil @@ -43,6 +44,7 @@ type WeComNotifier struct { Message string tmpl *template.Template log log.Logger + ns notifications.WebhookSender } // Notify send an alert notification to WeCom. @@ -74,7 +76,7 @@ func (w *WeComNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, e Body: string(body), } - if err := bus.Dispatch(ctx, cmd); err != nil { + if err := w.ns.SendWebhookSync(ctx, cmd); err != nil { w.log.Error("failed to send WeCom webhook", "error", err, "notification", w.Name) return false, err } diff --git a/pkg/services/ngalert/notifier/channels/wecom_test.go b/pkg/services/ngalert/notifier/channels/wecom_test.go index eb0b78774ec..46676e9b656 100644 --- a/pkg/services/ngalert/notifier/channels/wecom_test.go +++ b/pkg/services/ngalert/notifier/channels/wecom_test.go @@ -14,9 +14,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" ) func TestWeComNotifier(t *testing.T) { @@ -96,21 +94,16 @@ func TestWeComNotifier(t *testing.T) { Settings: settingsJSON, } + webhookSender := mockNotificationService() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - pn, err := NewWeComNotifier(m, tmpl, decryptFn) + pn, err := NewWeComNotifier(m, webhookSender, tmpl, decryptFn) if c.expInitError != "" { require.Equal(t, c.expInitError, err.Error()) return } require.NoError(t, err) - body := "" - bus.AddHandler("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { - body = webhook.Body - return nil - }) - ctx := notify.WithGroupKey(context.Background(), "alertname") ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) ok, err := pn.Notify(ctx, c.alerts...) @@ -126,7 +119,7 @@ func TestWeComNotifier(t *testing.T) { expBody, err := json.Marshal(c.expMsg) require.NoError(t, err) - require.JSONEq(t, string(expBody), body) + require.JSONEq(t, string(expBody), webhookSender.Webhook.Body) }) } } diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index 2dec096874b..c2e48151eba 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -10,6 +10,7 @@ import ( "time" "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels" + "github.com/grafana/grafana/pkg/services/notifications" "github.com/prometheus/alertmanager/cluster" "github.com/prometheus/client_golang/prometheus" @@ -45,10 +46,12 @@ type MultiOrgAlertmanager struct { decryptFn channels.GetDecryptedValueFn metrics *metrics.MultiOrgAlertmanager + ns notifications.Service } func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore, - kvStore kvstore.KVStore, decryptFn channels.GetDecryptedValueFn, m *metrics.MultiOrgAlertmanager, l log.Logger, + kvStore kvstore.KVStore, decryptFn channels.GetDecryptedValueFn, m *metrics.MultiOrgAlertmanager, + ns notifications.Service, l log.Logger, ) (*MultiOrgAlertmanager, error) { moa := &MultiOrgAlertmanager{ logger: l, @@ -59,6 +62,7 @@ func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, kvStore: kvStore, decryptFn: decryptFn, metrics: m, + ns: ns, } clusterLogger := l.New("component", "cluster") @@ -170,7 +174,7 @@ func (moa *MultiOrgAlertmanager) SyncAlertmanagersForOrgs(ctx context.Context, o // To export them, we need to translate the metrics from each individual registry and, // then aggregate them on the main registry. m := metrics.NewAlertmanagerMetrics(moa.metrics.GetOrCreateOrgRegistry(orgID)) - am, err := newAlertmanager(ctx, orgID, moa.settings, moa.configStore, moa.kvStore, moa.peer, moa.decryptFn, m) + am, err := newAlertmanager(ctx, orgID, moa.settings, moa.configStore, moa.kvStore, moa.peer, moa.decryptFn, moa.ns, m) if err != nil { moa.logger.Error("unable to create Alertmanager for org", "org", orgID, "err", err) } diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go index 228e6c55d68..9b270bd9b70 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go @@ -46,7 +46,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) { DisabledOrgs: map[int64]struct{}{5: {}}, }, // do not poll in tests. } - mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), log.New("testlogger")) + mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) require.NoError(t, err) ctx := context.Background() @@ -173,7 +173,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T) DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(), }, // do not poll in tests. } - mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), log.New("testlogger")) + mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) require.NoError(t, err) ctx := context.Background() @@ -224,7 +224,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) { decryptFn := secretsService.GetDecryptedValue reg := prometheus.NewPedanticRegistry() m := metrics.NewNGAlert(reg) - mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), log.New("testlogger")) + mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) require.NoError(t, err) ctx := context.Background() diff --git a/pkg/services/ngalert/schedule/fetcher.go b/pkg/services/ngalert/schedule/fetcher.go index 1bccd94d5fc..36c36b2acb9 100644 --- a/pkg/services/ngalert/schedule/fetcher.go +++ b/pkg/services/ngalert/schedule/fetcher.go @@ -1,10 +1,17 @@ package schedule import ( + "time" + "github.com/grafana/grafana/pkg/services/ngalert/models" ) -func (sch *schedule) fetchAllDetails(disabledOrgs []int64) []*models.AlertRule { +func (sch *schedule) getAlertRules(disabledOrgs []int64) []*models.AlertRule { + start := time.Now() + defer func() { + sch.metrics.GetAlertRulesDuration.Observe(time.Since(start).Seconds()) + }() + q := models.ListAlertRulesQuery{ ExcludeOrgs: disabledOrgs, } diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index 5ceef7a892e..569bcec2965 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -175,7 +175,7 @@ func (sch *schedule) Run(ctx context.Context) error { go func() { defer wg.Done() - if err := sch.ruleEvaluationLoop(ctx); err != nil { + if err := sch.schedulePeriodic(ctx); err != nil { sch.log.Error("failure while running the rule evaluation loop", "err", err) } }() @@ -352,17 +352,21 @@ func (sch *schedule) adminConfigSync(ctx context.Context) error { } } -func (sch *schedule) ruleEvaluationLoop(ctx context.Context) error { +func (sch *schedule) schedulePeriodic(ctx context.Context) error { dispatcherGroup, ctx := errgroup.WithContext(ctx) for { select { case tick := <-sch.heartbeat.C: + start := time.Now() + sch.metrics.BehindSeconds.Set(start.Sub(tick).Seconds()) + tickNum := tick.Unix() / int64(sch.baseInterval.Seconds()) disabledOrgs := make([]int64, 0, len(sch.disabledOrgs)) for disabledOrg := range sch.disabledOrgs { disabledOrgs = append(disabledOrgs, disabledOrg) } - alertRules := sch.fetchAllDetails(disabledOrgs) + + alertRules := sch.getAlertRules(disabledOrgs) sch.log.Debug("alert rules fetched", "count", len(alertRules), "disabled_orgs", disabledOrgs) // registeredDefinitions is a map used for finding deleted alert rules @@ -433,6 +437,8 @@ func (sch *schedule) ruleEvaluationLoop(ctx context.Context) error { for key := range registeredDefinitions { sch.DeleteAlertRule(key) } + + sch.metrics.SchedulePeriodicDuration.Observe(time.Since(start).Seconds()) case <-ctx.Done(): waitErr := dispatcherGroup.Wait() diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index a52fb7330b7..b3b4b991694 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -923,7 +923,7 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac m := metrics.NewNGAlert(registry) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue - moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}, ¬ifier.FakeKVStore{}, decryptFn, nil, log.New("testlogger")) + moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}, ¬ifier.FakeKVStore{}, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger")) require.NoError(t, err) schedCfg := SchedulerCfg{ diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index ee2b4fbeabe..9149bc31bb6 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -79,7 +79,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, LastEvaluationTime: evaluationTime, @@ -133,7 +133,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, LastEvaluationTime: evaluationTime, @@ -156,7 +156,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime, @@ -213,12 +213,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, LastEvaluationTime: evaluationTime.Add(1 * time.Minute), @@ -274,12 +274,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(1 * time.Minute), @@ -346,17 +346,17 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(80 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(80 * time.Second), @@ -440,22 +440,22 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(20 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(30 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(40 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(30 * time.Second), @@ -531,22 +531,22 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(20 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(30 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime, @@ -605,12 +605,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -669,12 +669,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime, @@ -733,12 +733,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -797,12 +797,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -860,7 +860,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -924,7 +924,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -991,7 +991,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime.Add(20 * time.Second), EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: time.Time{}, @@ -1049,12 +1049,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -1114,12 +1114,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -1179,12 +1179,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Error, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -1258,12 +1258,12 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationState: eval.Error, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(10 * time.Second), @@ -1339,22 +1339,22 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(30 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(40 * time.Second), EvaluationState: eval.Error, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(70 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(70 * time.Second), @@ -1431,22 +1431,22 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(30 * time.Second), EvaluationState: eval.Alerting, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(40 * time.Second), EvaluationState: eval.Error, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, { EvaluationTime: evaluationTime.Add(50 * time.Second), EvaluationState: eval.NoData, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, StartsAt: evaluationTime.Add(30 * time.Second), @@ -1498,7 +1498,7 @@ func TestProcessEvalResults(t *testing.T) { { EvaluationTime: evaluationTime, EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, LastEvaluationTime: evaluationTime, @@ -1603,7 +1603,7 @@ func TestStaleResultsHandler(t *testing.T) { { EvaluationTime: evaluationTime.Add(3 * time.Minute), EvaluationState: eval.Normal, - Values: make(map[string]state.EvaluationValue), + Values: make(map[string]*float64), }, }, LastEvaluationTime: evaluationTime.Add(3 * time.Minute), diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index ea516c1ab41..02f1f41440c 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -35,23 +35,14 @@ type Evaluation struct { // Values contains the RefID and value of reduce and math expressions. // It does not contain values for classic conditions as the values // in classic conditions do not have a RefID. - Values map[string]EvaluationValue -} - -// EvaluationValue contains the labels and value for a RefID in an evaluation. -type EvaluationValue struct { - Labels data.Labels - Value *float64 + Values map[string]*float64 } // NewEvaluationValues returns the labels and values for each RefID in the capture. -func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]EvaluationValue { - result := make(map[string]EvaluationValue, len(m)) +func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]*float64 { + result := make(map[string]*float64, len(m)) for k, v := range m { - result[k] = EvaluationValue{ - Labels: v.Labels, - Value: v.Value, - } + result[k] = v.Value } return result } diff --git a/pkg/services/ngalert/tests/util.go b/pkg/services/ngalert/tests/util.go index 7d70fba042b..d5fc939e3fa 100644 --- a/pkg/services/ngalert/tests/util.go +++ b/pkg/services/ngalert/tests/util.go @@ -39,7 +39,7 @@ func SetupTestEnv(t *testing.T, baseInterval time.Duration) (*ngalert.AlertNG, * secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) ng, err := ngalert.ProvideService( cfg, nil, routing.NewRouteRegister(), sqlStore, - nil, nil, nil, nil, secretsService, m, + nil, nil, nil, nil, secretsService, nil, m, ) require.NoError(t, err) return ng, &store.DBstore{ diff --git a/pkg/services/notifications/mailer.go b/pkg/services/notifications/mailer.go index f278c865300..1ba3b85e40a 100644 --- a/pkg/services/notifications/mailer.go +++ b/pkg/services/notifications/mailer.go @@ -6,21 +6,13 @@ package notifications import ( "bytes" - "crypto/tls" "fmt" "html/template" - "io" - "net" "net/mail" - "strconv" - "strings" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util/errutil" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - gomail "gopkg.in/mail.v2" ) var ( @@ -42,6 +34,10 @@ func init() { }) } +type Mailer interface { + Send(messages ...*Message) (int, error) +} + func (ns *NotificationService) Send(msg *Message) (int, error) { messages := []*Message{} @@ -55,123 +51,7 @@ func (ns *NotificationService) Send(msg *Message) (int, error) { } } - return ns.dialAndSend(messages...) -} - -func (ns *NotificationService) dialAndSend(messages ...*Message) (int, error) { - sentEmailsCount := 0 - dialer, err := ns.createDialer() - if err != nil { - return sentEmailsCount, err - } - - for _, msg := range messages { - m := ns.buildEmail(msg) - - innerError := dialer.DialAndSend(m) - emailsSentTotal.Inc() - if innerError != nil { - // As gomail does not returned typed errors we have to parse the error - // to catch invalid error when the address is invalid. - // https://github.com/go-gomail/gomail/blob/81ebce5c23dfd25c6c67194b37d3dd3f338c98b1/send.go#L113 - if !strings.HasPrefix(innerError.Error(), "gomail: invalid address") { - emailsSentFailed.Inc() - } - - err = errutil.Wrapf(innerError, "Failed to send notification to email addresses: %s", strings.Join(msg.To, ";")) - continue - } - - sentEmailsCount++ - } - - return sentEmailsCount, err -} - -func (ns *NotificationService) buildEmail(msg *Message) *gomail.Message { - m := gomail.NewMessage() - m.SetHeader("From", msg.From) - m.SetHeader("To", msg.To...) - m.SetHeader("Subject", msg.Subject) - ns.setFiles(m, msg) - for _, replyTo := range msg.ReplyTo { - m.SetAddressHeader("Reply-To", replyTo, "") - } - // loop over content types from settings in reverse order as they are ordered in according to descending - // preference while the alternatives should be ordered according to ascending preference - for i := len(ns.Cfg.Smtp.ContentTypes) - 1; i >= 0; i-- { - if i == len(ns.Cfg.Smtp.ContentTypes)-1 { - m.SetBody(ns.Cfg.Smtp.ContentTypes[i], msg.Body[ns.Cfg.Smtp.ContentTypes[i]]) - } else { - m.AddAlternative(ns.Cfg.Smtp.ContentTypes[i], msg.Body[ns.Cfg.Smtp.ContentTypes[i]]) - } - } - - return m -} - -// setFiles attaches files in various forms -func (ns *NotificationService) setFiles( - m *gomail.Message, - msg *Message, -) { - for _, file := range msg.EmbeddedFiles { - m.Embed(file) - } - - for _, file := range msg.AttachedFiles { - file := file - m.Attach(file.Name, gomail.SetCopyFunc(func(writer io.Writer) error { - _, err := writer.Write(file.Content) - return err - })) - } -} - -func (ns *NotificationService) createDialer() (*gomail.Dialer, error) { - host, port, err := net.SplitHostPort(ns.Cfg.Smtp.Host) - if err != nil { - return nil, err - } - iPort, err := strconv.Atoi(port) - if err != nil { - return nil, err - } - - tlsconfig := &tls.Config{ - InsecureSkipVerify: ns.Cfg.Smtp.SkipVerify, - ServerName: host, - } - - if ns.Cfg.Smtp.CertFile != "" { - cert, err := tls.LoadX509KeyPair(ns.Cfg.Smtp.CertFile, ns.Cfg.Smtp.KeyFile) - if err != nil { - return nil, fmt.Errorf("could not load cert or key file: %w", err) - } - tlsconfig.Certificates = []tls.Certificate{cert} - } - - d := gomail.NewDialer(host, iPort, ns.Cfg.Smtp.User, ns.Cfg.Smtp.Password) - d.TLSConfig = tlsconfig - d.StartTLSPolicy = getStartTLSPolicy(ns.Cfg.Smtp.StartTLSPolicy) - - if ns.Cfg.Smtp.EhloIdentity != "" { - d.LocalName = ns.Cfg.Smtp.EhloIdentity - } else { - d.LocalName = setting.InstanceName - } - return d, nil -} - -func getStartTLSPolicy(policy string) gomail.StartTLSPolicy { - switch policy { - case "NoStartTLS": - return -1 - case "MandatoryStartTLS": - return 1 - default: - return 0 - } + return ns.mailer.Send(messages...) } func (ns *NotificationService) buildEmailMessage(cmd *models.SendEmailCommand) (*Message, error) { diff --git a/pkg/services/notifications/mailer_test.go b/pkg/services/notifications/mailer_test.go deleted file mode 100644 index 263ec75abb6..00000000000 --- a/pkg/services/notifications/mailer_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package notifications - -import ( - "bytes" - "strings" - "testing" - - "github.com/grafana/grafana/pkg/setting" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBuildMail(t *testing.T) { - ns := &NotificationService{ - Cfg: setting.NewCfg(), - } - ns.Cfg.Smtp.ContentTypes = []string{"text/html", "text/plain"} - - message := &Message{ - To: []string{"to@address.com"}, - From: "from@address.com", - Subject: "Some subject", - Body: map[string]string{ - "text/html": "Some HTML body", - "text/plain": "Some plain text body", - }, - ReplyTo: []string{"from@address.com"}, - } - - t.Run("When building email", func(t *testing.T) { - email := ns.buildEmail(message) - - buf := new(bytes.Buffer) - _, err := email.WriteTo(buf) - require.NoError(t, err) - - assert.Contains(t, buf.String(), "Some HTML body") - assert.Contains(t, buf.String(), "Some plain text body") - assert.Less(t, strings.Index(buf.String(), "Some plain text body"), strings.Index(buf.String(), "Some HTML body")) - }) -} diff --git a/pkg/services/notifications/mock.go b/pkg/services/notifications/mock.go new file mode 100644 index 00000000000..14229fc797d --- /dev/null +++ b/pkg/services/notifications/mock.go @@ -0,0 +1,44 @@ +package notifications + +import ( + "context" + + "github.com/grafana/grafana/pkg/models" +) + +type NotificationServiceMock struct { + Webhook models.SendWebhookSync + EmailSync models.SendEmailCommandSync + Email models.SendEmailCommand + ShouldError error + + WebhookHandler func(context.Context, *models.SendWebhookSync) error + EmailHandlerSync func(context.Context, *models.SendEmailCommandSync) error + EmailHandler func(context.Context, *models.SendEmailCommand) error +} + +func (ns *NotificationServiceMock) SendWebhookSync(ctx context.Context, cmd *models.SendWebhookSync) error { + ns.Webhook = *cmd + if ns.WebhookHandler != nil { + return ns.WebhookHandler(ctx, cmd) + } + return ns.ShouldError +} + +func (ns *NotificationServiceMock) SendEmailCommandHandlerSync(ctx context.Context, cmd *models.SendEmailCommandSync) error { + ns.EmailSync = *cmd + if ns.EmailHandlerSync != nil { + return ns.EmailHandlerSync(ctx, cmd) + } + return ns.ShouldError +} + +func (ns *NotificationServiceMock) SendEmailCommandHandler(ctx context.Context, cmd *models.SendEmailCommand) error { + ns.Email = *cmd + if ns.EmailHandler != nil { + return ns.EmailHandler(ctx, cmd) + } + return ns.ShouldError +} + +func MockNotificationService() *NotificationServiceMock { return &NotificationServiceMock{} } diff --git a/pkg/services/notifications/notifications.go b/pkg/services/notifications/notifications.go index 60d500ea05f..eae202d6a04 100644 --- a/pkg/services/notifications/notifications.go +++ b/pkg/services/notifications/notifications.go @@ -17,25 +17,38 @@ import ( "github.com/grafana/grafana/pkg/util" ) +type WebhookSender interface { + SendWebhookSync(ctx context.Context, cmd *models.SendWebhookSync) error +} +type EmailSender interface { + SendEmailCommandHandlerSync(ctx context.Context, cmd *models.SendEmailCommandSync) error + SendEmailCommandHandler(ctx context.Context, cmd *models.SendEmailCommand) error +} +type Service interface { + WebhookSender + EmailSender +} + var mailTemplates *template.Template var tmplResetPassword = "reset_password" var tmplSignUpStarted = "signup_started" var tmplWelcomeOnSignUp = "welcome_on_signup" -func ProvideService(bus bus.Bus, cfg *setting.Cfg) (*NotificationService, error) { +func ProvideService(bus bus.Bus, cfg *setting.Cfg, mailer Mailer) (*NotificationService, error) { ns := &NotificationService{ Bus: bus, Cfg: cfg, log: log.New("notifications"), mailQueue: make(chan *Message, 10), webhookQueue: make(chan *Webhook, 10), + mailer: mailer, } ns.Bus.AddHandler(ns.sendResetPasswordEmail) ns.Bus.AddHandler(ns.validateResetPasswordCode) - ns.Bus.AddHandler(ns.sendEmailCommandHandler) + ns.Bus.AddHandler(ns.SendEmailCommandHandler) - ns.Bus.AddHandler(ns.sendEmailCommandHandlerSync) + ns.Bus.AddHandler(ns.SendEmailCommandHandlerSync) ns.Bus.AddHandler(ns.SendWebhookSync) ns.Bus.AddEventListener(ns.signUpStartedHandler) @@ -70,6 +83,7 @@ type NotificationService struct { mailQueue chan *Message webhookQueue chan *Webhook + mailer Mailer log log.Logger } @@ -117,7 +131,7 @@ func subjectTemplateFunc(obj map[string]interface{}, value string) string { return "" } -func (ns *NotificationService) sendEmailCommandHandlerSync(ctx context.Context, cmd *models.SendEmailCommandSync) error { +func (ns *NotificationService) SendEmailCommandHandlerSync(ctx context.Context, cmd *models.SendEmailCommandSync) error { message, err := ns.buildEmailMessage(&models.SendEmailCommand{ Data: cmd.Data, Info: cmd.Info, @@ -125,6 +139,7 @@ func (ns *NotificationService) sendEmailCommandHandlerSync(ctx context.Context, To: cmd.To, SingleEmail: cmd.SingleEmail, EmbeddedFiles: cmd.EmbeddedFiles, + AttachedFiles: cmd.AttachedFiles, Subject: cmd.Subject, ReplyTo: cmd.ReplyTo, }) @@ -137,7 +152,7 @@ func (ns *NotificationService) sendEmailCommandHandlerSync(ctx context.Context, return err } -func (ns *NotificationService) sendEmailCommandHandler(ctx context.Context, cmd *models.SendEmailCommand) error { +func (ns *NotificationService) SendEmailCommandHandler(ctx context.Context, cmd *models.SendEmailCommand) error { message, err := ns.buildEmailMessage(cmd) if err != nil { @@ -153,7 +168,7 @@ func (ns *NotificationService) sendResetPasswordEmail(ctx context.Context, cmd * if err != nil { return err } - return ns.sendEmailCommandHandler(ctx, &models.SendEmailCommand{ + return ns.SendEmailCommandHandler(ctx, &models.SendEmailCommand{ To: []string{cmd.User.Email}, Template: tmplResetPassword, Data: map[string]interface{}{ @@ -197,7 +212,7 @@ func (ns *NotificationService) signUpStartedHandler(ctx context.Context, evt *ev return nil } - err := ns.sendEmailCommandHandler(ctx, &models.SendEmailCommand{ + err := ns.SendEmailCommandHandler(ctx, &models.SendEmailCommand{ To: []string{evt.Email}, Template: tmplSignUpStarted, Data: map[string]interface{}{ @@ -220,7 +235,7 @@ func (ns *NotificationService) signUpCompletedHandler(ctx context.Context, evt * return nil } - return ns.sendEmailCommandHandler(ctx, &models.SendEmailCommand{ + return ns.SendEmailCommandHandler(ctx, &models.SendEmailCommand{ To: []string{evt.Email}, Template: tmplWelcomeOnSignUp, Data: map[string]interface{}{ diff --git a/pkg/services/notifications/notifications_test.go b/pkg/services/notifications/notifications_test.go index 55db3105569..3e40661dfab 100644 --- a/pkg/services/notifications/notifications_test.go +++ b/pkg/services/notifications/notifications_test.go @@ -11,30 +11,268 @@ import ( "github.com/stretchr/testify/require" ) -func TestNotificationService(t *testing.T) { - ns := &NotificationService{ - Cfg: setting.NewCfg(), - } - ns.Cfg.StaticRootPath = "../../../public/" - ns.Cfg.Smtp.Enabled = true - ns.Cfg.Smtp.TemplatesPatterns = []string{"emails/*.html", "emails/*.txt"} - ns.Cfg.Smtp.FromAddress = "from@address.com" - ns.Cfg.Smtp.FromName = "Grafana Admin" - ns.Cfg.Smtp.ContentTypes = []string{"text/html", "text/plain"} - ns.Bus = bus.New() +func TestProvideService(t *testing.T) { + bus := bus.New() - ns, err := ProvideService(bus.New(), ns.Cfg) - require.NoError(t, err) + t.Run("When invalid from_address in configuration", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.FromAddress = "@notanemail@" + _, _, err := createSutWithConfig(bus, cfg) - t.Run("When sending reset email password", func(t *testing.T) { - err := ns.sendResetPasswordEmail(context.Background(), &models.SendResetPasswordEmailCommand{User: &models.User{Email: "asd@asd.com"}}) + require.Error(t, err) + }) + + t.Run("When template_patterns fails to parse", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.TemplatesPatterns = append(cfg.Smtp.TemplatesPatterns, "/usr/not-a-dir/**") + _, _, err := createSutWithConfig(bus, cfg) + + require.Error(t, err) + }) +} + +func TestSendEmailSync(t *testing.T) { + bus := bus.New() + + t.Run("When sending emails synchronously", func(t *testing.T) { + _, mailer := createSut(t, bus) + cmd := &models.SendEmailCommandSync{ + SendEmailCommand: models.SendEmailCommand{ + Subject: "subject", + To: []string{"asdf@grafana.com"}, + SingleEmail: false, + Template: "welcome_on_signup", + }, + } + + err := bus.Dispatch(context.Background(), cmd) require.NoError(t, err) - sentMsg := <-ns.mailQueue + require.NotEmpty(t, mailer.Sent) + sent := mailer.Sent[len(mailer.Sent)-1] + require.Equal(t, "subject", sent.Subject) + require.Equal(t, []string{"asdf@grafana.com"}, sent.To) + }) + + t.Run("When using Single Email mode with multiple recipients", func(t *testing.T) { + _, mailer := createSut(t, bus) + cmd := &models.SendEmailCommandSync{ + SendEmailCommand: models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: true, + Template: "welcome_on_signup", + }, + } + + err := bus.Dispatch(context.Background(), cmd) + require.NoError(t, err) + + require.Len(t, mailer.Sent, 1) + }) + + t.Run("When using Multi Email mode with multiple recipients", func(t *testing.T) { + _, mailer := createSut(t, bus) + cmd := &models.SendEmailCommandSync{ + SendEmailCommand: models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: false, + Template: "welcome_on_signup", + }, + } + + err := bus.Dispatch(context.Background(), cmd) + require.NoError(t, err) + + require.Len(t, mailer.Sent, 3) + }) + + t.Run("When attaching files to emails", func(t *testing.T) { + _, mailer := createSut(t, bus) + cmd := &models.SendEmailCommandSync{ + SendEmailCommand: models.SendEmailCommand{ + Subject: "subject", + To: []string{"asdf@grafana.com"}, + SingleEmail: true, + Template: "welcome_on_signup", + AttachedFiles: []*models.SendEmailAttachFile{ + { + Name: "attachment.txt", + Content: []byte("text file content"), + }, + }, + }, + } + + err := bus.Dispatch(context.Background(), cmd) + require.NoError(t, err) + + require.NotEmpty(t, mailer.Sent) + sent := mailer.Sent[len(mailer.Sent)-1] + require.Len(t, sent.AttachedFiles, 1) + file := sent.AttachedFiles[len(sent.AttachedFiles)-1] + require.Equal(t, "attachment.txt", file.Name) + require.Equal(t, []byte("text file content"), file.Content) + }) + + t.Run("When SMTP disabled in configuration", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.Enabled = false + _, mailer, err := createSutWithConfig(bus, cfg) + require.NoError(t, err) + cmd := &models.SendEmailCommandSync{ + SendEmailCommand: models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: true, + Template: "welcome_on_signup", + }, + } + + err = bus.Dispatch(context.Background(), cmd) + + require.ErrorIs(t, err, models.ErrSmtpNotEnabled) + require.Empty(t, mailer.Sent) + }) + + t.Run("When invalid content type in configuration", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.ContentTypes = append(cfg.Smtp.ContentTypes, "multipart/form-data") + _, mailer, err := createSutWithConfig(bus, cfg) + require.NoError(t, err) + cmd := &models.SendEmailCommandSync{ + SendEmailCommand: models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: false, + Template: "welcome_on_signup", + }, + } + + err = bus.Dispatch(context.Background(), cmd) + + require.Error(t, err) + require.Empty(t, mailer.Sent) + }) + + t.Run("When SMTP dialer is disconnected", func(t *testing.T) { + _ = createDisconnectedSut(t, bus) + cmd := &models.SendEmailCommandSync{ + SendEmailCommand: models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: false, + Template: "welcome_on_signup", + }, + } + + err := bus.Dispatch(context.Background(), cmd) + + require.Error(t, err) + }) +} + +func TestSendEmailAsync(t *testing.T) { + bus := bus.New() + + t.Run("When sending reset email password", func(t *testing.T) { + sut, _ := createSut(t, bus) + err := sut.sendResetPasswordEmail(context.Background(), &models.SendResetPasswordEmailCommand{User: &models.User{Email: "asd@asd.com"}}) + require.NoError(t, err) + + sentMsg := <-sut.mailQueue assert.Contains(t, sentMsg.Body["text/html"], "body") assert.NotContains(t, sentMsg.Body["text/plain"], "body") assert.Equal(t, "Reset your Grafana password - asd@asd.com", sentMsg.Subject) assert.NotContains(t, sentMsg.Body["text/html"], "Subject") assert.NotContains(t, sentMsg.Body["text/plain"], "Subject") }) + + t.Run("When SMTP disabled in configuration", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.Enabled = false + _, mailer, err := createSutWithConfig(bus, cfg) + require.NoError(t, err) + cmd := &models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: true, + Template: "welcome_on_signup", + } + + err = bus.Dispatch(context.Background(), cmd) + + require.ErrorIs(t, err, models.ErrSmtpNotEnabled) + require.Empty(t, mailer.Sent) + }) + + t.Run("When invalid content type in configuration", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.ContentTypes = append(cfg.Smtp.ContentTypes, "multipart/form-data") + _, mailer, err := createSutWithConfig(bus, cfg) + require.NoError(t, err) + cmd := &models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: false, + Template: "welcome_on_signup", + } + + err = bus.Dispatch(context.Background(), cmd) + + require.Error(t, err) + require.Empty(t, mailer.Sent) + }) + + t.Run("When SMTP dialer is disconnected", func(t *testing.T) { + _ = createDisconnectedSut(t, bus) + cmd := &models.SendEmailCommand{ + Subject: "subject", + To: []string{"1@grafana.com", "2@grafana.com", "3@grafana.com"}, + SingleEmail: false, + Template: "welcome_on_signup", + } + + err := bus.Dispatch(context.Background(), cmd) + + // The async version should not surface connection errors via Bus. It should only log them. + require.NoError(t, err) + }) +} + +func createSut(t *testing.T, bus bus.Bus) (*NotificationService, *FakeMailer) { + t.Helper() + + cfg := createSmtpConfig() + ns, fm, err := createSutWithConfig(bus, cfg) + require.NoError(t, err) + return ns, fm +} + +func createSutWithConfig(bus bus.Bus, cfg *setting.Cfg) (*NotificationService, *FakeMailer, error) { + smtp := NewFakeMailer() + ns, err := ProvideService(bus, cfg, smtp) + return ns, smtp, err +} + +func createDisconnectedSut(t *testing.T, bus bus.Bus) *NotificationService { + t.Helper() + + cfg := createSmtpConfig() + smtp := NewFakeDisconnectedMailer() + ns, err := ProvideService(bus, cfg, smtp) + require.NoError(t, err) + return ns +} + +func createSmtpConfig() *setting.Cfg { + cfg := setting.NewCfg() + cfg.StaticRootPath = "../../../public/" + cfg.Smtp.Enabled = true + cfg.Smtp.TemplatesPatterns = []string{"emails/*.html", "emails/*.txt"} + cfg.Smtp.FromAddress = "from@address.com" + cfg.Smtp.FromName = "Grafana Admin" + cfg.Smtp.ContentTypes = []string{"text/html", "text/plain"} + return cfg } diff --git a/pkg/services/notifications/send_email_integration_test.go b/pkg/services/notifications/send_email_integration_test.go index ba9ae8e5dbb..d4ab18ec24f 100644 --- a/pkg/services/notifications/send_email_integration_test.go +++ b/pkg/services/notifications/send_email_integration_test.go @@ -57,7 +57,7 @@ func TestEmailIntegrationTest(t *testing.T) { Template: "alert_notification", } - err := ns.sendEmailCommandHandler(context.Background(), cmd) + err := ns.SendEmailCommandHandler(context.Background(), cmd) require.NoError(t, err) sentMsg := <-ns.mailQueue diff --git a/pkg/services/notifications/smtp.go b/pkg/services/notifications/smtp.go new file mode 100644 index 00000000000..9b548c2de64 --- /dev/null +++ b/pkg/services/notifications/smtp.go @@ -0,0 +1,147 @@ +package notifications + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "strconv" + "strings" + + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util/errutil" + gomail "gopkg.in/mail.v2" +) + +type SmtpClient struct { + cfg setting.SmtpSettings +} + +func ProvideSmtpService(cfg *setting.Cfg) (Mailer, error) { + return NewSmtpClient(cfg.Smtp) +} + +func NewSmtpClient(cfg setting.SmtpSettings) (*SmtpClient, error) { + client := &SmtpClient{ + cfg: cfg, + } + + return client, nil +} + +func (sc *SmtpClient) Send(messages ...*Message) (int, error) { + sentEmailsCount := 0 + dialer, err := sc.createDialer() + if err != nil { + return sentEmailsCount, err + } + + for _, msg := range messages { + m := sc.buildEmail(msg) + + innerError := dialer.DialAndSend(m) + emailsSentTotal.Inc() + if innerError != nil { + // As gomail does not returned typed errors we have to parse the error + // to catch invalid error when the address is invalid. + // https://github.com/go-gomail/gomail/blob/81ebce5c23dfd25c6c67194b37d3dd3f338c98b1/send.go#L113 + if !strings.HasPrefix(innerError.Error(), "gomail: invalid address") { + emailsSentFailed.Inc() + } + + err = errutil.Wrapf(innerError, "Failed to send notification to email addresses: %s", strings.Join(msg.To, ";")) + continue + } + + sentEmailsCount++ + } + + return sentEmailsCount, err +} + +// buildEmail converts the Message DTO to a gomail message. +func (sc *SmtpClient) buildEmail(msg *Message) *gomail.Message { + m := gomail.NewMessage() + m.SetHeader("From", msg.From) + m.SetHeader("To", msg.To...) + m.SetHeader("Subject", msg.Subject) + sc.setFiles(m, msg) + for _, replyTo := range msg.ReplyTo { + m.SetAddressHeader("Reply-To", replyTo, "") + } + // loop over content types from settings in reverse order as they are ordered in according to descending + // preference while the alternatives should be ordered according to ascending preference + for i := len(sc.cfg.ContentTypes) - 1; i >= 0; i-- { + if i == len(sc.cfg.ContentTypes)-1 { + m.SetBody(sc.cfg.ContentTypes[i], msg.Body[sc.cfg.ContentTypes[i]]) + } else { + m.AddAlternative(sc.cfg.ContentTypes[i], msg.Body[sc.cfg.ContentTypes[i]]) + } + } + + return m +} + +// setFiles attaches files in various forms. +func (sc *SmtpClient) setFiles( + m *gomail.Message, + msg *Message, +) { + for _, file := range msg.EmbeddedFiles { + m.Embed(file) + } + + for _, file := range msg.AttachedFiles { + file := file + m.Attach(file.Name, gomail.SetCopyFunc(func(writer io.Writer) error { + _, err := writer.Write(file.Content) + return err + })) + } +} + +func (sc *SmtpClient) createDialer() (*gomail.Dialer, error) { + host, port, err := net.SplitHostPort(sc.cfg.Host) + if err != nil { + return nil, err + } + iPort, err := strconv.Atoi(port) + if err != nil { + return nil, err + } + + tlsconfig := &tls.Config{ + InsecureSkipVerify: sc.cfg.SkipVerify, + ServerName: host, + } + + if sc.cfg.CertFile != "" { + cert, err := tls.LoadX509KeyPair(sc.cfg.CertFile, sc.cfg.KeyFile) + if err != nil { + return nil, fmt.Errorf("could not load cert or key file: %w", err) + } + tlsconfig.Certificates = []tls.Certificate{cert} + } + + d := gomail.NewDialer(host, iPort, sc.cfg.User, sc.cfg.Password) + d.TLSConfig = tlsconfig + d.StartTLSPolicy = getStartTLSPolicy(sc.cfg.StartTLSPolicy) + + if sc.cfg.EhloIdentity != "" { + d.LocalName = sc.cfg.EhloIdentity + } else { + d.LocalName = setting.InstanceName + } + return d, nil +} + +func getStartTLSPolicy(policy string) gomail.StartTLSPolicy { + switch policy { + case "NoStartTLS": + return -1 + case "MandatoryStartTLS": + return 1 + default: + return 0 + } +} diff --git a/pkg/services/notifications/smtp_test.go b/pkg/services/notifications/smtp_test.go new file mode 100644 index 00000000000..8ca5a11ed5d --- /dev/null +++ b/pkg/services/notifications/smtp_test.go @@ -0,0 +1,108 @@ +package notifications + +import ( + "bytes" + "strings" + "testing" + + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildMail(t *testing.T) { + cfg := setting.NewCfg() + cfg.Smtp.ContentTypes = []string{"text/html", "text/plain"} + + sc, err := NewSmtpClient(cfg.Smtp) + require.NoError(t, err) + + message := &Message{ + To: []string{"to@address.com"}, + From: "from@address.com", + Subject: "Some subject", + Body: map[string]string{ + "text/html": "Some HTML body", + "text/plain": "Some plain text body", + }, + ReplyTo: []string{"from@address.com"}, + } + + t.Run("When building email", func(t *testing.T) { + email := sc.buildEmail(message) + + buf := new(bytes.Buffer) + _, err := email.WriteTo(buf) + require.NoError(t, err) + + assert.Contains(t, buf.String(), "Some HTML body") + assert.Contains(t, buf.String(), "Some plain text body") + assert.Less(t, strings.Index(buf.String(), "Some plain text body"), strings.Index(buf.String(), "Some HTML body")) + }) +} + +func TestSmtpDialer(t *testing.T) { + t.Run("When SMTP hostname is invalid", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.Host = "invalid%hostname:123:456" + client, err := ProvideSmtpService(cfg) + require.NoError(t, err) + message := &Message{ + To: []string{"asdf@grafana.com"}, + SingleEmail: true, + Subject: "subject", + Body: map[string]string{ + "text/html": "body", + "text/plain": "body", + }, + } + + count, err := client.Send(message) + + require.Equal(t, 0, count) + require.EqualError(t, err, "address invalid%hostname:123:456: too many colons in address") + }) + + t.Run("When SMTP port is invalid", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.Host = "invalid%hostname:123a" + client, err := ProvideSmtpService(cfg) + require.NoError(t, err) + message := &Message{ + To: []string{"asdf@grafana.com"}, + SingleEmail: true, + Subject: "subject", + Body: map[string]string{ + "text/html": "body", + "text/plain": "body", + }, + } + + count, err := client.Send(message) + + require.Equal(t, 0, count) + require.EqualError(t, err, "strconv.Atoi: parsing \"123a\": invalid syntax") + }) + + t.Run("When TLS certificate does not exist", func(t *testing.T) { + cfg := createSmtpConfig() + cfg.Smtp.Host = "localhost:1234" + cfg.Smtp.CertFile = "/var/certs/does-not-exist.pem" + client, err := ProvideSmtpService(cfg) + require.NoError(t, err) + message := &Message{ + To: []string{"asdf@grafana.com"}, + SingleEmail: true, + Subject: "subject", + Body: map[string]string{ + "text/html": "body", + "text/plain": "body", + }, + } + + count, err := client.Send(message) + + require.Equal(t, 0, count) + require.EqualError(t, err, "could not load cert or key file: open /var/certs/does-not-exist.pem: no such file or directory") + }) +} diff --git a/pkg/services/notifications/testing.go b/pkg/services/notifications/testing.go new file mode 100644 index 00000000000..325ea0cc878 --- /dev/null +++ b/pkg/services/notifications/testing.go @@ -0,0 +1,32 @@ +package notifications + +import "fmt" + +type FakeMailer struct { + Sent []*Message +} + +func NewFakeMailer() *FakeMailer { + return &FakeMailer{ + Sent: make([]*Message, 0), + } +} + +func (fm *FakeMailer) Send(messages ...*Message) (int, error) { + sentEmailsCount := 0 + for _, msg := range messages { + fm.Sent = append(fm.Sent, msg) + sentEmailsCount++ + } + return sentEmailsCount, nil +} + +type FakeDisconnectedMailer struct{} + +func NewFakeDisconnectedMailer() *FakeDisconnectedMailer { + return &FakeDisconnectedMailer{} +} + +func (fdm *FakeDisconnectedMailer) Send(messages ...*Message) (int, error) { + return 0, fmt.Errorf("connect: connection refused") +} diff --git a/pkg/plugins/plugindashboards/service.go b/pkg/services/plugindashboards/plugindashboards.go similarity index 70% rename from pkg/plugins/plugindashboards/service.go rename to pkg/services/plugindashboards/plugindashboards.go index a7bbd190290..3f468c8c1e0 100644 --- a/pkg/plugins/plugindashboards/service.go +++ b/pkg/services/plugindashboards/plugindashboards.go @@ -8,34 +8,49 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/dashboardimport" "github.com/grafana/grafana/pkg/services/sqlstore" ) -func ProvideService(pluginStore plugins.Store, pluginDashboardManager plugins.PluginDashboardManager, - sqlStore *sqlstore.SQLStore) *Service { - s := &Service{ - sqlStore: sqlStore, - pluginStore: pluginStore, - pluginDashboardManager: pluginDashboardManager, - logger: log.New("plugindashboards"), - } - bus.AddEventListener(s.handlePluginStateChanged) +type pluginSettingsStore interface { + GetPluginSettings(ctx context.Context, orgID int64) ([]*models.PluginSettingInfoDTO, error) +} + +func ProvideService(sqlStore *sqlstore.SQLStore, bus bus.Bus, pluginStore plugins.Store, + pluginDashboardManager plugins.PluginDashboardManager, dashboardImportService dashboardimport.Service) *Service { + s := new(sqlStore, bus, pluginStore, pluginDashboardManager, dashboardImportService) s.updateAppDashboards() return s } +func new(pluginSettingsStore pluginSettingsStore, bus bus.Bus, pluginStore plugins.Store, + pluginDashboardManager plugins.PluginDashboardManager, dashboardImportService dashboardimport.Service) *Service { + s := &Service{ + pluginSettingsStore: pluginSettingsStore, + bus: bus, + pluginStore: pluginStore, + pluginDashboardManager: pluginDashboardManager, + dashboardImportService: dashboardImportService, + logger: log.New("plugindashboards"), + } + bus.AddEventListener(s.handlePluginStateChanged) + + return s +} + type Service struct { - sqlStore *sqlstore.SQLStore + pluginSettingsStore pluginSettingsStore + bus bus.Bus pluginStore plugins.Store pluginDashboardManager plugins.PluginDashboardManager - - logger log.Logger + dashboardImportService dashboardimport.Service + logger log.Logger } func (s *Service) updateAppDashboards() { s.logger.Debug("Looking for app dashboard updates") - pluginSettings, err := s.sqlStore.GetPluginSettings(context.Background(), 0) + pluginSettings, err := s.pluginSettingsStore.GetPluginSettings(context.Background(), 0) if err != nil { s.logger.Error("Failed to get all plugin settings", "error", err) return @@ -72,7 +87,7 @@ func (s *Service) syncPluginDashboards(ctx context.Context, plugin plugins.Plugi s.logger.Info("Deleting plugin dashboard", "pluginId", plugin.ID, "dashboard", dash.Slug) deleteCmd := models.DeleteDashboardCommand{OrgId: orgID, Id: dash.DashboardId} - if err := bus.Dispatch(ctx, &deleteCmd); err != nil { + if err := s.bus.Dispatch(ctx, &deleteCmd); err != nil { s.logger.Error("Failed to auto update app dashboard", "pluginId", plugin.ID, "error", err) return } @@ -91,7 +106,7 @@ func (s *Service) syncPluginDashboards(ctx context.Context, plugin plugins.Plugi // update version in plugin_setting table to mark that we have processed the update query := models.GetPluginSettingByIdQuery{PluginId: plugin.ID, OrgId: orgID} - if err := bus.Dispatch(ctx, &query); err != nil { + if err := s.bus.Dispatch(ctx, &query); err != nil { s.logger.Error("Failed to read plugin setting by ID", "error", err) return } @@ -103,7 +118,7 @@ func (s *Service) syncPluginDashboards(ctx context.Context, plugin plugins.Plugi PluginVersion: plugin.Info.Version, } - if err := bus.Dispatch(ctx, &cmd); err != nil { + if err := s.bus.Dispatch(ctx, &cmd); err != nil { s.logger.Error("Failed to update plugin setting version", "error", err) } } @@ -120,14 +135,14 @@ func (s *Service) handlePluginStateChanged(ctx context.Context, event *models.Pl s.syncPluginDashboards(ctx, p, event.OrgId) } else { query := models.GetDashboardsByPluginIdQuery{PluginId: event.PluginId, OrgId: event.OrgId} - if err := bus.Dispatch(ctx, &query); err != nil { + if err := s.bus.Dispatch(ctx, &query); err != nil { return err } for _, dash := range query.Result { s.logger.Info("Deleting plugin dashboard", "pluginId", event.PluginId, "dashboard", dash.Slug) deleteCmd := models.DeleteDashboardCommand{OrgId: dash.OrgId, Id: dash.Id} - if err := bus.Dispatch(ctx, &deleteCmd); err != nil { + if err := s.bus.Dispatch(ctx, &deleteCmd); err != nil { return err } } @@ -143,8 +158,14 @@ func (s *Service) autoUpdateAppDashboard(ctx context.Context, pluginDashInfo *pl } s.logger.Info("Auto updating App dashboard", "dashboard", dash.Title, "newRev", pluginDashInfo.Revision, "oldRev", pluginDashInfo.ImportedRevision) - user := &models.SignedInUser{UserId: 0, OrgRole: models.ROLE_ADMIN} - _, _, err = s.pluginDashboardManager.ImportDashboard(ctx, pluginDashInfo.PluginId, pluginDashInfo.Path, orgID, 0, dash.Data, true, - nil, user) + _, err = s.dashboardImportService.ImportDashboard(ctx, &dashboardimport.ImportDashboardRequest{ + PluginId: pluginDashInfo.PluginId, + User: &models.SignedInUser{UserId: 0, OrgRole: models.ROLE_ADMIN, OrgId: orgID}, + Path: pluginDashInfo.Path, + FolderId: 0, + Dashboard: dash.Data, + Overwrite: true, + Inputs: nil, + }) return err } diff --git a/pkg/services/plugindashboards/plugindashboards_test.go b/pkg/services/plugindashboards/plugindashboards_test.go new file mode 100644 index 00000000000..204d1cb3959 --- /dev/null +++ b/pkg/services/plugindashboards/plugindashboards_test.go @@ -0,0 +1,575 @@ +package plugindashboards + +import ( + "context" + "fmt" + "testing" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/dashboardimport" + "github.com/stretchr/testify/require" +) + +func TestService(t *testing.T) { + t.Run("updateAppDashboards", func(t *testing.T) { + scenario(t, "Without any stored plugin settings shouldn't delete/import any dashboards", + scenarioInput{}, func(ctx *scenarioContext) { + ctx.s.updateAppDashboards() + + require.Len(t, ctx.getPluginSettingsArgs, 1) + require.Equal(t, int64(0), ctx.getPluginSettingsArgs[0]) + require.Empty(t, ctx.deleteDashboardArgs) + require.Empty(t, ctx.importDashboardArgs) + }) + + scenario(t, "Without any stored enabled plugin shouldn't delete/import any dashboards", + scenarioInput{ + storedPluginSettings: []*models.PluginSettingInfoDTO{ + { + PluginId: "test", + Enabled: false, + }, + }, + pluginDashboards: []*plugins.PluginDashboardInfoDTO{ + { + PluginId: "test", + Path: "dashboard.json", + }, + }, + }, func(ctx *scenarioContext) { + ctx.s.updateAppDashboards() + + require.NotEmpty(t, ctx.getPluginSettingsArgs) + require.Empty(t, ctx.deleteDashboardArgs) + require.Empty(t, ctx.importDashboardArgs) + }) + + scenario(t, "With stored enabled plugin, but not installed shouldn't delete/import any dashboards", + scenarioInput{ + storedPluginSettings: []*models.PluginSettingInfoDTO{ + { + PluginId: "test", + Enabled: true, + }, + }, + pluginDashboards: []*plugins.PluginDashboardInfoDTO{ + { + PluginId: "test", + Path: "dashboard.json", + }, + }, + }, func(ctx *scenarioContext) { + ctx.s.updateAppDashboards() + + require.NotEmpty(t, ctx.getPluginSettingsArgs) + require.Empty(t, ctx.deleteDashboardArgs) + require.Empty(t, ctx.importDashboardArgs) + }) + + scenario(t, "With stored enabled plugin and installed with same version shouldn't delete/import any dashboards", + scenarioInput{ + storedPluginSettings: []*models.PluginSettingInfoDTO{ + { + PluginId: "test", + Enabled: true, + PluginVersion: "1.0.0", + }, + }, + installedPlugins: []plugins.PluginDTO{ + { + JSONData: plugins.JSONData{ + Info: plugins.Info{ + Version: "1.0.0", + }, + }, + }, + }, + pluginDashboards: []*plugins.PluginDashboardInfoDTO{ + { + PluginId: "test", + Path: "dashboard.json", + }, + }, + }, func(ctx *scenarioContext) { + ctx.s.updateAppDashboards() + + require.NotEmpty(t, ctx.getPluginSettingsArgs) + require.Empty(t, ctx.deleteDashboardArgs) + require.Empty(t, ctx.importDashboardArgs) + }) + + scenario(t, "With stored enabled plugin and installed with different versions, but no dashboard updates shouldn't delete/import dashboards", + scenarioInput{ + storedPluginSettings: []*models.PluginSettingInfoDTO{ + { + PluginId: "test", + Enabled: true, + PluginVersion: "1.0.0", + }, + }, + installedPlugins: []plugins.PluginDTO{ + { + JSONData: plugins.JSONData{ + Info: plugins.Info{ + Version: "1.0.1", + }, + }, + }, + }, + pluginDashboards: []*plugins.PluginDashboardInfoDTO{ + { + PluginId: "test", + Path: "dashboard.json", + Removed: false, + Revision: 1, + ImportedRevision: 1, + }, + }, + }, func(ctx *scenarioContext) { + ctx.s.updateAppDashboards() + + require.NotEmpty(t, ctx.getPluginSettingsArgs) + require.Empty(t, ctx.deleteDashboardArgs) + require.Empty(t, ctx.importDashboardArgs) + }) + + scenario(t, "With stored enabled plugin and installed with different versions and with dashboard updates should delete/import dashboards", + scenarioInput{ + storedPluginSettings: []*models.PluginSettingInfoDTO{ + { + PluginId: "test", + Enabled: true, + PluginVersion: "1.0.0", + OrgId: 2, + }, + }, + installedPlugins: []plugins.PluginDTO{ + { + JSONData: plugins.JSONData{ + ID: "test", + Info: plugins.Info{ + Version: "1.0.1", + }, + }, + }, + }, + pluginDashboards: []*plugins.PluginDashboardInfoDTO{ + { + DashboardId: 3, + PluginId: "test", + Path: "removed.json", + Removed: true, + }, + { + DashboardId: 4, + PluginId: "test", + Path: "not-updated.json", + }, + { + DashboardId: 5, + PluginId: "test", + Path: "updated.json", + Revision: 1, + ImportedRevision: 2, + }, + }, + }, func(ctx *scenarioContext) { + ctx.s.updateAppDashboards() + + require.NotEmpty(t, ctx.getPluginSettingsArgs) + require.Len(t, ctx.deleteDashboardArgs, 1) + require.Equal(t, int64(2), ctx.deleteDashboardArgs[0].OrgId) + require.Equal(t, int64(3), ctx.deleteDashboardArgs[0].Id) + + require.Len(t, ctx.importDashboardArgs, 1) + require.Equal(t, "test", ctx.importDashboardArgs[0].PluginId) + require.Equal(t, "updated.json", ctx.importDashboardArgs[0].Path) + require.Equal(t, int64(2), ctx.importDashboardArgs[0].User.OrgId) + require.Equal(t, models.ROLE_ADMIN, ctx.importDashboardArgs[0].User.OrgRole) + require.Equal(t, int64(0), ctx.importDashboardArgs[0].FolderId) + require.True(t, ctx.importDashboardArgs[0].Overwrite) + + require.Len(t, ctx.getPluginSettingsByIdArgs, 1) + require.Equal(t, int64(2), ctx.getPluginSettingsByIdArgs[0].OrgId) + require.Equal(t, "test", ctx.getPluginSettingsByIdArgs[0].PluginId) + + require.Len(t, ctx.updatePluginSettingVersionArgs, 1) + require.Equal(t, int64(2), ctx.updatePluginSettingVersionArgs[0].OrgId) + require.Equal(t, "test", ctx.updatePluginSettingVersionArgs[0].PluginId) + require.Equal(t, "1.0.1", ctx.updatePluginSettingVersionArgs[0].PluginVersion) + }) + }) + + t.Run("handlePluginStateChanged", func(t *testing.T) { + scenario(t, "When app plugin is disabled that doesn't have any imported dashboards shouldn't delete any", + scenarioInput{}, func(ctx *scenarioContext) { + err := ctx.bus.Publish(context.Background(), &models.PluginStateChangedEvent{ + PluginId: "test", + OrgId: 2, + Enabled: false, + }) + require.NoError(t, err) + + require.Len(t, ctx.getDashboardsByPluginIdQueryArgs, 1) + require.Equal(t, int64(2), ctx.getDashboardsByPluginIdQueryArgs[0].OrgId) + require.Equal(t, "test", ctx.getDashboardsByPluginIdQueryArgs[0].PluginId) + require.Empty(t, ctx.deleteDashboardArgs) + }) + }) + + scenario(t, "When app plugin is disabled that have imported dashboards should delete them", + scenarioInput{ + storedPluginSettings: []*models.PluginSettingInfoDTO{ + { + PluginId: "test", + Enabled: true, + OrgId: 2, + }, + }, + installedPlugins: []plugins.PluginDTO{ + { + JSONData: plugins.JSONData{ + ID: "test", + }, + }, + }, + pluginDashboards: []*plugins.PluginDashboardInfoDTO{ + { + DashboardId: 3, + PluginId: "test", + Path: "dashboard1.json", + }, + { + DashboardId: 4, + PluginId: "test", + Path: "dashboard2.json", + }, + { + DashboardId: 5, + PluginId: "test", + Path: "dashboard3.json", + }, + }, + }, func(ctx *scenarioContext) { + err := ctx.bus.Publish(context.Background(), &models.PluginStateChangedEvent{ + PluginId: "test", + OrgId: 2, + Enabled: false, + }) + require.NoError(t, err) + + require.Len(t, ctx.getDashboardsByPluginIdQueryArgs, 1) + require.Equal(t, int64(2), ctx.getDashboardsByPluginIdQueryArgs[0].OrgId) + require.Equal(t, "test", ctx.getDashboardsByPluginIdQueryArgs[0].PluginId) + require.Len(t, ctx.deleteDashboardArgs, 3) + }) + + scenario(t, "When app plugin is enabled, stored disabled plugin and with dashboard updates should import dashboards", + scenarioInput{ + storedPluginSettings: []*models.PluginSettingInfoDTO{ + { + PluginId: "test", + Enabled: false, + OrgId: 2, + PluginVersion: "1.0.0", + }, + }, + installedPlugins: []plugins.PluginDTO{ + { + JSONData: plugins.JSONData{ + ID: "test", + Info: plugins.Info{ + Version: "1.0.0", + }, + }, + }, + }, + pluginDashboards: []*plugins.PluginDashboardInfoDTO{ + { + DashboardId: 3, + PluginId: "test", + Path: "dashboard1.json", + Revision: 1, + ImportedRevision: 0, + }, + { + DashboardId: 4, + PluginId: "test", + Path: "dashboard2.json", + Revision: 1, + ImportedRevision: 0, + }, + { + DashboardId: 5, + PluginId: "test", + Path: "dashboard3.json", + Revision: 1, + ImportedRevision: 0, + }, + }, + }, func(ctx *scenarioContext) { + err := ctx.bus.Publish(context.Background(), &models.PluginStateChangedEvent{ + PluginId: "test", + OrgId: 2, + Enabled: true, + }) + require.NoError(t, err) + + require.Empty(t, ctx.deleteDashboardArgs) + + require.Len(t, ctx.importDashboardArgs, 3) + require.Equal(t, "test", ctx.importDashboardArgs[0].PluginId) + require.Equal(t, "dashboard1.json", ctx.importDashboardArgs[0].Path) + require.Equal(t, int64(2), ctx.importDashboardArgs[0].User.OrgId) + require.Equal(t, models.ROLE_ADMIN, ctx.importDashboardArgs[0].User.OrgRole) + require.Equal(t, int64(0), ctx.importDashboardArgs[0].FolderId) + require.True(t, ctx.importDashboardArgs[0].Overwrite) + + require.Equal(t, "test", ctx.importDashboardArgs[1].PluginId) + require.Equal(t, "dashboard2.json", ctx.importDashboardArgs[1].Path) + require.Equal(t, int64(2), ctx.importDashboardArgs[1].User.OrgId) + require.Equal(t, models.ROLE_ADMIN, ctx.importDashboardArgs[1].User.OrgRole) + require.Equal(t, int64(0), ctx.importDashboardArgs[1].FolderId) + require.True(t, ctx.importDashboardArgs[1].Overwrite) + + require.Equal(t, "test", ctx.importDashboardArgs[2].PluginId) + require.Equal(t, "dashboard3.json", ctx.importDashboardArgs[2].Path) + require.Equal(t, int64(2), ctx.importDashboardArgs[2].User.OrgId) + require.Equal(t, models.ROLE_ADMIN, ctx.importDashboardArgs[2].User.OrgRole) + require.Equal(t, int64(0), ctx.importDashboardArgs[2].FolderId) + require.True(t, ctx.importDashboardArgs[2].Overwrite) + + require.Len(t, ctx.getPluginSettingsByIdArgs, 1) + require.Equal(t, int64(2), ctx.getPluginSettingsByIdArgs[0].OrgId) + require.Equal(t, "test", ctx.getPluginSettingsByIdArgs[0].PluginId) + + require.Len(t, ctx.updatePluginSettingVersionArgs, 1) + require.Equal(t, int64(2), ctx.updatePluginSettingVersionArgs[0].OrgId) + require.Equal(t, "test", ctx.updatePluginSettingVersionArgs[0].PluginId) + require.Equal(t, "1.0.0", ctx.updatePluginSettingVersionArgs[0].PluginVersion) + }) +} + +type pluginSettingsStoreMock struct { + getPluginSettingsFunc func(ctx context.Context, orgID int64) ([]*models.PluginSettingInfoDTO, error) +} + +func (m *pluginSettingsStoreMock) GetPluginSettings(ctx context.Context, orgID int64) ([]*models.PluginSettingInfoDTO, error) { + if m.getPluginSettingsFunc != nil { + return m.getPluginSettingsFunc(ctx, orgID) + } + + return nil, nil +} + +type pluginStoreMock struct { + plugins.Store + pluginFunc func(ctx context.Context, pluginID string) (plugins.PluginDTO, bool) +} + +func (m *pluginStoreMock) Plugin(ctx context.Context, pluginID string) (plugins.PluginDTO, bool) { + if m.pluginFunc != nil { + return m.pluginFunc(ctx, pluginID) + } + + return plugins.PluginDTO{}, false +} + +type pluginDashboardManagerMock struct { + plugins.PluginDashboardManager + getPluginDashboardsFunc func(ctx context.Context, orgID int64, pluginID string) ([]*plugins.PluginDashboardInfoDTO, error) + loadPluginDashboardFunc func(ctx context.Context, pluginID, path string) (*models.Dashboard, error) +} + +func (m *pluginDashboardManagerMock) GetPluginDashboards(ctx context.Context, orgID int64, pluginID string) ([]*plugins.PluginDashboardInfoDTO, error) { + if m.getPluginDashboardsFunc != nil { + return m.getPluginDashboardsFunc(ctx, orgID, pluginID) + } + + return []*plugins.PluginDashboardInfoDTO{}, nil +} + +func (m *pluginDashboardManagerMock) LoadPluginDashboard(ctx context.Context, pluginID, path string) (*models.Dashboard, error) { + if m.loadPluginDashboardFunc != nil { + return m.loadPluginDashboardFunc(ctx, pluginID, path) + } + + return nil, nil +} + +type importDashboardServiceMock struct { + dashboardimport.Service + importDashboardFunc func(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) +} + +func (m *importDashboardServiceMock) ImportDashboard(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) { + if m.importDashboardFunc != nil { + return m.importDashboardFunc(ctx, req) + } + + return nil, nil +} + +type scenarioInput struct { + storedPluginSettings []*models.PluginSettingInfoDTO + installedPlugins []plugins.PluginDTO + pluginDashboards []*plugins.PluginDashboardInfoDTO +} + +type scenarioContext struct { + t *testing.T + bus bus.Bus + pluginSettingsStore pluginSettingsStore + getPluginSettingsArgs []int64 + pluginStore plugins.Store + pluginDashboardManager plugins.PluginDashboardManager + importDashboardService dashboardimport.Service + importDashboardArgs []*dashboardimport.ImportDashboardRequest + deleteDashboardArgs []*models.DeleteDashboardCommand + getPluginSettingsByIdArgs []*models.GetPluginSettingByIdQuery + updatePluginSettingVersionArgs []*models.UpdatePluginSettingVersionCmd + getDashboardsByPluginIdQueryArgs []*models.GetDashboardsByPluginIdQuery + s *Service +} + +func scenario(t *testing.T, desc string, input scenarioInput, f func(ctx *scenarioContext)) { + t.Helper() + + sCtx := &scenarioContext{ + t: t, + bus: bus.New(), + getPluginSettingsArgs: []int64{}, + importDashboardArgs: []*dashboardimport.ImportDashboardRequest{}, + deleteDashboardArgs: []*models.DeleteDashboardCommand{}, + getPluginSettingsByIdArgs: []*models.GetPluginSettingByIdQuery{}, + updatePluginSettingVersionArgs: []*models.UpdatePluginSettingVersionCmd{}, + getDashboardsByPluginIdQueryArgs: []*models.GetDashboardsByPluginIdQuery{}, + } + + getPluginSettings := func(_ context.Context, orgID int64) ([]*models.PluginSettingInfoDTO, error) { + sCtx.getPluginSettingsArgs = append(sCtx.getPluginSettingsArgs, orgID) + return input.storedPluginSettings, nil + } + + sCtx.pluginSettingsStore = &pluginSettingsStoreMock{ + getPluginSettingsFunc: getPluginSettings, + } + + getPlugin := func(ctx context.Context, pluginID string) (plugins.PluginDTO, bool) { + for _, p := range input.installedPlugins { + if p.ID == pluginID { + return p, true + } + } + + return plugins.PluginDTO{}, false + } + + sCtx.pluginStore = &pluginStoreMock{ + pluginFunc: getPlugin, + } + + getPluginDashboards := func(ctx context.Context, orgID int64, pluginID string) ([]*plugins.PluginDashboardInfoDTO, error) { + dashboards := []*plugins.PluginDashboardInfoDTO{} + + for _, d := range input.pluginDashboards { + if d.PluginId == pluginID { + dashboards = append(dashboards, d) + } + } + + return dashboards, nil + } + + loadPluginDashboard := func(ctx context.Context, pluginID, path string) (*models.Dashboard, error) { + for _, d := range input.pluginDashboards { + if d.PluginId == pluginID && path == d.Path { + return &models.Dashboard{}, nil + } + } + + return nil, fmt.Errorf("no match for loading plugin dashboard") + } + + sCtx.pluginDashboardManager = &pluginDashboardManagerMock{ + getPluginDashboardsFunc: getPluginDashboards, + loadPluginDashboardFunc: loadPluginDashboard, + } + + importDashboard := func(ctx context.Context, req *dashboardimport.ImportDashboardRequest) (*dashboardimport.ImportDashboardResponse, error) { + sCtx.importDashboardArgs = append(sCtx.importDashboardArgs, req) + + return &dashboardimport.ImportDashboardResponse{ + PluginId: req.PluginId, + }, nil + } + + sCtx.importDashboardService = &importDashboardServiceMock{ + importDashboardFunc: importDashboard, + } + + sCtx.bus.AddHandler(func(ctx context.Context, cmd *models.DeleteDashboardCommand) error { + sCtx.deleteDashboardArgs = append(sCtx.deleteDashboardArgs, cmd) + + return nil + }) + + sCtx.bus.AddHandler(func(ctx context.Context, query *models.GetPluginSettingByIdQuery) error { + for _, p := range input.storedPluginSettings { + if p.PluginId == query.PluginId { + query.Result = &models.PluginSetting{ + PluginId: p.PluginId, + OrgId: p.OrgId, + } + } + } + + sCtx.getPluginSettingsByIdArgs = append(sCtx.getPluginSettingsByIdArgs, query) + return nil + }) + + sCtx.bus.AddHandler(func(ctx context.Context, cmd *models.UpdatePluginSettingVersionCmd) error { + sCtx.updatePluginSettingVersionArgs = append(sCtx.updatePluginSettingVersionArgs, cmd) + return nil + }) + + sCtx.bus.AddHandler(func(ctx context.Context, query *models.GetDashboardsByPluginIdQuery) error { + sCtx.getDashboardsByPluginIdQueryArgs = append(sCtx.getDashboardsByPluginIdQueryArgs, query) + dashboards := []*models.Dashboard{} + + var plugin *models.PluginSettingInfoDTO + + for _, p := range input.storedPluginSettings { + if p.PluginId == query.PluginId { + plugin = p + } + } + + if plugin == nil { + return nil + } + + for _, d := range input.pluginDashboards { + if d.PluginId == plugin.PluginId { + dashboards = append(dashboards, &models.Dashboard{ + Id: d.DashboardId, + OrgId: plugin.OrgId, + }) + } + } + + query.Result = dashboards + + return nil + }) + + sCtx.s = new(sCtx.pluginSettingsStore, sCtx.bus, sCtx.pluginStore, sCtx.pluginDashboardManager, sCtx.importDashboardService) + + t.Cleanup(bus.ClearBusHandlers) + + t.Run(desc, func(t *testing.T) { + f(sCtx) + }) +} diff --git a/pkg/services/provisioning/dashboards/validator.go b/pkg/services/provisioning/dashboards/validator.go index e5402b28382..bfd39d4e7ae 100644 --- a/pkg/services/provisioning/dashboards/validator.go +++ b/pkg/services/provisioning/dashboards/validator.go @@ -56,61 +56,75 @@ func newDuplicateValidator(logger log.Logger, readers []*FileReader) duplicateVa return duplicateValidator{logger: logger, readers: readers} } -func (c *duplicateValidator) getDuplicates() *duplicateEntries { - duplicates := duplicateEntries{ - Titles: make(map[dashboardIdentity]*duplicate), - UIDs: make(map[string]*duplicate), - } +func (c *duplicateValidator) getDuplicates() map[int64]duplicateEntries { + duplicatesByOrg := map[int64]duplicateEntries{} for _, reader := range c.readers { readerName := reader.Cfg.Name + orgID := reader.Cfg.OrgID tracker := reader.getUsageTracker() - for uid, times := range tracker.uidUsage { - if _, ok := duplicates.UIDs[uid]; !ok { - duplicates.UIDs[uid] = newDuplicate() + if _, exists := duplicatesByOrg[orgID]; !exists { + duplicatesByOrg[orgID] = duplicateEntries{ + Titles: make(map[dashboardIdentity]*duplicate), + UIDs: make(map[string]*duplicate), } - duplicates.UIDs[uid].Sum += times - duplicates.UIDs[uid].InvolvedReaders[readerName] = struct{}{} + } + + for uid, times := range tracker.uidUsage { + if _, ok := duplicatesByOrg[orgID].UIDs[uid]; !ok { + duplicatesByOrg[orgID].UIDs[uid] = newDuplicate() + } + duplicatesByOrg[orgID].UIDs[uid].Sum += times + duplicatesByOrg[orgID].UIDs[uid].InvolvedReaders[readerName] = struct{}{} } for id, times := range tracker.titleUsage { - if _, ok := duplicates.Titles[id]; !ok { - duplicates.Titles[id] = newDuplicate() + if _, ok := duplicatesByOrg[orgID].Titles[id]; !ok { + duplicatesByOrg[orgID].Titles[id] = newDuplicate() } - duplicates.Titles[id].Sum += times - duplicates.Titles[id].InvolvedReaders[readerName] = struct{}{} + duplicatesByOrg[orgID].Titles[id].Sum += times + duplicatesByOrg[orgID].Titles[id].InvolvedReaders[readerName] = struct{}{} } } - return &duplicates + return duplicatesByOrg } -func (c *duplicateValidator) logWarnings(duplicates *duplicateEntries) { - for uid, usage := range duplicates.UIDs { - if usage.Sum > 1 { - c.logger.Warn("the same UID is used more than once", "uid", uid, "times", usage.Sum, "providers", - keysToSlice(usage.InvolvedReaders)) +func (c *duplicateValidator) logWarnings(duplicatesByOrg map[int64]duplicateEntries) { + for orgID, duplicates := range duplicatesByOrg { + for uid, usage := range duplicates.UIDs { + if usage.Sum > 1 { + c.logger.Warn("the same UID is used more than once", "orgId", orgID, "uid", uid, "times", usage.Sum, "providers", + keysToSlice(usage.InvolvedReaders)) + } } - } - for id, usage := range duplicates.Titles { - if usage.Sum > 1 { - c.logger.Warn("dashboard title is not unique in folder", "title", id.title, "folderID", id.folderID, "times", - usage.Sum, "providers", keysToSlice(usage.InvolvedReaders)) + for id, usage := range duplicates.Titles { + if usage.Sum > 1 { + c.logger.Warn("dashboard title is not unique in folder", "orgId", orgID, "title", id.title, "folderID", id.folderID, "times", + usage.Sum, "providers", keysToSlice(usage.InvolvedReaders)) + } } } } -func (c *duplicateValidator) takeAwayWritePermissions(duplicates *duplicateEntries) { - involvedReaders := duplicates.InvolvedReaders() +func (c *duplicateValidator) takeAwayWritePermissions(duplicatesByOrg map[int64]duplicateEntries) { + // reset write permissions for all readers for _, reader := range c.readers { - _, isReaderWithDuplicates := involvedReaders[reader.Cfg.Name] - // We restrict reader permissions to write to the database here to prevent overloading - reader.changeWritePermissions(isReaderWithDuplicates) + reader.changeWritePermissions(false) + } - if isReaderWithDuplicates { - c.logger.Warn("dashboards provisioning provider has no database write permissions because of duplicates", "provider", reader.Cfg.Name) + for orgID, duplicates := range duplicatesByOrg { + involvedReaders := duplicates.InvolvedReaders() + for _, reader := range c.readers { + _, exists := involvedReaders[reader.Cfg.Name] + + if exists { + // We restrict reader permissions to write to the database here to prevent overloading + reader.changeWritePermissions(true) + c.logger.Warn("dashboards provisioning provider has no database write permissions because of duplicates", "provider", reader.Cfg.Name, "orgId", orgID) + } } } } diff --git a/pkg/services/provisioning/dashboards/validator_test.go b/pkg/services/provisioning/dashboards/validator_test.go index d3bbcdada79..e1e588e75f9 100644 --- a/pkg/services/provisioning/dashboards/validator_test.go +++ b/pkg/services/provisioning/dashboards/validator_test.go @@ -62,13 +62,13 @@ func TestDuplicatesValidator(t *testing.T) { duplicates := duplicateValidator.getDuplicates() - require.Equal(t, uint8(2), duplicates.UIDs["Z-phNqGmz"].Sum) - uidUsageReaders := keysToSlice(duplicates.UIDs["Z-phNqGmz"].InvolvedReaders) + require.Equal(t, uint8(2), duplicates[1].UIDs["Z-phNqGmz"].Sum) + uidUsageReaders := keysToSlice(duplicates[1].UIDs["Z-phNqGmz"].InvolvedReaders) sort.Strings(uidUsageReaders) require.Equal(t, []string{"first", "second"}, uidUsageReaders) - require.Equal(t, uint8(2), duplicates.Titles[identity].Sum) - titleUsageReaders := keysToSlice(duplicates.Titles[identity].InvolvedReaders) + require.Equal(t, uint8(2), duplicates[1].Titles[identity].Sum) + titleUsageReaders := keysToSlice(duplicates[1].Titles[identity].InvolvedReaders) sort.Strings(titleUsageReaders) require.Equal(t, []string{"first", "second"}, titleUsageReaders) @@ -77,14 +77,20 @@ func TestDuplicatesValidator(t *testing.T) { require.True(t, reader2.isDatabaseAccessRestricted()) }) - t.Run("Duplicates validator should restrict write access only for readers with duplicates", func(t *testing.T) { + t.Run("Duplicates validator should not collect info about duplicate UIDs and titles within folders for different orgs", func(t *testing.T) { + const folderName = "duplicates-validator-folder" + folderID, err := getOrCreateFolderID(context.Background(), cfg, fakeService, folderName) + require.NoError(t, err) + + identity := dashboardIdentity{folderID: folderID, title: "Grafana"} + cfg1 := &config{ - Name: "first", Type: "file", OrgID: 1, Folder: "duplicates-validator-folder", - Options: map[string]interface{}{"path": twoDashboardsWithUID}, + Name: "first", Type: "file", OrgID: 1, Folder: folderName, + Options: map[string]interface{}{"path": dashboardContainingUID}, } cfg2 := &config{ - Name: "second", Type: "file", OrgID: 1, Folder: "root", - Options: map[string]interface{}{"path": defaultDashboards}, + Name: "second", Type: "file", OrgID: 2, Folder: folderName, + Options: map[string]interface{}{"path": dashboardContainingUID}, } reader1, err := NewDashboardFileReader(cfg1, logger, nil) @@ -103,23 +109,100 @@ func TestDuplicatesValidator(t *testing.T) { duplicates := duplicateValidator.getDuplicates() + require.Equal(t, uint8(1), duplicates[1].UIDs["Z-phNqGmz"].Sum) + uidUsageReaders := keysToSlice(duplicates[1].UIDs["Z-phNqGmz"].InvolvedReaders) + sort.Strings(uidUsageReaders) + require.Equal(t, []string{"first"}, uidUsageReaders) + + require.Equal(t, uint8(1), duplicates[2].UIDs["Z-phNqGmz"].Sum) + uidUsageReaders = keysToSlice(duplicates[2].UIDs["Z-phNqGmz"].InvolvedReaders) + sort.Strings(uidUsageReaders) + require.Equal(t, []string{"second"}, uidUsageReaders) + + require.Equal(t, uint8(1), duplicates[1].Titles[identity].Sum) + titleUsageReaders := keysToSlice(duplicates[1].Titles[identity].InvolvedReaders) + sort.Strings(titleUsageReaders) + require.Equal(t, []string{"first"}, titleUsageReaders) + + require.Equal(t, uint8(1), duplicates[2].Titles[identity].Sum) + titleUsageReaders = keysToSlice(duplicates[2].Titles[identity].InvolvedReaders) + sort.Strings(titleUsageReaders) + require.Equal(t, []string{"second"}, titleUsageReaders) + + duplicateValidator.validate() + require.False(t, reader1.isDatabaseAccessRestricted()) + require.False(t, reader2.isDatabaseAccessRestricted()) + }) + + t.Run("Duplicates validator should restrict write access only for readers with duplicates", func(t *testing.T) { + cfg1 := &config{ + Name: "first", Type: "file", OrgID: 1, Folder: "duplicates-validator-folder", + Options: map[string]interface{}{"path": twoDashboardsWithUID}, + } + cfg2 := &config{ + Name: "second", Type: "file", OrgID: 1, Folder: "root", + Options: map[string]interface{}{"path": defaultDashboards}, + } + cfg3 := &config{ + Name: "third", Type: "file", OrgID: 2, Folder: "duplicates-validator-folder", + Options: map[string]interface{}{"path": twoDashboardsWithUID}, + } + + reader1, err := NewDashboardFileReader(cfg1, logger, nil) + require.NoError(t, err) + + reader2, err := NewDashboardFileReader(cfg2, logger, nil) + require.NoError(t, err) + + reader3, err := NewDashboardFileReader(cfg3, logger, nil) + require.NoError(t, err) + + duplicateValidator := newDuplicateValidator(logger, []*FileReader{reader1, reader2, reader3}) + + err = reader1.walkDisk(context.Background()) + require.NoError(t, err) + + err = reader2.walkDisk(context.Background()) + require.NoError(t, err) + + err = reader3.walkDisk(context.Background()) + require.NoError(t, err) + + duplicates := duplicateValidator.getDuplicates() + folderID, err := getOrCreateFolderID(context.Background(), cfg, fakeService, cfg1.Folder) require.NoError(t, err) identity := dashboardIdentity{folderID: folderID, title: "Grafana"} - require.Equal(t, uint8(2), duplicates.UIDs["Z-phNqGmz"].Sum) - uidUsageReaders := keysToSlice(duplicates.UIDs["Z-phNqGmz"].InvolvedReaders) + require.Equal(t, uint8(2), duplicates[1].UIDs["Z-phNqGmz"].Sum) + uidUsageReaders := keysToSlice(duplicates[1].UIDs["Z-phNqGmz"].InvolvedReaders) sort.Strings(uidUsageReaders) require.Equal(t, []string{"first"}, uidUsageReaders) - require.Equal(t, uint8(2), duplicates.Titles[identity].Sum) - titleUsageReaders := keysToSlice(duplicates.Titles[identity].InvolvedReaders) + require.Equal(t, uint8(2), duplicates[1].Titles[identity].Sum) + titleUsageReaders := keysToSlice(duplicates[1].Titles[identity].InvolvedReaders) sort.Strings(titleUsageReaders) require.Equal(t, []string{"first"}, titleUsageReaders) + folderID, err = getOrCreateFolderID(context.Background(), cfg3, fakeService, cfg3.Folder) + require.NoError(t, err) + + identity = dashboardIdentity{folderID: folderID, title: "Grafana"} + + require.Equal(t, uint8(2), duplicates[2].UIDs["Z-phNqGmz"].Sum) + uidUsageReaders = keysToSlice(duplicates[2].UIDs["Z-phNqGmz"].InvolvedReaders) + sort.Strings(uidUsageReaders) + require.Equal(t, []string{"third"}, uidUsageReaders) + + require.Equal(t, uint8(2), duplicates[2].Titles[identity].Sum) + titleUsageReaders = keysToSlice(duplicates[2].Titles[identity].InvolvedReaders) + sort.Strings(titleUsageReaders) + require.Equal(t, []string{"third"}, titleUsageReaders) + duplicateValidator.validate() require.True(t, reader1.isDatabaseAccessRestricted()) require.False(t, reader2.isDatabaseAccessRestricted()) + require.True(t, reader3.isDatabaseAccessRestricted()) }) } diff --git a/pkg/services/provisioning/plugins/plugin_provisioner_test.go b/pkg/services/provisioning/plugins/plugin_provisioner_test.go index ba83768b158..98a30436cb5 100644 --- a/pkg/services/provisioning/plugins/plugin_provisioner_test.go +++ b/pkg/services/provisioning/plugins/plugin_provisioner_test.go @@ -59,6 +59,7 @@ func TestPluginProvisioner(t *testing.T) { } reader := &testConfigReader{result: cfg} ap := PluginProvisioner{log: log.New("test"), cfgProvider: reader} + err := ap.applyChanges(context.Background(), "") require.NoError(t, err) require.Len(t, sentCommands, 4) diff --git a/pkg/services/provisioning/utils/utils_test.go b/pkg/services/provisioning/utils/utils_test.go index 88019f932b7..33b5710bb45 100644 --- a/pkg/services/provisioning/utils/utils_test.go +++ b/pkg/services/provisioning/utils/utils_test.go @@ -15,6 +15,7 @@ func TestCheckOrgExists(t *testing.T) { sqlstore.InitTestDB(t) defaultOrg := models.CreateOrgCommand{Name: "Main Org."} + err := sqlstore.CreateOrg(context.Background(), &defaultOrg) require.NoError(t, err) diff --git a/pkg/services/queryhistory/api.go b/pkg/services/queryhistory/api.go new file mode 100644 index 00000000000..929b99feb32 --- /dev/null +++ b/pkg/services/queryhistory/api.go @@ -0,0 +1,31 @@ +package queryhistory + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/middleware" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/web" +) + +func (s *QueryHistoryService) registerAPIEndpoints() { + s.RouteRegister.Group("/api/query-history", func(entities routing.RouteRegister) { + entities.Post("/", middleware.ReqSignedIn, routing.Wrap(s.createHandler)) + }) +} + +func (s *QueryHistoryService) createHandler(c *models.ReqContext) response.Response { + cmd := CreateQueryInQueryHistoryCommand{} + if err := web.Bind(c.Req, &cmd); err != nil { + return response.Error(http.StatusBadRequest, "bad request data", err) + } + + err := s.CreateQueryInQueryHistory(c.Req.Context(), c.SignedInUser, cmd) + if err != nil { + return response.Error(500, "Failed to create query history", err) + } + + return response.Success("Query successfully added to query history") +} diff --git a/pkg/services/queryhistory/database.go b/pkg/services/queryhistory/database.go new file mode 100644 index 00000000000..a7ed8951268 --- /dev/null +++ b/pkg/services/queryhistory/database.go @@ -0,0 +1,32 @@ +package queryhistory + +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/util" +) + +func (s QueryHistoryService) createQuery(ctx context.Context, user *models.SignedInUser, cmd CreateQueryInQueryHistoryCommand) error { + queryHistory := QueryHistory{ + OrgId: user.OrgId, + Uid: util.GenerateShortUID(), + Queries: cmd.Queries, + DatasourceUid: cmd.DatasourceUid, + CreatedBy: user.UserId, + CreatedAt: time.Now().Unix(), + Comment: "", + } + + err := s.SQLStore.WithDbSession(ctx, func(session *sqlstore.DBSession) error { + _, err := session.Insert(&queryHistory) + return err + }) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/services/queryhistory/models.go b/pkg/services/queryhistory/models.go new file mode 100644 index 00000000000..21490529da7 --- /dev/null +++ b/pkg/services/queryhistory/models.go @@ -0,0 +1,21 @@ +package queryhistory + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" +) + +type QueryHistory struct { + Id int64 `json:"id"` + Uid string `json:"uid"` + DatasourceUid string `json:"datasourceUid"` + OrgId int64 `json:"orgId"` + CreatedBy int64 `json:"createdBy"` + CreatedAt int64 `json:"createdAt"` + Comment string `json:"comment"` + Queries *simplejson.Json `json:"queries"` +} + +type CreateQueryInQueryHistoryCommand struct { + DatasourceUid string `json:"datasourceUid"` + Queries *simplejson.Json `json:"queries"` +} diff --git a/pkg/services/queryhistory/queryhistory.go b/pkg/services/queryhistory/queryhistory.go new file mode 100644 index 00000000000..7614121205b --- /dev/null +++ b/pkg/services/queryhistory/queryhistory.go @@ -0,0 +1,42 @@ +package queryhistory + +import ( + "context" + + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" +) + +func ProvideService(cfg *setting.Cfg, sqlStore *sqlstore.SQLStore, routeRegister routing.RouteRegister) *QueryHistoryService { + s := &QueryHistoryService{ + SQLStore: sqlStore, + Cfg: cfg, + RouteRegister: routeRegister, + log: log.New("query-history"), + } + + // Register routes only when query history is enabled + if s.Cfg.QueryHistoryEnabled { + s.registerAPIEndpoints() + } + + return s +} + +type Service interface { + CreateQueryInQueryHistory(ctx context.Context, user *models.SignedInUser, cmd CreateQueryInQueryHistoryCommand) error +} + +type QueryHistoryService struct { + SQLStore *sqlstore.SQLStore + Cfg *setting.Cfg + RouteRegister routing.RouteRegister + log log.Logger +} + +func (s QueryHistoryService) CreateQueryInQueryHistory(ctx context.Context, user *models.SignedInUser, cmd CreateQueryInQueryHistoryCommand) error { + return s.createQuery(ctx, user, cmd) +} diff --git a/pkg/services/queryhistory/queryhistory_create_test.go b/pkg/services/queryhistory/queryhistory_create_test.go new file mode 100644 index 00000000000..588bc572e26 --- /dev/null +++ b/pkg/services/queryhistory/queryhistory_create_test.go @@ -0,0 +1,23 @@ +package queryhistory + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/stretchr/testify/require" +) + +func TestCreateQueryInQueryHistory(t *testing.T) { + testScenario(t, "When users tries to create query in query history it should succeed", + func(t *testing.T, sc scenarioContext) { + command := CreateQueryInQueryHistoryCommand{ + DatasourceUid: "NCzh67i", + Queries: simplejson.NewFromAny(map[string]interface{}{ + "expr": "test", + }), + } + sc.reqContext.Req.Body = mockRequestBody(command) + resp := sc.service.createHandler(sc.reqContext) + require.Equal(t, 200, resp.Status()) + }) +} diff --git a/pkg/services/queryhistory/queryhistory_test.go b/pkg/services/queryhistory/queryhistory_test.go new file mode 100644 index 00000000000..599526fb89b --- /dev/null +++ b/pkg/services/queryhistory/queryhistory_test.go @@ -0,0 +1,77 @@ +package queryhistory + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "testing" + "time" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/web" + "github.com/stretchr/testify/require" +) + +var ( + testOrgID = int64(1) + testUserID = int64(1) +) + +type scenarioContext struct { + ctx *web.Context + service *QueryHistoryService + reqContext *models.ReqContext + sqlStore *sqlstore.SQLStore +} + +func testScenario(t *testing.T, desc string, fn func(t *testing.T, sc scenarioContext)) { + t.Helper() + + t.Run(desc, func(t *testing.T) { + ctx := web.Context{Req: &http.Request{}} + sqlStore := sqlstore.InitTestDB(t) + service := QueryHistoryService{ + Cfg: setting.NewCfg(), + SQLStore: sqlStore, + } + + service.Cfg.QueryHistoryEnabled = true + + user := models.SignedInUser{ + UserId: testUserID, + Name: "Signed In User", + Login: "signed_in_user", + Email: "signed.in.user@test.com", + OrgId: testOrgID, + OrgRole: models.ROLE_VIEWER, + LastSeenAt: time.Now(), + } + + _, err := sqlStore.CreateUser(context.Background(), models.CreateUserCommand{ + Email: "signed.in.user@test.com", + Name: "Signed In User", + Login: "signed_in_user", + }) + require.NoError(t, err) + + sc := scenarioContext{ + ctx: &ctx, + service: &service, + sqlStore: sqlStore, + reqContext: &models.ReqContext{ + Context: &ctx, + SignedInUser: &user, + }, + } + fn(t, sc) + }) +} + +func mockRequestBody(v interface{}) io.ReadCloser { + b, _ := json.Marshal(v) + return io.NopCloser(bytes.NewReader(b)) +} diff --git a/pkg/services/rendering/auth.go b/pkg/services/rendering/auth.go new file mode 100644 index 00000000000..f68067c5ed9 --- /dev/null +++ b/pkg/services/rendering/auth.go @@ -0,0 +1,120 @@ +package rendering + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/remotecache" + "github.com/grafana/grafana/pkg/util" +) + +const renderKeyPrefix = "render-%s" + +type RenderUser struct { + OrgID int64 + UserID int64 + OrgRole string +} + +func (rs *RenderingService) GetRenderUser(ctx context.Context, key string) (*RenderUser, bool) { + val, err := rs.RemoteCacheService.Get(ctx, fmt.Sprintf(renderKeyPrefix, key)) + if err != nil { + rs.log.Error("Failed to get render key from cache", "error", err) + } + + if val != nil { + if user, ok := val.(*RenderUser); ok { + return user, true + } + } + + return nil, false +} + +func setRenderKey(cache *remotecache.RemoteCache, ctx context.Context, opts AuthOpts, renderKey string, expiry time.Duration) error { + err := cache.Set(ctx, fmt.Sprintf(renderKeyPrefix, renderKey), &RenderUser{ + OrgID: opts.OrgID, + UserID: opts.UserID, + OrgRole: string(opts.OrgRole), + }, expiry) + return err +} + +func generateAndSetRenderKey(cache *remotecache.RemoteCache, ctx context.Context, opts AuthOpts, expiry time.Duration) (string, error) { + key, err := util.GetRandomString(32) + if err != nil { + return "", err + } + + err = setRenderKey(cache, ctx, opts, key, expiry) + if err != nil { + return "", err + } + + return key, nil +} + +type longLivedRenderKeyProvider struct { + cache *remotecache.RemoteCache + log log.Logger + renderKey string + authOpts AuthOpts + sessionOpts SessionOpts +} + +func (rs *RenderingService) CreateRenderingSession(ctx context.Context, opts AuthOpts, sessionOpts SessionOpts) (Session, error) { + renderKey, err := generateAndSetRenderKey(rs.RemoteCacheService, ctx, opts, sessionOpts.Expiry) + if err != nil { + return nil, err + } + + return &longLivedRenderKeyProvider{ + log: rs.log, + renderKey: renderKey, + cache: rs.RemoteCacheService, + authOpts: opts, + sessionOpts: sessionOpts, + }, nil +} + +func deleteRenderKey(cache *remotecache.RemoteCache, log log.Logger, ctx context.Context, renderKey string) { + err := cache.Delete(ctx, fmt.Sprintf(renderKeyPrefix, renderKey)) + if err != nil { + log.Error("Failed to delete render key", "error", err) + } +} + +type perRequestRenderKeyProvider struct { + cache *remotecache.RemoteCache + log log.Logger + keyExpiry time.Duration +} + +func (r *perRequestRenderKeyProvider) get(ctx context.Context, opts AuthOpts) (string, error) { + return generateAndSetRenderKey(r.cache, ctx, opts, r.keyExpiry) +} + +func (r *perRequestRenderKeyProvider) afterRequest(ctx context.Context, opts AuthOpts, renderKey string) { + deleteRenderKey(r.cache, r.log, ctx, renderKey) +} + +func (r *longLivedRenderKeyProvider) get(ctx context.Context, opts AuthOpts) (string, error) { + if r.sessionOpts.RefreshExpiryOnEachRequest { + err := setRenderKey(r.cache, ctx, opts, r.renderKey, r.sessionOpts.Expiry) + if err != nil { + r.log.Error("Failed to refresh render key", "error", err, "renderKey", r.renderKey) + } + } + return r.renderKey, nil +} + +func (r *longLivedRenderKeyProvider) afterRequest(ctx context.Context, opts AuthOpts, renderKey string) { + // do nothing - renderKey from longLivedRenderKeyProvider is deleted only after session expires + // or someone calls session.Dispose() +} + +func (r *longLivedRenderKeyProvider) Dispose(ctx context.Context) { + deleteRenderKey(r.cache, r.log, ctx, r.renderKey) +} diff --git a/pkg/services/rendering/capabilities.go b/pkg/services/rendering/capabilities.go new file mode 100644 index 00000000000..9339bfc4915 --- /dev/null +++ b/pkg/services/rendering/capabilities.go @@ -0,0 +1,55 @@ +package rendering + +import ( + "errors" + + "github.com/Masterminds/semver" +) + +type Capability struct { + name CapabilityName + semverConstraint string +} + +type CapabilityName string + +const ( + ScalingDownImages CapabilityName = "ScalingDownImages" + FullHeightImages CapabilityName = "FullHeightImages" +) + +var ErrUnknownCapability = errors.New("unknown capability") +var ErrInvalidPluginVersion = errors.New("invalid plugin version") + +func (rs *RenderingService) HasCapability(capability CapabilityName) (CapabilitySupportRequestResult, error) { + if !rs.IsAvailable() { + return CapabilitySupportRequestResult{IsSupported: false, SemverConstraint: ""}, ErrRenderUnavailable + } + + var semverConstraint string + for i := range rs.capabilities { + if rs.capabilities[i].name == capability { + semverConstraint = rs.capabilities[i].semverConstraint + break + } + } + + if semverConstraint == "" { + return CapabilitySupportRequestResult{}, ErrUnknownCapability + } + + compiledSemverConstraint, err := semver.NewConstraint(semverConstraint) + if err != nil { + rs.log.Error("Failed to parse semver constraint", "constraint", semverConstraint, "capability", capability, "error", err.Error()) + return CapabilitySupportRequestResult{IsSupported: false, SemverConstraint: semverConstraint}, ErrUnknownCapability + } + + imageRendererVersion := rs.Version() + compiledImageRendererVersion, err := semver.NewVersion(imageRendererVersion) + if err != nil { + rs.log.Error("Failed to parse plugin version", "version", imageRendererVersion, "error", err.Error()) + return CapabilitySupportRequestResult{IsSupported: false, SemverConstraint: semverConstraint}, ErrInvalidPluginVersion + } + + return CapabilitySupportRequestResult{IsSupported: compiledSemverConstraint.Check(compiledImageRendererVersion), SemverConstraint: semverConstraint}, nil +} diff --git a/pkg/services/rendering/capabilities_test.go b/pkg/services/rendering/capabilities_test.go new file mode 100644 index 00000000000..eec3382c96f --- /dev/null +++ b/pkg/services/rendering/capabilities_test.go @@ -0,0 +1,136 @@ +package rendering + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" +) + +type dummyPluginManager struct{} + +func (d *dummyPluginManager) Renderer() *plugins.Plugin { + return nil +} + +var dummyRendererUrl = "http://dummyurl.com" +var testCapabilitySemverConstraint = "> 1.0.0" +var testCapabilityName = CapabilityName("TestCap") +var testCapabilityNameInvalidSemver = CapabilityName("TestCapInvalidSemver") + +func TestCapabilities(t *testing.T) { + cfg := setting.NewCfg() + rs := &RenderingService{ + Cfg: cfg, + RendererPluginManager: &dummyPluginManager{}, + log: log.New("test-capabilities-rendering-service"), + capabilities: []Capability{ + {name: testCapabilityName, semverConstraint: testCapabilitySemverConstraint}, + {name: testCapabilityNameInvalidSemver, semverConstraint: "asfasf"}, + }, + } + + tests := []struct { + name string + rendererUrl string + rendererVersion string + capabilityName CapabilityName + expectedError error + expectedResult CapabilitySupportRequestResult + }{ + { + name: "when image-renderer plugin is not available", + rendererUrl: "", + rendererVersion: "", + capabilityName: testCapabilityName, + expectedError: ErrRenderUnavailable, + expectedResult: CapabilitySupportRequestResult{ + IsSupported: false, + SemverConstraint: "", + }, + }, + { + name: "when image-renderer plugin version is not populated", + rendererUrl: dummyRendererUrl, + rendererVersion: "", + capabilityName: testCapabilityName, + expectedError: ErrInvalidPluginVersion, + expectedResult: CapabilitySupportRequestResult{ + IsSupported: false, + SemverConstraint: testCapabilitySemverConstraint, + }, + }, + { + name: "when image-renderer plugin version is not valid", + rendererUrl: dummyRendererUrl, + rendererVersion: "abcd", + capabilityName: testCapabilityName, + expectedError: ErrInvalidPluginVersion, + expectedResult: CapabilitySupportRequestResult{ + IsSupported: false, + SemverConstraint: testCapabilitySemverConstraint, + }, + }, + { + name: "when image-renderer version does not match target constraint", + rendererUrl: dummyRendererUrl, + rendererVersion: "1.0.0", + capabilityName: testCapabilityName, + expectedError: nil, + expectedResult: CapabilitySupportRequestResult{ + IsSupported: false, + SemverConstraint: testCapabilitySemverConstraint, + }, + }, + { + name: "when image-renderer version matches target constraint", + rendererUrl: dummyRendererUrl, + rendererVersion: "2.0.0", + capabilityName: testCapabilityName, + expectedError: nil, + expectedResult: CapabilitySupportRequestResult{ + IsSupported: true, + SemverConstraint: testCapabilitySemverConstraint, + }, + }, + { + name: "when capability is unknown", + rendererUrl: dummyRendererUrl, + rendererVersion: "1.0.0", + capabilityName: CapabilityName("unknown"), + expectedError: ErrUnknownCapability, + expectedResult: CapabilitySupportRequestResult{ + IsSupported: false, + SemverConstraint: "", + }, + }, + { + name: "when capability has invalid semver constraint", + rendererUrl: dummyRendererUrl, + rendererVersion: "1.0.0", + capabilityName: testCapabilityNameInvalidSemver, + expectedError: ErrUnknownCapability, + expectedResult: CapabilitySupportRequestResult{ + IsSupported: false, + SemverConstraint: "asfasf", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rs.Cfg.RendererUrl = tt.rendererUrl + rs.version = tt.rendererVersion + res, err := rs.HasCapability(tt.capabilityName) + + if tt.expectedError == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, tt.expectedError) + } + require.Equal(t, tt.expectedResult, res) + }) + } +} diff --git a/pkg/services/rendering/http_mode.go b/pkg/services/rendering/http_mode.go index 5511b300387..3617b3296bd 100644 --- a/pkg/services/rendering/http_mode.go +++ b/pkg/services/rendering/http_mode.go @@ -58,7 +58,7 @@ func (rs *RenderingService) renderViaHTTP(ctx context.Context, renderKey string, rendererURL.RawQuery = queryParams.Encode() // gives service some additional time to timeout and return possible errors. - reqContext, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2) + reqContext, cancel := context.WithTimeout(ctx, getRequestTimeout(opts.TimeoutOpts)) defer cancel() resp, err := rs.doRequest(reqContext, rendererURL, opts.Headers) @@ -103,7 +103,7 @@ func (rs *RenderingService) renderCSVViaHTTP(ctx context.Context, renderKey stri rendererURL.RawQuery = queryParams.Encode() // gives service some additional time to timeout and return possible errors. - reqContext, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2) + reqContext, cancel := context.WithTimeout(ctx, getRequestTimeout(opts.TimeoutOpts)) defer cancel() resp, err := rs.doRequest(reqContext, rendererURL, opts.Headers) diff --git a/pkg/services/rendering/interface.go b/pkg/services/rendering/interface.go index aa085427806..9eaa166b2a1 100644 --- a/pkg/services/rendering/interface.go +++ b/pkg/services/rendering/interface.go @@ -26,13 +26,30 @@ const ( ThemeDark Theme = "dark" ) +type TimeoutOpts struct { + Timeout time.Duration // Timeout param passed to image-renderer service + RequestTimeoutMultiplier time.Duration // RequestTimeoutMultiplier used for plugin/HTTP request context timeout +} + +type AuthOpts struct { + OrgID int64 + UserID int64 + OrgRole models.RoleType +} + +func getRequestTimeout(opt TimeoutOpts) time.Duration { + if opt.RequestTimeoutMultiplier == 0 { + return opt.Timeout * 2 // default + } + + return opt.Timeout * opt.RequestTimeoutMultiplier +} + type Opts struct { + TimeoutOpts + AuthOpts Width int Height int - Timeout time.Duration - OrgID int64 - UserID int64 - OrgRole models.RoleType Path string Encoding string Timezone string @@ -43,10 +60,8 @@ type Opts struct { } type CSVOpts struct { - Timeout time.Duration - OrgID int64 - UserID int64 - OrgRole models.RoleType + TimeoutOpts + AuthOpts Path string Encoding string Timezone string @@ -66,11 +81,33 @@ type RenderCSVResult struct { type renderFunc func(ctx context.Context, renderKey string, options Opts) (*RenderResult, error) type renderCSVFunc func(ctx context.Context, renderKey string, options CSVOpts) (*RenderCSVResult, error) +type renderKeyProvider interface { + get(ctx context.Context, opts AuthOpts) (string, error) + afterRequest(ctx context.Context, opts AuthOpts, renderKey string) +} + +type SessionOpts struct { + Expiry time.Duration + RefreshExpiryOnEachRequest bool +} + +type Session interface { + renderKeyProvider + Dispose(ctx context.Context) +} + +type CapabilitySupportRequestResult struct { + IsSupported bool + SemverConstraint string +} + type Service interface { IsAvailable() bool Version() string - Render(ctx context.Context, opts Opts) (*RenderResult, error) - RenderCSV(ctx context.Context, opts CSVOpts) (*RenderCSVResult, error) + Render(ctx context.Context, opts Opts, session Session) (*RenderResult, error) + RenderCSV(ctx context.Context, opts CSVOpts, session Session) (*RenderCSVResult, error) RenderErrorImage(theme Theme, error error) (*RenderResult, error) GetRenderUser(ctx context.Context, key string) (*RenderUser, bool) + HasCapability(capability CapabilityName) (CapabilitySupportRequestResult, error) + CreateRenderingSession(ctx context.Context, authOpts AuthOpts, sessionOpts SessionOpts) (Session, error) } diff --git a/pkg/services/rendering/plugin_mode.go b/pkg/services/rendering/plugin_mode.go index 83bee4fe6e5..5e5c247f7b3 100644 --- a/pkg/services/rendering/plugin_mode.go +++ b/pkg/services/rendering/plugin_mode.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "time" "github.com/grafana/grafana/pkg/plugins/backendplugin/pluginextensionv2" ) @@ -15,7 +14,7 @@ func (rs *RenderingService) startPlugin(ctx context.Context) error { func (rs *RenderingService) renderViaPlugin(ctx context.Context, renderKey string, opts Opts) (*RenderResult, error) { // gives plugin some additional time to timeout and return possible errors. - ctx, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2) + ctx, cancel := context.WithTimeout(ctx, getRequestTimeout(opts.TimeoutOpts)) defer cancel() filePath, err := rs.getNewFilePath(RenderPNG) @@ -62,7 +61,7 @@ func (rs *RenderingService) renderViaPlugin(ctx context.Context, renderKey strin func (rs *RenderingService) renderCSVViaPlugin(ctx context.Context, renderKey string, opts CSVOpts) (*RenderCSVResult, error) { // gives plugin some additional time to timeout and return possible errors. - ctx, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2) + ctx, cancel := context.WithTimeout(ctx, getRequestTimeout(opts.TimeoutOpts)) defer cancel() filePath, err := rs.getNewFilePath(RenderCSV) diff --git a/pkg/services/rendering/rendering.go b/pkg/services/rendering/rendering.go index b6ec052f959..6e069cd4eb7 100644 --- a/pkg/services/rendering/rendering.go +++ b/pkg/services/rendering/rendering.go @@ -17,7 +17,6 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/infra/remotecache" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -28,13 +27,6 @@ func init() { } const ServiceName = "RenderingService" -const renderKeyPrefix = "render-%s" - -type RenderUser struct { - OrgID int64 - UserID int64 - OrgRole string -} type RenderingService struct { log log.Logger @@ -45,10 +37,12 @@ type RenderingService struct { inProgressCount int32 version string versionMutex sync.RWMutex + capabilities []Capability - Cfg *setting.Cfg - RemoteCacheService *remotecache.RemoteCache - RendererPluginManager plugins.RendererManager + perRequestRenderKeyProvider renderKeyProvider + Cfg *setting.Cfg + RemoteCacheService *remotecache.RemoteCache + RendererPluginManager plugins.RendererManager } func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, rm plugins.RendererManager) (*RenderingService, error) { @@ -81,11 +75,27 @@ func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, rm p domain = "localhost" } + logger := log.New("rendering") s := &RenderingService{ + perRequestRenderKeyProvider: &perRequestRenderKeyProvider{ + cache: remoteCache, + log: logger, + keyExpiry: 5 * time.Minute, + }, + capabilities: []Capability{ + { + name: FullHeightImages, + semverConstraint: ">= 3.4.0", + }, + { + name: ScalingDownImages, + semverConstraint: ">= 3.4.0", + }, + }, Cfg: cfg, RemoteCacheService: remoteCache, RendererPluginManager: rm, - log: log.New("rendering"), + log: logger, domain: domain, } return s, nil @@ -195,9 +205,14 @@ func (rs *RenderingService) renderUnavailableImage() *RenderResult { } } -func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResult, error) { +func (rs *RenderingService) Render(ctx context.Context, opts Opts, session Session) (*RenderResult, error) { startTime := time.Now() - result, err := rs.render(ctx, opts) + + renderKeyProvider := rs.perRequestRenderKeyProvider + if session != nil { + renderKeyProvider = session + } + result, err := rs.render(ctx, opts, renderKeyProvider) elapsedTime := time.Since(startTime).Milliseconds() saveMetrics(elapsedTime, err, RenderPNG) @@ -205,7 +220,7 @@ func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResul return result, err } -func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResult, error) { +func (rs *RenderingService) render(ctx context.Context, opts Opts, renderKeyProvider renderKeyProvider) (*RenderResult, error) { if int(atomic.LoadInt32(&rs.inProgressCount)) > opts.ConcurrentLimit { rs.log.Warn("Could not render image, hit the currency limit", "concurrencyLimit", opts.ConcurrentLimit, "path", opts.Path) @@ -230,12 +245,12 @@ func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResul if math.IsInf(opts.DeviceScaleFactor, 0) || math.IsNaN(opts.DeviceScaleFactor) || opts.DeviceScaleFactor == 0 { opts.DeviceScaleFactor = 1 } - renderKey, err := rs.generateAndStoreRenderKey(ctx, opts.OrgID, opts.UserID, opts.OrgRole) + renderKey, err := renderKeyProvider.get(ctx, opts.AuthOpts) if err != nil { return nil, err } - defer rs.deleteRenderKey(ctx, renderKey) + defer renderKeyProvider.afterRequest(ctx, opts.AuthOpts, renderKey) defer func() { metrics.MRenderingQueue.Set(float64(atomic.AddInt32(&rs.inProgressCount, -1))) @@ -245,9 +260,14 @@ func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResul return rs.renderAction(ctx, renderKey, opts) } -func (rs *RenderingService) RenderCSV(ctx context.Context, opts CSVOpts) (*RenderCSVResult, error) { +func (rs *RenderingService) RenderCSV(ctx context.Context, opts CSVOpts, session Session) (*RenderCSVResult, error) { startTime := time.Now() - result, err := rs.renderCSV(ctx, opts) + + renderKeyProvider := rs.perRequestRenderKeyProvider + if session != nil { + renderKeyProvider = session + } + result, err := rs.renderCSV(ctx, opts, renderKeyProvider) elapsedTime := time.Since(startTime).Milliseconds() saveMetrics(elapsedTime, err, RenderCSV) @@ -255,7 +275,7 @@ func (rs *RenderingService) RenderCSV(ctx context.Context, opts CSVOpts) (*Rende return result, err } -func (rs *RenderingService) renderCSV(ctx context.Context, opts CSVOpts) (*RenderCSVResult, error) { +func (rs *RenderingService) renderCSV(ctx context.Context, opts CSVOpts, renderKeyProvider renderKeyProvider) (*RenderCSVResult, error) { if int(atomic.LoadInt32(&rs.inProgressCount)) > opts.ConcurrentLimit { return nil, ErrConcurrentLimitReached } @@ -265,12 +285,12 @@ func (rs *RenderingService) renderCSV(ctx context.Context, opts CSVOpts) (*Rende } rs.log.Info("Rendering", "path", opts.Path) - renderKey, err := rs.generateAndStoreRenderKey(ctx, opts.OrgID, opts.UserID, opts.OrgRole) + renderKey, err := renderKeyProvider.get(ctx, opts.AuthOpts) if err != nil { return nil, err } - defer rs.deleteRenderKey(ctx, renderKey) + defer renderKeyProvider.afterRequest(ctx, opts.AuthOpts, renderKey) defer func() { metrics.MRenderingQueue.Set(float64(atomic.AddInt32(&rs.inProgressCount, -1))) @@ -280,21 +300,6 @@ func (rs *RenderingService) renderCSV(ctx context.Context, opts CSVOpts) (*Rende return rs.renderCSVAction(ctx, renderKey, opts) } -func (rs *RenderingService) GetRenderUser(ctx context.Context, key string) (*RenderUser, bool) { - val, err := rs.RemoteCacheService.Get(ctx, fmt.Sprintf(renderKeyPrefix, key)) - if err != nil { - rs.log.Error("Failed to get render key from cache", "error", err) - } - - if val != nil { - if user, ok := val.(*RenderUser); ok { - return user, true - } - } - - return nil, false -} - func (rs *RenderingService) getNewFilePath(rt RenderType) (string, error) { rand, err := util.GetRandomString(20) if err != nil { @@ -340,31 +345,6 @@ func (rs *RenderingService) getURL(path string) string { return fmt.Sprintf("%s://%s:%s%s/%s&render=1", protocol, rs.domain, rs.Cfg.HTTPPort, subPath, path) } -func (rs *RenderingService) generateAndStoreRenderKey(ctx context.Context, orgId, userId int64, orgRole models.RoleType) (string, error) { - key, err := util.GetRandomString(32) - if err != nil { - return "", err - } - - err = rs.RemoteCacheService.Set(ctx, fmt.Sprintf(renderKeyPrefix, key), &RenderUser{ - OrgID: orgId, - UserID: userId, - OrgRole: string(orgRole), - }, 5*time.Minute) - if err != nil { - return "", err - } - - return key, nil -} - -func (rs *RenderingService) deleteRenderKey(ctx context.Context, key string) { - err := rs.RemoteCacheService.Delete(ctx, fmt.Sprintf(renderKeyPrefix, key)) - if err != nil { - rs.log.Error("Failed to delete render key", "error", err) - } -} - func isoTimeOffsetToPosixTz(isoOffset string) string { // invert offset if strings.HasPrefix(isoOffset, "UTC+") { diff --git a/pkg/services/rendering/rendering_test.go b/pkg/services/rendering/rendering_test.go index 06ddcc7c9f3..954f739ac20 100644 --- a/pkg/services/rendering/rendering_test.go +++ b/pkg/services/rendering/rendering_test.go @@ -138,7 +138,7 @@ func TestRenderLimitImage(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { opts := Opts{Theme: tc.theme, ConcurrentLimit: 1} - result, err := rs.Render(context.Background(), opts) + result, err := rs.Render(context.Background(), opts, nil) assert.NoError(t, err) assert.Equal(t, tc.expected, result.FilePath) }) diff --git a/pkg/services/schemaloader/schemaloader.go b/pkg/services/schemaloader/schemaloader.go index e10a2661dbc..2b9a3e6ddb7 100644 --- a/pkg/services/schemaloader/schemaloader.go +++ b/pkg/services/schemaloader/schemaloader.go @@ -8,9 +8,9 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/schema" "github.com/grafana/grafana/pkg/schema/load" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/setting" ) const ServiceName = "SchemaLoader" @@ -26,13 +26,13 @@ type RenderUser struct { OrgRole string } -func ProvideService(cfg *setting.Cfg) (*SchemaLoaderService, error) { +func ProvideService(features featuremgmt.FeatureToggles) (*SchemaLoaderService, error) { dashFam, err := load.BaseDashboardFamily(baseLoadPath) if err != nil { return nil, fmt.Errorf("failed to load dashboard cue schema from path %q: %w", baseLoadPath, err) } s := &SchemaLoaderService{ - Cfg: cfg, + features: features, DashFamily: dashFam, log: log.New("schemaloader"), } @@ -42,14 +42,14 @@ func ProvideService(cfg *setting.Cfg) (*SchemaLoaderService, error) { type SchemaLoaderService struct { log log.Logger DashFamily schema.VersionedCueSchema - Cfg *setting.Cfg + features featuremgmt.FeatureToggles } func (rs *SchemaLoaderService) IsDisabled() bool { - if rs.Cfg == nil { + if rs.features == nil { return true } - return !rs.Cfg.IsTrimDefaultsEnabled() + return !rs.features.IsEnabled(featuremgmt.FlagTrimDefaults) } func (rs *SchemaLoaderService) DashboardApplyDefaults(input *simplejson.Json) (*simplejson.Json, error) { diff --git a/pkg/services/search/service.go b/pkg/services/search/service.go index 6e729353a31..42e0a224ec1 100644 --- a/pkg/services/search/service.go +++ b/pkg/services/search/service.go @@ -19,7 +19,7 @@ func ProvideService(cfg *setting.Cfg, bus bus.Bus) *SearchService { SortAlphaDesc.Name: SortAlphaDesc, }, } - s.Bus.AddHandler(s.searchHandler) + s.Bus.AddHandler(s.SearchHandler) return s } @@ -66,7 +66,7 @@ type SearchService struct { sortOptions map[string]SortOption } -func (s *SearchService) searchHandler(ctx context.Context, query *Query) error { +func (s *SearchService) SearchHandler(ctx context.Context, query *Query) error { dashboardQuery := FindPersistedDashboardsQuery{ Title: query.Title, SignedInUser: query.SignedInUser, diff --git a/pkg/services/search/service_test.go b/pkg/services/search/service_test.go index af84f275216..0f00565e1b1 100644 --- a/pkg/services/search/service_test.go +++ b/pkg/services/search/service_test.go @@ -41,7 +41,7 @@ func TestSearch_SortedResults(t *testing.T) { }, } - err := svc.searchHandler(context.Background(), query) + err := svc.SearchHandler(context.Background(), query) require.Nil(t, err) // Assert results are sorted. diff --git a/pkg/services/searchusers/searchusers.go b/pkg/services/searchusers/searchusers.go index 1cca2ed704e..c6fbfbc9402 100644 --- a/pkg/services/searchusers/searchusers.go +++ b/pkg/services/searchusers/searchusers.go @@ -96,6 +96,8 @@ func GetAuthProviderLabel(authModule string) string { return "SAML" case "ldap", "": return "LDAP" + case "jwt": + return "JWT" default: return "OAuth" } diff --git a/pkg/services/secrets/manager/helpers.go b/pkg/services/secrets/manager/helpers.go index 0419370d60e..958d31f3624 100644 --- a/pkg/services/secrets/manager/helpers.go +++ b/pkg/services/secrets/manager/helpers.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/services/encryption/ossencryption" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/kmsproviders/osskmsproviders" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/setting" @@ -23,10 +24,15 @@ func SetupTestService(tb testing.TB, store secrets.Store) *SecretsService { [security] secret_key = ` + defaultKey)) require.NoError(tb, err) + + features := featuremgmt.WithFeatures(featuremgmt.FlagEnvelopeEncryption) + cfg := &setting.Cfg{Raw: raw} - cfg.FeatureToggles = map[string]bool{secrets.EnvelopeEncryptionFeatureToggle: true} + cfg.IsFeatureToggleEnabled = features.IsEnabled + settings := &setting.OSSImpl{Cfg: cfg} - assert.True(tb, settings.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle)) + assert.True(tb, settings.IsFeatureToggleEnabled(featuremgmt.FlagEnvelopeEncryption)) + assert.True(tb, features.IsEnabled(featuremgmt.FlagEnvelopeEncryption)) encryption := ossencryption.ProvideService() secretsService, err := ProvideSecretsService( diff --git a/pkg/services/secrets/manager/manager.go b/pkg/services/secrets/manager/manager.go index 637f5349e99..c7c461671df 100644 --- a/pkg/services/secrets/manager/manager.go +++ b/pkg/services/secrets/manager/manager.go @@ -12,9 +12,11 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/services/encryption" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/kmsproviders" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/setting" + "golang.org/x/sync/errgroup" "xorm.io/xorm" ) @@ -43,7 +45,7 @@ func ProvideSecretsService( } logger := log.New("secrets") - enabled := settings.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle) + enabled := settings.IsFeatureToggleEnabled(featuremgmt.FlagEnvelopeEncryption) currentProviderID := readCurrentProviderID(settings) if _, ok := providers[currentProviderID]; enabled && !ok { @@ -87,7 +89,7 @@ func (s *SecretsService) registerUsageMetrics() { // Enabled / disabled usageMetrics["stats.encryption.envelope_encryption_enabled.count"] = 0 - if s.settings.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle) { + if s.settings.IsFeatureToggleEnabled(featuremgmt.FlagEnvelopeEncryption) { usageMetrics["stats.encryption.envelope_encryption_enabled.count"] = 1 } @@ -130,11 +132,11 @@ func (s *SecretsService) Encrypt(ctx context.Context, payload []byte, opt secret func (s *SecretsService) EncryptWithDBSession(ctx context.Context, payload []byte, opt secrets.EncryptionOptions, sess *xorm.Session) ([]byte, error) { // Use legacy encryption service if envelopeEncryptionFeatureToggle toggle is off - if !s.settings.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle) { + if !s.settings.IsFeatureToggleEnabled(featuremgmt.FlagEnvelopeEncryption) { return s.enc.Encrypt(ctx, payload, setting.SecretKey) } - // If encryption secrets.EnvelopeEncryptionFeatureToggle toggle is on, use envelope encryption + // If encryption featuremgmt.FlagEnvelopeEncryption toggle is on, use envelope encryption scope := opt() keyName := s.keyName(scope) @@ -172,12 +174,12 @@ func (s *SecretsService) keyName(scope string) string { } func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, error) { - // Use legacy encryption service if secrets.EnvelopeEncryptionFeatureToggle toggle is off - if !s.settings.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle) { + // Use legacy encryption service if featuremgmt.FlagEnvelopeEncryption toggle is off + if !s.settings.IsFeatureToggleEnabled(featuremgmt.FlagEnvelopeEncryption) { return s.enc.Decrypt(ctx, payload, setting.SecretKey) } - // If encryption secrets.EnvelopeEncryptionFeatureToggle toggle is on, use envelope encryption + // If encryption featuremgmt.FlagEnvelopeEncryption toggle is on, use envelope encryption if len(payload) == 0 { return nil, fmt.Errorf("unable to decrypt empty payload") } @@ -358,6 +360,15 @@ var ( func (s *SecretsService) Run(ctx context.Context) error { gc := time.NewTicker(gcInterval) + grp, gCtx := errgroup.WithContext(ctx) + + for _, p := range s.providers { + if svc, ok := p.(secrets.BackgroundProvider); ok { + grp.Go(func() error { + return svc.Run(gCtx) + }) + } + } for { select { @@ -365,9 +376,14 @@ func (s *SecretsService) Run(ctx context.Context) error { s.log.Debug("removing expired data encryption keys from cache...") s.removeExpiredItems() s.log.Debug("done removing expired data encryption keys from cache") - case <-ctx.Done(): + case <-gCtx.Done(): s.log.Debug("grafana is shutting down; stopping...") gc.Stop() + + if err := grp.Wait(); err != nil && !errors.Is(err, context.Canceled) { + return err + } + return nil } } diff --git a/pkg/services/secrets/manager/manager_test.go b/pkg/services/secrets/manager/manager_test.go index df442b49d79..8b0b37ab0f7 100644 --- a/pkg/services/secrets/manager/manager_test.go +++ b/pkg/services/secrets/manager/manager_test.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/services/encryption/ossencryption" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/kmsproviders/osskmsproviders" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets/database" @@ -180,8 +181,8 @@ func TestSecretsService_UseCurrentProvider(t *testing.T) { providerID := secrets.ProviderID("fakeProvider.v1") settings := &setting.OSSImpl{ Cfg: &setting.Cfg{ - Raw: raw, - FeatureToggles: map[string]bool{secrets.EnvelopeEncryptionFeatureToggle: true}, + Raw: raw, + IsFeatureToggleEnabled: featuremgmt.WithFeatures(featuremgmt.FlagEnvelopeEncryption).IsEnabled, }, } encr := ossencryption.ProvideService() diff --git a/pkg/services/secrets/secrets.go b/pkg/services/secrets/secrets.go index 55af27a78ac..d1d3ee3cf68 100644 --- a/pkg/services/secrets/secrets.go +++ b/pkg/services/secrets/secrets.go @@ -8,10 +8,6 @@ import ( "xorm.io/xorm" ) -const ( - EnvelopeEncryptionFeatureToggle = "envelopeEncryption" -) - // Service is an envelope encryption service in charge of encrypting/decrypting secrets. // It is a replacement for encryption.Service type Service interface { @@ -57,3 +53,8 @@ func (id ProviderID) Kind() (string, error) { return parts[0], nil } + +// BackgroundProvider should be implemented for a provider that has a task that needs to be run in the background. +type BackgroundProvider interface { + Run(ctx context.Context) error +} diff --git a/pkg/services/serviceaccounts/api/api.go b/pkg/services/serviceaccounts/api/api.go index 29fd6027d38..985ad0f5637 100644 --- a/pkg/services/serviceaccounts/api/api.go +++ b/pkg/services/serviceaccounts/api/api.go @@ -3,6 +3,9 @@ package api import ( "errors" "net/http" + "strconv" + + "time" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/api/routing" @@ -10,8 +13,8 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" acmiddleware "github.com/grafana/grafana/pkg/services/accesscontrol/middleware" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/serviceaccounts" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" ) @@ -37,17 +40,21 @@ func NewServiceAccountsAPI( } func (api *ServiceAccountsAPI) RegisterAPIEndpoints( - cfg *setting.Cfg, + features featuremgmt.FeatureToggles, ) { - if !cfg.FeatureToggles["service-accounts"] { + if !features.IsEnabled(featuremgmt.FlagServiceAccounts) { return } + auth := acmiddleware.Middleware(api.accesscontrol) - api.RouterRegister.Group("/api/serviceaccounts", func(serviceAccountsRoute routing.RouteRegister) { + api.RouterRegister.Group("/api/org/serviceaccounts", func(serviceAccountsRoute routing.RouteRegister) { serviceAccountsRoute.Get("/", auth(middleware.ReqOrgAdmin, accesscontrol.EvalPermission(serviceaccounts.ActionRead, serviceaccounts.ScopeAll)), routing.Wrap(api.ListServiceAccounts)) + serviceAccountsRoute.Get("/:serviceAccountId", auth(middleware.ReqOrgAdmin, accesscontrol.EvalPermission(serviceaccounts.ActionRead, serviceaccounts.ScopeID)), routing.Wrap(api.RetrieveServiceAccount)) serviceAccountsRoute.Delete("/:serviceAccountId", auth(middleware.ReqOrgAdmin, accesscontrol.EvalPermission(serviceaccounts.ActionDelete, serviceaccounts.ScopeID)), routing.Wrap(api.DeleteServiceAccount)) serviceAccountsRoute.Get("/upgrade", auth(middleware.ReqOrgAdmin, accesscontrol.EvalPermission(serviceaccounts.ActionCreate, serviceaccounts.ScopeID)), routing.Wrap(api.UpgradeServiceAccounts)) + serviceAccountsRoute.Post("/convert/:keyId", auth(middleware.ReqOrgAdmin, accesscontrol.EvalPermission(serviceaccounts.ActionCreate, serviceaccounts.ScopeID)), routing.Wrap(api.ConvertToServiceAccount)) serviceAccountsRoute.Post("/", auth(middleware.ReqOrgAdmin, accesscontrol.EvalPermission(serviceaccounts.ActionCreate, serviceaccounts.ScopeID)), routing.Wrap(api.CreateServiceAccount)) + serviceAccountsRoute.Get("/:serviceAccountId/tokens", auth(middleware.ReqOrgAdmin, accesscontrol.EvalPermission(serviceaccounts.ActionRead, serviceaccounts.ScopeID)), routing.Wrap(api.ListTokens)) }) } @@ -69,8 +76,11 @@ func (api *ServiceAccountsAPI) CreateServiceAccount(c *models.ReqContext) respon } func (api *ServiceAccountsAPI) DeleteServiceAccount(ctx *models.ReqContext) response.Response { - scopeID := ctx.ParamsInt64(":serviceAccountId") - err := api.service.DeleteServiceAccount(ctx.Req.Context(), ctx.OrgId, scopeID) + scopeID, err := strconv.ParseInt(web.Params(ctx.Req)[":serviceAccountId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "serviceAccountId is invalid", err) + } + err = api.service.DeleteServiceAccount(ctx.Req.Context(), ctx.OrgId, scopeID) if err != nil { return response.Error(http.StatusInternalServerError, "Service account deletion error", err) } @@ -85,10 +95,67 @@ func (api *ServiceAccountsAPI) UpgradeServiceAccounts(ctx *models.ReqContext) re } } +func (api *ServiceAccountsAPI) ConvertToServiceAccount(ctx *models.ReqContext) response.Response { + keyId, err := strconv.ParseInt(web.Params(ctx.Req)[":keyId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "keyId is invalid", err) + } + if err := api.store.ConvertToServiceAccounts(ctx.Req.Context(), []int64{keyId}); err == nil { + return response.Success("service accounts converted") + } else { + return response.Error(500, "Internal server error", err) + } +} + +func (api *ServiceAccountsAPI) ListTokens(ctx *models.ReqContext) response.Response { + saID, err := strconv.ParseInt(web.Params(ctx.Req)[":serviceAccountId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "serviceAccountId is invalid", err) + } + if saTokens, err := api.store.ListTokens(ctx.Req.Context(), ctx.OrgId, saID); err == nil { + result := make([]*models.ApiKeyDTO, len(saTokens)) + for i, t := range saTokens { + var expiration *time.Time = nil + if t.Expires != nil { + v := time.Unix(*t.Expires, 0) + expiration = &v + } + result[i] = &models.ApiKeyDTO{ + Id: t.Id, + Name: t.Name, + Role: t.Role, + Expiration: expiration, + } + } + + return response.JSON(200, result) + } else { + return response.Error(500, "Internal server error", err) + } +} + func (api *ServiceAccountsAPI) ListServiceAccounts(ctx *models.ReqContext) response.Response { serviceAccounts, err := api.store.ListServiceAccounts(ctx.Req.Context(), ctx.OrgId) if err != nil { - return response.Error(http.StatusInternalServerError, "Failed to list roles", err) + return response.Error(http.StatusInternalServerError, "Failed to list service accounts", err) } return response.JSON(http.StatusOK, serviceAccounts) } + +func (api *ServiceAccountsAPI) RetrieveServiceAccount(ctx *models.ReqContext) response.Response { + scopeID, err := strconv.ParseInt(web.Params(ctx.Req)[":serviceAccountId"], 10, 64) + if err != nil { + return response.Error(http.StatusBadRequest, "serviceAccountId is invalid", err) + } + + serviceAccount, err := api.store.RetrieveServiceAccount(ctx.Req.Context(), ctx.OrgId, scopeID) + if err != nil { + switch { + case errors.Is(err, serviceaccounts.ErrServiceAccountNotFound): + return response.Error(http.StatusNotFound, "Failed to retrieve service account", err) + default: + return response.Error(http.StatusInternalServerError, "Failed to retrieve service account", err) + } + } + return response.JSON(http.StatusOK, serviceAccount) +} diff --git a/pkg/services/serviceaccounts/api/api_test.go b/pkg/services/serviceaccounts/api/api_test.go index a1b9bb8e7d0..c02c7a815a2 100644 --- a/pkg/services/serviceaccounts/api/api_test.go +++ b/pkg/services/serviceaccounts/api/api_test.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -12,10 +13,10 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/serviceaccounts/tests" "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" "github.com/stretchr/testify/require" @@ -23,7 +24,7 @@ import ( ) var ( - serviceaccountIDPath = "/api/serviceaccounts/%s" + serviceaccountIDPath = "/api/org/serviceaccounts/%v" ) // test the accesscontrol endpoints @@ -56,9 +57,9 @@ func TestServiceAccountsAPI_DeleteServiceAccount(t *testing.T) { ), expectedCode: http.StatusOK, } - serviceAccountDeletionScenario(t, http.MethodDelete, serviceaccountIDPath, &testcase.user, func(httpmethod string, endpoint string, user *tests.TestUser) { + serviceAccountRequestScenario(t, http.MethodDelete, serviceaccountIDPath, &testcase.user, func(httpmethod string, endpoint string, user *tests.TestUser) { createduser := tests.SetupUserServiceAccount(t, store, testcase.user) - server := setupTestServer(t, &svcmock, routing.NewRouteRegister(), testcase.acmock) + server := setupTestServer(t, &svcmock, routing.NewRouteRegister(), testcase.acmock, store) actual := requestResponse(server, httpmethod, fmt.Sprintf(endpoint, fmt.Sprint(createduser.Id))).Code require.Equal(t, testcase.expectedCode, actual) }) @@ -80,24 +81,23 @@ func TestServiceAccountsAPI_DeleteServiceAccount(t *testing.T) { ), expectedCode: http.StatusForbidden, } - serviceAccountDeletionScenario(t, http.MethodDelete, serviceaccountIDPath, &testcase.user, func(httpmethod string, endpoint string, user *tests.TestUser) { + serviceAccountRequestScenario(t, http.MethodDelete, serviceaccountIDPath, &testcase.user, func(httpmethod string, endpoint string, user *tests.TestUser) { createduser := tests.SetupUserServiceAccount(t, store, testcase.user) - server := setupTestServer(t, &svcmock, routing.NewRouteRegister(), testcase.acmock) - actual := requestResponse(server, httpmethod, fmt.Sprintf(endpoint, fmt.Sprint(createduser.Id))).Code + server := setupTestServer(t, &svcmock, routing.NewRouteRegister(), testcase.acmock, store) + actual := requestResponse(server, httpmethod, fmt.Sprintf(endpoint, createduser.Id)).Code require.Equal(t, testcase.expectedCode, actual) }) }) } -func serviceAccountDeletionScenario(t *testing.T, httpMethod string, endpoint string, user *tests.TestUser, fn func(httpmethod string, endpoint string, user *tests.TestUser)) { +func serviceAccountRequestScenario(t *testing.T, httpMethod string, endpoint string, user *tests.TestUser, fn func(httpmethod string, endpoint string, user *tests.TestUser)) { t.Helper() fn(httpMethod, endpoint, user) } -func setupTestServer(t *testing.T, svc *tests.ServiceAccountMock, routerRegister routing.RouteRegister, acmock *accesscontrolmock.Mock) *web.Mux { - store := sqlstore.InitTestDB(t) - a := NewServiceAccountsAPI(svc, acmock, routerRegister, database.NewServiceAccountsStore(store)) - a.RegisterAPIEndpoints(&setting.Cfg{FeatureToggles: map[string]bool{"service-accounts": true}}) +func setupTestServer(t *testing.T, svc *tests.ServiceAccountMock, routerRegister routing.RouteRegister, acmock *accesscontrolmock.Mock, sqlStore *sqlstore.SQLStore) *web.Mux { + a := NewServiceAccountsAPI(svc, acmock, routerRegister, database.NewServiceAccountsStore(sqlStore)) + a.RegisterAPIEndpoints(featuremgmt.WithFeatures(featuremgmt.FlagServiceAccounts)) m := web.New() signedUser := &models.SignedInUser{ @@ -117,3 +117,88 @@ func setupTestServer(t *testing.T, svc *tests.ServiceAccountMock, routerRegister a.RouterRegister.Register(m.Router) return m } + +func TestServiceAccountsAPI_RetrieveServiceAccount(t *testing.T) { + store := sqlstore.InitTestDB(t) + svcmock := tests.ServiceAccountMock{} + type testRetrieveSATestCase struct { + desc string + user *tests.TestUser + expectedCode int + acmock *accesscontrolmock.Mock + userID int + } + testCases := []testRetrieveSATestCase{ + { + desc: "should be ok to retrieve serviceaccount with permissions", + user: &tests.TestUser{Login: "servicetest1@admin", IsServiceAccount: true}, + acmock: tests.SetupMockAccesscontrol( + t, + func(c context.Context, siu *models.SignedInUser) ([]*accesscontrol.Permission, error) { + return []*accesscontrol.Permission{{Action: serviceaccounts.ActionRead, Scope: serviceaccounts.ScopeAll}}, nil + }, + false, + ), + expectedCode: http.StatusOK, + }, + { + desc: "should be forbidden to retrieve serviceaccount if no permissions", + user: &tests.TestUser{Login: "servicetest2@admin", IsServiceAccount: true}, + acmock: tests.SetupMockAccesscontrol( + t, + func(c context.Context, siu *models.SignedInUser) ([]*accesscontrol.Permission, error) { + return []*accesscontrol.Permission{}, nil + }, + false, + ), + expectedCode: http.StatusForbidden, + }, + { + desc: "should be not found when the user doesnt exist", + user: nil, + userID: 12, + acmock: tests.SetupMockAccesscontrol( + t, + func(c context.Context, siu *models.SignedInUser) ([]*accesscontrol.Permission, error) { + return []*accesscontrol.Permission{{Action: serviceaccounts.ActionRead, Scope: serviceaccounts.ScopeAll}}, nil + }, + false, + ), + expectedCode: http.StatusNotFound, + }, + } + + var requestResponse = func(server *web.Mux, httpMethod, requestpath string) *httptest.ResponseRecorder { + req, err := http.NewRequest(httpMethod, requestpath, nil) + require.NoError(t, err) + recorder := httptest.NewRecorder() + server.ServeHTTP(recorder, req) + return recorder + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + serviceAccountRequestScenario(t, http.MethodGet, serviceaccountIDPath, tc.user, func(httpmethod string, endpoint string, user *tests.TestUser) { + scopeID := tc.userID + if tc.user != nil { + createdUser := tests.SetupUserServiceAccount(t, store, *tc.user) + scopeID = int(createdUser.Id) + } + server := setupTestServer(t, &svcmock, routing.NewRouteRegister(), tc.acmock, store) + + actual := requestResponse(server, httpmethod, fmt.Sprintf(endpoint, scopeID)) + + actualCode := actual.Code + require.Equal(t, tc.expectedCode, actualCode) + + if actualCode == http.StatusOK { + actualBody := map[string]interface{}{} + err := json.Unmarshal(actual.Body.Bytes(), &actualBody) + require.NoError(t, err) + require.Equal(t, scopeID, int(actualBody["userId"].(float64))) + require.Equal(t, tc.user.Login, actualBody["login"].(string)) + } + }) + }) + } +} diff --git a/pkg/services/serviceaccounts/database/database.go b/pkg/services/serviceaccounts/database/database.go index bbd362e47d4..3cdd1cc1f09 100644 --- a/pkg/services/serviceaccounts/database/database.go +++ b/pkg/services/serviceaccounts/database/database.go @@ -1,14 +1,17 @@ package database +//nolint:goimports import ( "context" "fmt" + "time" "github.com/google/uuid" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/sqlstore" + "xorm.io/xorm" ) type ServiceAccountsStoreImpl struct { @@ -67,24 +70,66 @@ func (s *ServiceAccountsStoreImpl) UpgradeServiceAccounts(ctx context.Context) e s.log.Info("Launching background thread to upgrade API keys to service accounts", "numberKeys", len(basicKeys)) go func() { for _, key := range basicKeys { - sa, err := s.sqlStore.CreateServiceAccountForApikey(ctx, key.OrgId, key.Name, key.Role) + err := s.CreateServiceAccountFromApikey(ctx, key) if err != nil { - s.log.Error("Failed to create service account for API key", "err", err, "keyId", key.Id) - continue + s.log.Error("migating to service accounts failed with error", err) } - - err = s.sqlStore.UpdateApikeyServiceAccount(ctx, key.Id, sa.Id) - if err != nil { - s.log.Error("Failed to attach new service account to API key", "err", err, "keyId", key.Id, "newServiceAccountId", sa.Id) - continue - } - s.log.Debug("Updated basic api key", "keyId", key.Id, "newServiceAccountId", sa.Id) } }() } return nil } +func (s *ServiceAccountsStoreImpl) ConvertToServiceAccounts(ctx context.Context, keys []int64) error { + basicKeys := s.sqlStore.GetNonServiceAccountAPIKeys(ctx) + if len(basicKeys) == 0 { + return nil + } + if len(basicKeys) != len(keys) { + return fmt.Errorf("one of the keys already has a serviceaccount") + } + for _, key := range basicKeys { + if !contains(keys, key.Id) { + s.log.Error("convert service accounts stopped for keyId %d as it is not part of the query to convert or already has a service account", key.Id) + continue + } + err := s.CreateServiceAccountFromApikey(ctx, key) + if err != nil { + s.log.Error("converting to service accounts failed with error", err) + } + } + return nil +} + +func (s *ServiceAccountsStoreImpl) CreateServiceAccountFromApikey(ctx context.Context, key *models.ApiKey) error { + sa, err := s.sqlStore.CreateServiceAccountForApikey(ctx, key.OrgId, key.Name, key.Role) + if err != nil { + return fmt.Errorf("failed to create service account for API key with error : %w", err) + } + + err = s.sqlStore.UpdateApikeyServiceAccount(ctx, key.Id, sa.Id) + if err != nil { + return fmt.Errorf("failed to attach new service account to API key for keyId: %d and newServiceAccountId: %d with error: %w", key.Id, sa.Id, err) + } + s.log.Debug("Updated basic api key", "keyId", key.Id, "newServiceAccountId", sa.Id) + return nil +} + +//nolint:gosimple +func (s *ServiceAccountsStoreImpl) ListTokens(ctx context.Context, orgID int64, serviceAccount int64) ([]*models.ApiKey, error) { + result := make([]*models.ApiKey, 0) + err := s.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + var sess *xorm.Session + + sess = dbSession.Limit(100, 0). + Join("inner", "user", "user.id = api_key.service_account_id"). + Where("user.org_id=? AND user.id=? AND ( expires IS NULL or expires >= ?)", orgID, serviceAccount, time.Now().Unix()). + Asc("name") + + return sess.Find(&result) + }) + return result, err +} func (s *ServiceAccountsStoreImpl) ListServiceAccounts(ctx context.Context, orgID int64) ([]*models.OrgUserDTO, error) { query := models.GetOrgUsersQuery{OrgId: orgID, IsServiceAccount: true} err := s.sqlStore.GetOrgUsers(ctx, &query) @@ -93,3 +138,27 @@ func (s *ServiceAccountsStoreImpl) ListServiceAccounts(ctx context.Context, orgI } return query.Result, err } + +// RetrieveServiceAccountByID returns a service account by its ID +func (s *ServiceAccountsStoreImpl) RetrieveServiceAccount(ctx context.Context, orgID, serviceAccountID int64) (*models.OrgUserDTO, error) { + query := models.GetOrgUsersQuery{UserID: serviceAccountID, OrgId: orgID, IsServiceAccount: true} + err := s.sqlStore.GetOrgUsers(ctx, &query) + if err != nil { + return nil, err + } + + if len(query.Result) != 1 { + return nil, serviceaccounts.ErrServiceAccountNotFound + } + + return query.Result[0], err +} + +func contains(s []int64, e int64) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} diff --git a/pkg/services/serviceaccounts/database/database_test.go b/pkg/services/serviceaccounts/database/database_test.go index a26dada9a13..799cc1ac8ce 100644 --- a/pkg/services/serviceaccounts/database/database_test.go +++ b/pkg/services/serviceaccounts/database/database_test.go @@ -47,3 +47,36 @@ func setupTestDatabase(t *testing.T) (*sqlstore.SQLStore, *ServiceAccountsStoreI db := sqlstore.InitTestDB(t) return db, NewServiceAccountsStore(db) } + +func TestStore_RetrieveServiceAccount(t *testing.T) { + cases := []struct { + desc string + user tests.TestUser + expectedErr error + }{ + { + desc: "service accounts should exist and get retrieved", + user: tests.TestUser{Login: "servicetest1@admin", IsServiceAccount: true}, + expectedErr: nil, + }, + { + desc: "service accounts is false should not retrieve user", + user: tests.TestUser{Login: "test1@admin", IsServiceAccount: false}, + expectedErr: serviceaccounts.ErrServiceAccountNotFound, + }, + } + + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + db, store := setupTestDatabase(t) + user := tests.SetupUserServiceAccount(t, db, c.user) + dto, err := store.RetrieveServiceAccount(context.Background(), user.OrgId, user.Id) + if c.expectedErr != nil { + require.ErrorIs(t, err, c.expectedErr) + } else { + require.NoError(t, err) + require.Equal(t, c.user.Login, dto.Login) + } + }) + } +} diff --git a/pkg/services/serviceaccounts/manager/service.go b/pkg/services/serviceaccounts/manager/service.go index 8f7a440d012..a48576097cf 100644 --- a/pkg/services/serviceaccounts/manager/service.go +++ b/pkg/services/serviceaccounts/manager/service.go @@ -7,11 +7,11 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/serviceaccounts/api" "github.com/grafana/grafana/pkg/services/serviceaccounts/database" "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/setting" ) var ( @@ -19,21 +19,21 @@ var ( ) type ServiceAccountsService struct { - store serviceaccounts.Store - cfg *setting.Cfg - log log.Logger + store serviceaccounts.Store + features featuremgmt.FeatureToggles + log log.Logger } func ProvideServiceAccountsService( - cfg *setting.Cfg, + features featuremgmt.FeatureToggles, store *sqlstore.SQLStore, ac accesscontrol.AccessControl, routeRegister routing.RouteRegister, ) (*ServiceAccountsService, error) { s := &ServiceAccountsService{ - cfg: cfg, - store: database.NewServiceAccountsStore(store), - log: log.New("serviceaccounts"), + features: features, + store: database.NewServiceAccountsStore(store), + log: log.New("serviceaccounts"), } if err := RegisterRoles(ac); err != nil { @@ -41,13 +41,13 @@ func ProvideServiceAccountsService( } serviceaccountsAPI := api.NewServiceAccountsAPI(s, ac, routeRegister, s.store) - serviceaccountsAPI.RegisterAPIEndpoints(cfg) + serviceaccountsAPI.RegisterAPIEndpoints(features) return s, nil } func (sa *ServiceAccountsService) CreateServiceAccount(ctx context.Context, saForm *serviceaccounts.CreateServiceaccountForm) (*models.User, error) { - if !sa.cfg.FeatureToggles["service-accounts"] { + if !sa.features.IsEnabled(featuremgmt.FlagServiceAccounts) { sa.log.Debug(ServiceAccountFeatureToggleNotFound) return nil, nil } @@ -55,7 +55,7 @@ func (sa *ServiceAccountsService) CreateServiceAccount(ctx context.Context, saFo } func (sa *ServiceAccountsService) DeleteServiceAccount(ctx context.Context, orgID, serviceAccountID int64) error { - if !sa.cfg.FeatureToggles["service-accounts"] { + if !sa.features.IsEnabled(featuremgmt.FlagServiceAccounts) { sa.log.Debug(ServiceAccountFeatureToggleNotFound) return nil } diff --git a/pkg/services/serviceaccounts/manager/service_test.go b/pkg/services/serviceaccounts/manager/service_test.go index 460e5d61e1d..57f986a3fe9 100644 --- a/pkg/services/serviceaccounts/manager/service_test.go +++ b/pkg/services/serviceaccounts/manager/service_test.go @@ -5,31 +5,29 @@ import ( "testing" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/serviceaccounts/tests" - "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestProvideServiceAccount_DeleteServiceAccount(t *testing.T) { t.Run("feature toggle present, should call store function", func(t *testing.T) { - cfg := setting.NewCfg() storeMock := &tests.ServiceAccountsStoreMock{Calls: tests.Calls{}} - cfg.FeatureToggles = map[string]bool{"service-accounts": true} - svc := ServiceAccountsService{cfg: cfg, store: storeMock} + svc := ServiceAccountsService{ + features: featuremgmt.WithFeatures("service-accounts", true), + store: storeMock} err := svc.DeleteServiceAccount(context.Background(), 1, 1) require.NoError(t, err) assert.Len(t, storeMock.Calls.DeleteServiceAccount, 1) }) t.Run("no feature toggle present, should not call store function", func(t *testing.T) { - cfg := setting.NewCfg() svcMock := &tests.ServiceAccountsStoreMock{Calls: tests.Calls{}} - cfg.FeatureToggles = map[string]bool{"service-accounts": false} svc := ServiceAccountsService{ - cfg: cfg, - store: svcMock, - log: log.New("serviceaccounts-manager-test"), + features: featuremgmt.WithFeatures("service-accounts", false), + store: svcMock, + log: log.New("serviceaccounts-manager-test"), } err := svc.DeleteServiceAccount(context.Background(), 1, 1) require.NoError(t, err) diff --git a/pkg/services/serviceaccounts/serviceaccounts.go b/pkg/services/serviceaccounts/serviceaccounts.go index 7ad36dc7a4a..a5718ff9043 100644 --- a/pkg/services/serviceaccounts/serviceaccounts.go +++ b/pkg/services/serviceaccounts/serviceaccounts.go @@ -15,6 +15,9 @@ type Service interface { type Store interface { CreateServiceAccount(ctx context.Context, saForm *CreateServiceaccountForm) (*models.User, error) ListServiceAccounts(ctx context.Context, orgID int64) ([]*models.OrgUserDTO, error) + RetrieveServiceAccount(ctx context.Context, orgID, serviceAccountID int64) (*models.OrgUserDTO, error) DeleteServiceAccount(ctx context.Context, orgID, serviceAccountID int64) error UpgradeServiceAccounts(ctx context.Context) error + ConvertToServiceAccounts(ctx context.Context, keys []int64) error + ListTokens(ctx context.Context, orgID int64, serviceAccount int64) ([]*models.ApiKey, error) } diff --git a/pkg/services/serviceaccounts/tests/common.go b/pkg/services/serviceaccounts/tests/common.go index c8fc4f56dd3..4451c265ba6 100644 --- a/pkg/services/serviceaccounts/tests/common.go +++ b/pkg/services/serviceaccounts/tests/common.go @@ -58,8 +58,11 @@ var _ serviceaccounts.Store = new(ServiceAccountsStoreMock) type Calls struct { CreateServiceAccount []interface{} ListServiceAccounts []interface{} + RetrieveServiceAccount []interface{} DeleteServiceAccount []interface{} UpgradeServiceAccounts []interface{} + ConvertServiceAccounts []interface{} + ListTokens []interface{} } type ServiceAccountsStoreMock struct { @@ -79,11 +82,25 @@ func (s *ServiceAccountsStoreMock) DeleteServiceAccount(ctx context.Context, org } func (s *ServiceAccountsStoreMock) UpgradeServiceAccounts(ctx context.Context) error { - s.Calls.DeleteServiceAccount = append(s.Calls.UpgradeServiceAccounts, []interface{}{ctx}) + s.Calls.UpgradeServiceAccounts = append(s.Calls.UpgradeServiceAccounts, []interface{}{ctx}) return nil } +func (s *ServiceAccountsStoreMock) ConvertToServiceAccounts(ctx context.Context, keys []int64) error { + s.Calls.ConvertServiceAccounts = append(s.Calls.ConvertServiceAccounts, []interface{}{ctx}) + return nil +} + +func (s *ServiceAccountsStoreMock) ListTokens(ctx context.Context, orgID int64, serviceAccount int64) ([]*models.ApiKey, error) { + s.Calls.ListTokens = append(s.Calls.ListTokens, []interface{}{ctx, orgID, serviceAccount}) + return nil, nil +} func (s *ServiceAccountsStoreMock) ListServiceAccounts(ctx context.Context, orgID int64) ([]*models.OrgUserDTO, error) { s.Calls.ListServiceAccounts = append(s.Calls.ListServiceAccounts, []interface{}{ctx, orgID}) return nil, nil } + +func (s *ServiceAccountsStoreMock) RetrieveServiceAccount(ctx context.Context, orgID, serviceAccountID int64) (*models.OrgUserDTO, error) { + s.Calls.RetrieveServiceAccount = append(s.Calls.RetrieveServiceAccount, []interface{}{ctx, orgID, serviceAccountID}) + return nil, nil +} diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index b884a888767..98902413698 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -15,14 +15,13 @@ import ( var timeNow = time.Now func (ss *SQLStore) addAlertQueryAndCommandHandlers() { - bus.AddHandler("sql", SaveAlerts) bus.AddHandler("sql", ss.HandleAlertsQuery) bus.AddHandler("sql", ss.GetAlertById) bus.AddHandler("sql", ss.GetAllAlertQueryHandler) bus.AddHandler("sql", ss.SetAlertState) bus.AddHandler("sql", ss.GetAlertStatesForDashboard) - bus.AddHandler("sql", PauseAlert) - bus.AddHandler("sql", PauseAllAlerts) + bus.AddHandler("sql", ss.PauseAlert) + bus.AddHandler("sql", ss.PauseAllAlerts) } func (ss *SQLStore) GetAlertById(ctx context.Context, query *models.GetAlertByIdQuery) error { @@ -174,7 +173,7 @@ func deleteAlertDefinition(dashboardId int64, sess *DBSession) error { } func (ss *SQLStore) SaveAlerts(ctx context.Context, dashID int64, alerts []*models.Alert) error { - return ss.WithTransactionalDbSession(context.Background(), func(sess *DBSession) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { existingAlerts, err := GetAlertsByDashboardId2(dashID, sess) if err != nil { return err @@ -192,25 +191,6 @@ func (ss *SQLStore) SaveAlerts(ctx context.Context, dashID int64, alerts []*mode }) } -func SaveAlerts(ctx context.Context, cmd *models.SaveAlertsCommand) error { - return inTransaction(func(sess *DBSession) error { - existingAlerts, err := GetAlertsByDashboardId2(cmd.DashboardId, sess) - if err != nil { - return err - } - - if err := updateAlerts(existingAlerts, cmd.Alerts, sess); err != nil { - return err - } - - if err := deleteMissingAlerts(existingAlerts, cmd.Alerts, sess); err != nil { - return err - } - - return nil - }) -} - func updateAlerts(existingAlerts []*models.Alert, alerts []*models.Alert, sess *DBSession) error { for _, alert := range alerts { update := false @@ -344,8 +324,8 @@ func (ss *SQLStore) SetAlertState(ctx context.Context, cmd *models.SetAlertState }) } -func PauseAlert(ctx context.Context, cmd *models.PauseAlertCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) PauseAlert(ctx context.Context, cmd *models.PauseAlertCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { if len(cmd.AlertIds) == 0 { return fmt.Errorf("command contains no alertids") } @@ -378,8 +358,8 @@ func PauseAlert(ctx context.Context, cmd *models.PauseAlertCommand) error { }) } -func PauseAllAlerts(ctx context.Context, cmd *models.PauseAllAlertCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) PauseAllAlerts(ctx context.Context, cmd *models.PauseAllAlertCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { var newState string if cmd.Paused { newState = string(models.AlertStatePaused) diff --git a/pkg/services/sqlstore/alert_test.go b/pkg/services/sqlstore/alert_test.go index 8fece7a975d..9430fa63819 100644 --- a/pkg/services/sqlstore/alert_test.go +++ b/pkg/services/sqlstore/alert_test.go @@ -34,7 +34,6 @@ func TestAlertingDataAccess(t *testing.T) { var sqlStore *SQLStore var testDash *models.Dashboard - var cmd models.SaveAlertsCommand var items []*models.Alert setup := func(t *testing.T) { @@ -56,14 +55,7 @@ func TestAlertingDataAccess(t *testing.T) { }, } - cmd = models.SaveAlertsCommand{ - Alerts: items, - DashboardId: testDash.Id, - OrgId: 1, - UserId: 1, - } - - err = SaveAlerts(context.Background(), &cmd) + err = sqlStore.SaveAlerts(context.Background(), testDash.Id, items) require.Nil(t, err) } @@ -91,7 +83,7 @@ func TestAlertingDataAccess(t *testing.T) { stateDateBeforePause := alert.NewStateDate t.Run("can pause all alerts", func(t *testing.T) { - err := pauseAllAlerts(t, true) + err := sqlStore.pauseAllAlerts(t, true) require.Nil(t, err) t.Run("cannot updated paused alert", func(t *testing.T) { @@ -117,7 +109,7 @@ func TestAlertingDataAccess(t *testing.T) { }) t.Run("unpausing alerts should update their NewStateDate again", func(t *testing.T) { - err := pauseAllAlerts(t, false) + err := sqlStore.pauseAllAlerts(t, false) require.Nil(t, err) alert, _ = getAlertById(t, insertedAlert.Id, sqlStore) stateDateAfterUnpause := alert.NewStateDate @@ -162,14 +154,7 @@ func TestAlertingDataAccess(t *testing.T) { modifiedItems := items modifiedItems[0].Name = "Name" - modifiedCmd := models.SaveAlertsCommand{ - DashboardId: testDash.Id, - OrgId: 1, - UserId: 1, - Alerts: modifiedItems, - } - - err := SaveAlerts(context.Background(), &modifiedCmd) + err := sqlStore.SaveAlerts(context.Background(), testDash.Id, items) t.Run("Can save alerts with same dashboard and panel id", func(t *testing.T) { require.Nil(t, err) @@ -189,7 +174,7 @@ func TestAlertingDataAccess(t *testing.T) { }) t.Run("Updates without changes should be ignored", func(t *testing.T) { - err3 := SaveAlerts(context.Background(), &modifiedCmd) + err3 := sqlStore.SaveAlerts(context.Background(), testDash.Id, items) require.Nil(t, err3) }) }) @@ -220,8 +205,7 @@ func TestAlertingDataAccess(t *testing.T) { }, } - cmd.Alerts = multipleItems - err := SaveAlerts(context.Background(), &cmd) + err := sqlStore.SaveAlerts(context.Background(), testDash.Id, multipleItems) t.Run("Should save 3 dashboards", func(t *testing.T) { require.Nil(t, err) @@ -236,8 +220,7 @@ func TestAlertingDataAccess(t *testing.T) { t.Run("should updated two dashboards and delete one", func(t *testing.T) { missingOneAlert := multipleItems[:2] - cmd.Alerts = missingOneAlert - err = SaveAlerts(context.Background(), &cmd) + err = sqlStore.SaveAlerts(context.Background(), testDash.Id, missingOneAlert) t.Run("should delete the missing alert", func(t *testing.T) { query := models.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &models.SignedInUser{OrgRole: models.ROLE_ADMIN}} @@ -259,17 +242,10 @@ func TestAlertingDataAccess(t *testing.T) { }, } - cmd := models.SaveAlertsCommand{ - Alerts: items, - DashboardId: testDash.Id, - OrgId: 1, - UserId: 1, - } - - err := SaveAlerts(context.Background(), &cmd) + err := sqlStore.SaveAlerts(context.Background(), testDash.Id, items) require.Nil(t, err) - err = DeleteDashboard(context.Background(), &models.DeleteDashboardCommand{ + err = sqlStore.DeleteDashboard(context.Background(), &models.DeleteDashboardCommand{ OrgId: 1, Id: testDash.Id, }) @@ -293,7 +269,7 @@ func TestPausingAlerts(t *testing.T) { sqlStore := InitTestDB(t) testDash := insertTestDashboard(t, sqlStore, "dashboard with alerts", 1, 0, false, "alert") - alert, err := insertTestAlert("Alerting title", "Alerting message", testDash.OrgId, testDash.Id, simplejson.New()) + alert, err := insertTestAlert("Alerting title", "Alerting message", testDash.OrgId, testDash.Id, simplejson.New(), sqlStore) require.Nil(t, err) stateDateBeforePause := alert.NewStateDate @@ -307,7 +283,7 @@ func TestPausingAlerts(t *testing.T) { insertedAlert := alertQuery.Result[0] t.Run("when paused", func(t *testing.T) { - _, err := pauseAlert(t, testDash.OrgId, insertedAlert.Id, true) + _, err := sqlStore.pauseAlert(t, testDash.OrgId, insertedAlert.Id, true) require.Nil(t, err) t.Run("the NewStateDate should be updated", func(t *testing.T) { @@ -320,7 +296,7 @@ func TestPausingAlerts(t *testing.T) { }) t.Run("when unpaused", func(t *testing.T) { - _, err := pauseAlert(t, testDash.OrgId, insertedAlert.Id, false) + _, err := sqlStore.pauseAlert(t, testDash.OrgId, insertedAlert.Id, false) require.Nil(t, err) t.Run("the NewStateDate should be updated again", func(t *testing.T) { @@ -333,17 +309,17 @@ func TestPausingAlerts(t *testing.T) { }) }) } -func pauseAlert(t *testing.T, orgId int64, alertId int64, pauseState bool) (int64, error) { +func (ss *SQLStore) pauseAlert(t *testing.T, orgId int64, alertId int64, pauseState bool) (int64, error) { cmd := &models.PauseAlertCommand{ OrgId: orgId, AlertIds: []int64{alertId}, Paused: pauseState, } - err := PauseAlert(context.Background(), cmd) + err := ss.PauseAlert(context.Background(), cmd) require.Nil(t, err) return cmd.ResultCount, err } -func insertTestAlert(title string, message string, orgId int64, dashId int64, settings *simplejson.Json) (*models.Alert, error) { +func insertTestAlert(title string, message string, orgId int64, dashId int64, settings *simplejson.Json, ss *SQLStore) (*models.Alert, error) { items := []*models.Alert{ { PanelId: 1, @@ -356,15 +332,8 @@ func insertTestAlert(title string, message string, orgId int64, dashId int64, se }, } - cmd := models.SaveAlertsCommand{ - Alerts: items, - DashboardId: dashId, - OrgId: orgId, - UserId: 1, - } - - err := SaveAlerts(context.Background(), &cmd) - return cmd.Alerts[0], err + err := ss.SaveAlerts(context.Background(), dashId, items) + return items[0], err } func getAlertById(t *testing.T, id int64, ss *SQLStore) (*models.Alert, error) { @@ -376,11 +345,11 @@ func getAlertById(t *testing.T, id int64, ss *SQLStore) (*models.Alert, error) { return q.Result, err } -func pauseAllAlerts(t *testing.T, pauseState bool) error { +func (ss *SQLStore) pauseAllAlerts(t *testing.T, pauseState bool) error { cmd := &models.PauseAllAlertCommand{ Paused: pauseState, } - err := PauseAllAlerts(context.Background(), cmd) + err := ss.PauseAllAlerts(context.Background(), cmd) require.Nil(t, err) return err } diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index 46c362e8dab..ba2f812a22f 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -28,8 +28,6 @@ var shadowSearchCounter = prometheus.NewCounterVec( func init() { bus.AddHandler("sql", GetDashboard) - bus.AddHandler("sql", GetDashboards) - bus.AddHandler("sql", DeleteDashboard) bus.AddHandler("sql", GetDashboardTags) bus.AddHandler("sql", GetDashboardSlugById) bus.AddHandler("sql", GetDashboardsByPluginId) @@ -44,6 +42,8 @@ func init() { func (ss *SQLStore) addDashboardQueryAndCommandHandlers() { bus.AddHandler("sql", ss.GetDashboardUIDById) bus.AddHandler("sql", ss.SearchDashboards) + bus.AddHandler("sql", ss.DeleteDashboard) + bus.AddHandler("sql", ss.GetDashboards) } var generateNewUid func() string = util.GenerateShortUID @@ -410,8 +410,8 @@ func GetDashboardTags(ctx context.Context, query *models.GetDashboardTagsQuery) return err } -func DeleteDashboard(ctx context.Context, cmd *models.DeleteDashboardCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) DeleteDashboard(ctx context.Context, cmd *models.DeleteDashboardCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { return deleteDashboard(cmd, sess) }) } @@ -509,7 +509,7 @@ func deleteDashboard(cmd *models.DeleteDashboardCommand, sess *DBSession) error return nil } -func GetDashboards(ctx context.Context, query *models.GetDashboardsQuery) error { +func (ss *SQLStore) GetDashboards(ctx context.Context, query *models.GetDashboardsQuery) error { if len(query.DashboardIds) == 0 { return models.ErrCommandValidationFailed } diff --git a/pkg/services/sqlstore/dashboard_provisioning.go b/pkg/services/sqlstore/dashboard_provisioning.go index 048fd76476d..053f5ad74c8 100644 --- a/pkg/services/sqlstore/dashboard_provisioning.go +++ b/pkg/services/sqlstore/dashboard_provisioning.go @@ -8,9 +8,9 @@ import ( "github.com/grafana/grafana/pkg/models" ) -func init() { +func (ss *SQLStore) addDashboardProvisioningQueryAndCommandHandlers() { bus.AddHandler("sql", UnprovisionDashboard) - bus.AddHandler("sql", DeleteOrphanedProvisionedDashboards) + bus.AddHandler("sql", ss.DeleteOrphanedProvisionedDashboards) } type DashboardExtras struct { @@ -111,7 +111,7 @@ func UnprovisionDashboard(ctx context.Context, cmd *models.UnprovisionDashboardC return nil } -func DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *models.DeleteOrphanedProvisionedDashboardsCommand) error { +func (ss *SQLStore) DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *models.DeleteOrphanedProvisionedDashboardsCommand) error { var result []*models.DashboardProvisioning convertedReaderNames := make([]interface{}, len(cmd.ReaderNames)) @@ -125,7 +125,7 @@ func DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *models.Delete } for _, deleteDashCommand := range result { - err := DeleteDashboard(ctx, &models.DeleteDashboardCommand{Id: deleteDashCommand.DashboardId}) + err := ss.DeleteDashboard(ctx, &models.DeleteDashboardCommand{Id: deleteDashCommand.DashboardId}) if err != nil && !errors.Is(err, models.ErrDashboardNotFound) { return err } diff --git a/pkg/services/sqlstore/dashboard_provisioning_test.go b/pkg/services/sqlstore/dashboard_provisioning_test.go index b58b62e573b..f74800d8851 100644 --- a/pkg/services/sqlstore/dashboard_provisioning_test.go +++ b/pkg/services/sqlstore/dashboard_provisioning_test.go @@ -75,15 +75,15 @@ func TestDashboardProvisioningTest(t *testing.T) { require.Nil(t, err) query := &models.GetDashboardsQuery{DashboardIds: []int64{anotherDash.Id}} - err = GetDashboards(context.Background(), query) + err = sqlStore.GetDashboards(context.Background(), query) require.Nil(t, err) require.NotNil(t, query.Result) deleteCmd := &models.DeleteOrphanedProvisionedDashboardsCommand{ReaderNames: []string{"default"}} - require.Nil(t, DeleteOrphanedProvisionedDashboards(context.Background(), deleteCmd)) + require.Nil(t, sqlStore.DeleteOrphanedProvisionedDashboards(context.Background(), deleteCmd)) query = &models.GetDashboardsQuery{DashboardIds: []int64{dash.Id, anotherDash.Id}} - err = GetDashboards(context.Background(), query) + err = sqlStore.GetDashboards(context.Background(), query) require.Nil(t, err) require.Equal(t, 1, len(query.Result)) @@ -117,7 +117,7 @@ func TestDashboardProvisioningTest(t *testing.T) { OrgId: 1, } - require.Nil(t, DeleteDashboard(context.Background(), deleteCmd)) + require.Nil(t, sqlStore.DeleteDashboard(context.Background(), deleteCmd)) data, err := sqlStore.GetProvisionedDataByDashboardID(dash.Id) require.Nil(t, err) diff --git a/pkg/services/sqlstore/dashboard_test.go b/pkg/services/sqlstore/dashboard_test.go index 5f24b14b8a9..96106172075 100644 --- a/pkg/services/sqlstore/dashboard_test.go +++ b/pkg/services/sqlstore/dashboard_test.go @@ -8,14 +8,15 @@ import ( "encoding/json" "errors" "fmt" + "testing" + "time" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/search" "github.com/grafana/grafana/pkg/services/sqlstore/searchstore" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" - "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -116,7 +117,7 @@ func TestDashboardDataAccess(t *testing.T) { setup() dash := insertTestDashboard(t, sqlStore, "delete me", 1, 0, false, "delete this") - err := DeleteDashboard(context.Background(), &models.DeleteDashboardCommand{ + err := sqlStore.DeleteDashboard(context.Background(), &models.DeleteDashboardCommand{ Id: dash.Id, OrgId: 1, }) @@ -213,21 +214,21 @@ func TestDashboardDataAccess(t *testing.T) { emptyFolder := insertTestDashboard(t, sqlStore, "2 test dash folder", 1, 0, true, "prod", "webapp") deleteCmd := &models.DeleteDashboardCommand{Id: emptyFolder.Id} - err := DeleteDashboard(context.Background(), deleteCmd) + err := sqlStore.DeleteDashboard(context.Background(), deleteCmd) require.NoError(t, err) }) t.Run("Should be not able to delete a dashboard if force delete rules is disabled", func(t *testing.T) { setup() deleteCmd := &models.DeleteDashboardCommand{Id: savedFolder.Id, ForceDeleteFolderRules: false} - err := DeleteDashboard(context.Background(), deleteCmd) + err := sqlStore.DeleteDashboard(context.Background(), deleteCmd) require.True(t, errors.Is(err, models.ErrFolderContainsAlertRules)) }) t.Run("Should be able to delete a dashboard folder and its children if force delete rules is enabled", func(t *testing.T) { setup() deleteCmd := &models.DeleteDashboardCommand{Id: savedFolder.Id, ForceDeleteFolderRules: true} - err := DeleteDashboard(context.Background(), deleteCmd) + err := sqlStore.DeleteDashboard(context.Background(), deleteCmd) require.NoError(t, err) query := search.FindPersistedDashboardsQuery{ @@ -589,7 +590,7 @@ func createUser(t *testing.T, sqlStore *SQLStore, name string, role string, isAd currentUser, err := sqlStore.CreateUser(context.Background(), currentUserCmd) require.NoError(t, err) q1 := models.GetUserOrgListQuery{UserId: currentUser.Id} - err = GetUserOrgList(context.Background(), &q1) + err = sqlStore.GetUserOrgList(context.Background(), &q1) require.NoError(t, err) require.Equal(t, models.RoleType(role), q1.Result[0].Role) return *currentUser diff --git a/pkg/services/sqlstore/database_wrapper.go b/pkg/services/sqlstore/database_wrapper.go index 136c3f161be..81079412cd3 100644 --- a/pkg/services/sqlstore/database_wrapper.go +++ b/pkg/services/sqlstore/database_wrapper.go @@ -11,11 +11,10 @@ import ( "github.com/gchaincl/sqlhooks" "github.com/go-sql-driver/mysql" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" "github.com/lib/pq" "github.com/mattn/go-sqlite3" - "github.com/opentracing/opentracing-go" - ol "github.com/opentracing/opentracing-go/log" "github.com/prometheus/client_golang/prometheus" cw "github.com/weaveworks/common/tracing" "xorm.io/core" @@ -39,7 +38,7 @@ func init() { // WrapDatabaseDriverWithHooks creates a fake database driver that // executes pre and post functions which we use to gather metrics about // database queries. It also registers the metrics. -func WrapDatabaseDriverWithHooks(dbType string) string { +func WrapDatabaseDriverWithHooks(dbType string, tracer tracing.Tracer) string { drivers := map[string]driver.Driver{ migrator.SQLite: &sqlite3.SQLiteDriver{}, migrator.MySQL: &mysql.MySQLDriver{}, @@ -52,7 +51,7 @@ func WrapDatabaseDriverWithHooks(dbType string) string { } driverWithHooks := dbType + "WithHooks" - sql.Register(driverWithHooks, sqlhooks.Wrap(d, &databaseQueryWrapper{log: log.New("sqlstore.metrics")})) + sql.Register(driverWithHooks, sqlhooks.Wrap(d, &databaseQueryWrapper{log: log.New("sqlstore.metrics"), tracer: tracer})) core.RegisterDriver(driverWithHooks, &databaseQueryWrapperDriver{dbType: dbType}) return driverWithHooks } @@ -60,7 +59,8 @@ func WrapDatabaseDriverWithHooks(dbType string) string { // databaseQueryWrapper satisfies the sqlhook.databaseQueryWrapper interface // which allow us to wrap all SQL queries with a `Before` & `After` hook. type databaseQueryWrapper struct { - log log.Logger + log log.Logger + tracer tracing.Tracer } // databaseQueryWrapperKey is used as key to save values in `context.Context` @@ -94,15 +94,13 @@ func (h *databaseQueryWrapper) instrument(ctx context.Context, status string, qu histogram.Observe(elapsed.Seconds()) } - span, _ := opentracing.StartSpanFromContext(ctx, "database query") - defer span.Finish() + _, span := h.tracer.Start(ctx, "database query") + defer span.End() - span.LogFields( - ol.String("query", query), - ol.String("status", status)) + span.AddEvents([]string{"query", "status"}, []tracing.EventValue{{Str: query}, {Str: status}}) if err != nil { - span.LogFields(ol.String("error", err.Error())) + span.AddEvents([]string{"error"}, []tracing.EventValue{{Str: err.Error()}}) } h.log.Debug("query finished", "status", status, "elapsed time", elapsed, "sql", query, "error", err) diff --git a/pkg/services/sqlstore/login_attempt.go b/pkg/services/sqlstore/login_attempt.go index 58d0ddf38bf..cf2b78dac2e 100644 --- a/pkg/services/sqlstore/login_attempt.go +++ b/pkg/services/sqlstore/login_attempt.go @@ -11,14 +11,14 @@ import ( var getTimeNow = time.Now -func init() { - bus.AddHandler("sql", CreateLoginAttempt) - bus.AddHandler("sql", DeleteOldLoginAttempts) +func (ss *SQLStore) addLoginAttemptQueryAndCommandHandlers() { + bus.AddHandler("sql", ss.CreateLoginAttempt) + bus.AddHandler("sql", ss.DeleteOldLoginAttempts) bus.AddHandler("sql", GetUserLoginAttemptCount) } -func CreateLoginAttempt(ctx context.Context, cmd *models.CreateLoginAttemptCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) CreateLoginAttempt(ctx context.Context, cmd *models.CreateLoginAttemptCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { loginAttempt := models.LoginAttempt{ Username: cmd.Username, IpAddress: cmd.IpAddress, @@ -35,8 +35,8 @@ func CreateLoginAttempt(ctx context.Context, cmd *models.CreateLoginAttemptComma }) } -func DeleteOldLoginAttempts(ctx context.Context, cmd *models.DeleteOldLoginAttemptsCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) DeleteOldLoginAttempts(ctx context.Context, cmd *models.DeleteOldLoginAttemptsCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { var maxId int64 sql := "SELECT max(id) as id FROM login_attempt WHERE created < ?" result, err := sess.Query(sql, cmd.OlderThan.Unix()) diff --git a/pkg/services/sqlstore/login_attempt_test.go b/pkg/services/sqlstore/login_attempt_test.go index dbe16866a38..5d8f0a4bd1f 100644 --- a/pkg/services/sqlstore/login_attempt_test.go +++ b/pkg/services/sqlstore/login_attempt_test.go @@ -20,24 +20,25 @@ func mockTime(mock time.Time) time.Time { func TestLoginAttempts(t *testing.T) { var beginningOfTime, timePlusOneMinute, timePlusTwoMinutes time.Time + var sqlStore *SQLStore user := "user" setup := func(t *testing.T) { - InitTestDB(t) + sqlStore = InitTestDB(t) beginningOfTime = mockTime(time.Date(2017, 10, 22, 8, 0, 0, 0, time.Local)) - err := CreateLoginAttempt(context.Background(), &models.CreateLoginAttemptCommand{ + err := sqlStore.CreateLoginAttempt(context.Background(), &models.CreateLoginAttemptCommand{ Username: user, IpAddress: "192.168.0.1", }) require.Nil(t, err) timePlusOneMinute = mockTime(beginningOfTime.Add(time.Minute * 1)) - err = CreateLoginAttempt(context.Background(), &models.CreateLoginAttemptCommand{ + err = sqlStore.CreateLoginAttempt(context.Background(), &models.CreateLoginAttemptCommand{ Username: user, IpAddress: "192.168.0.1", }) require.Nil(t, err) timePlusTwoMinutes = mockTime(beginningOfTime.Add(time.Minute * 2)) - err = CreateLoginAttempt(context.Background(), &models.CreateLoginAttemptCommand{ + err = sqlStore.CreateLoginAttempt(context.Background(), &models.CreateLoginAttemptCommand{ Username: user, IpAddress: "192.168.0.1", }) @@ -93,7 +94,7 @@ func TestLoginAttempts(t *testing.T) { cmd := models.DeleteOldLoginAttemptsCommand{ OlderThan: beginningOfTime, } - err := DeleteOldLoginAttempts(context.Background(), &cmd) + err := sqlStore.DeleteOldLoginAttempts(context.Background(), &cmd) require.Nil(t, err) require.Equal(t, int64(0), cmd.DeletedRows) @@ -104,7 +105,7 @@ func TestLoginAttempts(t *testing.T) { cmd := models.DeleteOldLoginAttemptsCommand{ OlderThan: timePlusOneMinute, } - err := DeleteOldLoginAttempts(context.Background(), &cmd) + err := sqlStore.DeleteOldLoginAttempts(context.Background(), &cmd) require.Nil(t, err) require.Equal(t, int64(1), cmd.DeletedRows) @@ -115,7 +116,7 @@ func TestLoginAttempts(t *testing.T) { cmd := models.DeleteOldLoginAttemptsCommand{ OlderThan: timePlusTwoMinutes, } - err := DeleteOldLoginAttempts(context.Background(), &cmd) + err := sqlStore.DeleteOldLoginAttempts(context.Background(), &cmd) require.Nil(t, err) require.Equal(t, int64(2), cmd.DeletedRows) @@ -126,7 +127,7 @@ func TestLoginAttempts(t *testing.T) { cmd := models.DeleteOldLoginAttemptsCommand{ OlderThan: timePlusTwoMinutes.Add(time.Second * 1), } - err := DeleteOldLoginAttempts(context.Background(), &cmd) + err := sqlStore.DeleteOldLoginAttempts(context.Background(), &cmd) require.Nil(t, err) require.Equal(t, int64(3), cmd.DeletedRows) diff --git a/pkg/services/sqlstore/migrations/accesscontrol/errors.go b/pkg/services/sqlstore/migrations/accesscontrol/errors.go new file mode 100644 index 00000000000..fd1116f2e62 --- /dev/null +++ b/pkg/services/sqlstore/migrations/accesscontrol/errors.go @@ -0,0 +1,17 @@ +package accesscontrol + +import "fmt" + +var ErrAddTeamMembershipMigrations = fmt.Errorf("Error migrating team memberships") + +type ErrUnknownRole struct { + key string +} + +func (e *ErrUnknownRole) Error() string { + return fmt.Sprintf("%v: Unable to find role in map: %s", ErrAddTeamMembershipMigrations, e.key) +} + +func (e *ErrUnknownRole) Unwrap() error { + return ErrAddTeamMembershipMigrations +} diff --git a/pkg/services/sqlstore/migrations/accesscontrol/team_membership.go b/pkg/services/sqlstore/migrations/accesscontrol/team_membership.go new file mode 100644 index 00000000000..8e1cd545ce1 --- /dev/null +++ b/pkg/services/sqlstore/migrations/accesscontrol/team_membership.go @@ -0,0 +1,416 @@ +package accesscontrol + +import ( + "fmt" + "strconv" + "strings" + "time" + + "xorm.io/xorm" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/util" +) + +const ( + TeamsMigrationID = "teams permissions migration" + batchSize = 500 +) + +func AddTeamMembershipMigrations(mg *migrator.Migrator) { + mg.AddMigration(TeamsMigrationID, &teamPermissionMigrator{editorsCanAdmin: mg.Cfg.EditorsCanAdmin}) +} + +var _ migrator.CodeMigration = new(teamPermissionMigrator) + +type teamPermissionMigrator struct { + migrator.MigrationBase + editorsCanAdmin bool + sess *xorm.Session + dialect migrator.Dialect +} + +func (p *teamPermissionMigrator) getAssignmentKey(orgID int64, name string) string { + return fmt.Sprint(orgID, "-", name) +} + +func (p *teamPermissionMigrator) SQL(dialect migrator.Dialect) string { + return "code migration" +} + +func (p *teamPermissionMigrator) Exec(sess *xorm.Session, migrator *migrator.Migrator) error { + p.sess = sess + p.dialect = migrator.Dialect + return p.migrateMemberships() +} + +func generateNewRoleUID(sess *xorm.Session, orgID int64) (string, error) { + for i := 0; i < 3; i++ { + uid := util.GenerateShortUID() + + exists, err := sess.Where("org_id=? AND uid=?", orgID, uid).Get(&accesscontrol.Role{}) + if err != nil { + return "", err + } + + if !exists { + return uid, nil + } + } + + return "", fmt.Errorf("failed to generate uid") +} + +func (p *teamPermissionMigrator) findRole(orgID int64, name string) (accesscontrol.Role, error) { + // check if role exists + var role accesscontrol.Role + _, err := p.sess.Table("role").Where("org_id = ? AND name = ?", orgID, name).Get(&role) + return role, err +} + +func batch(count, batchSize int, eachFn func(start, end int) error) error { + for i := 0; i < count; { + end := i + batchSize + if end > count { + end = count + } + + if err := eachFn(i, end); err != nil { + return err + } + + i = end + } + + return nil +} + +func (p *teamPermissionMigrator) bulkCreateRoles(allRoles []*accesscontrol.Role) ([]*accesscontrol.Role, error) { + if len(allRoles) == 0 { + return nil, nil + } + + allCreatedRoles := make([]*accesscontrol.Role, 0, len(allRoles)) + + createRoles := p.createRoles + if p.dialect.DriverName() == migrator.MySQL { + createRoles = p.createRolesMySQL + } + + // bulk role creations + err := batch(len(allRoles), batchSize, func(start, end int) error { + roles := allRoles[start:end] + createdRoles, err := createRoles(roles, start, end) + if err != nil { + return err + } + allCreatedRoles = append(allCreatedRoles, createdRoles...) + return nil + }) + + return allCreatedRoles, err +} + +// createRoles creates a list of roles and returns their id, orgID, name in a single query +func (p *teamPermissionMigrator) createRoles(roles []*accesscontrol.Role, start int, end int) ([]*accesscontrol.Role, error) { + ts := time.Now() + createdRoles := make([]*accesscontrol.Role, 0, len(roles)) + valueStrings := make([]string, len(roles)) + args := make([]interface{}, 0, len(roles)*5) + + for i, r := range roles { + uid, err := generateNewRoleUID(p.sess, r.OrgID) + if err != nil { + return nil, err + } + + valueStrings[i] = "(?, ?, ?, 1, ?, ?)" + args = append(args, r.OrgID, uid, r.Name, ts, ts) + } + + // Insert and fetch at once + valueString := strings.Join(valueStrings, ",") + sql := fmt.Sprintf("INSERT INTO role (org_id, uid, name, version, created, updated) VALUES %s RETURNING id, org_id, name", valueString) + if errCreate := p.sess.SQL(sql, args...).Find(&createdRoles); errCreate != nil { + return nil, errCreate + } + + return createdRoles, nil +} + +// createRolesMySQL creates a list of roles then fetches them +func (p *teamPermissionMigrator) createRolesMySQL(roles []*accesscontrol.Role, start int, end int) ([]*accesscontrol.Role, error) { + ts := time.Now() + createdRoles := make([]*accesscontrol.Role, 0, len(roles)) + + where := make([]string, len(roles)) + args := make([]interface{}, 0, len(roles)*2) + + for i := range roles { + uid, err := generateNewRoleUID(p.sess, roles[i].OrgID) + if err != nil { + return nil, err + } + + roles[i].UID = uid + roles[i].Created = ts + roles[i].Updated = ts + + where[i] = ("(org_id = ? AND uid = ?)") + args = append(args, roles[i].OrgID, uid) + } + + // Insert roles + if _, errCreate := p.sess.Table("role").Insert(&roles); errCreate != nil { + return nil, errCreate + } + + // Fetch newly created roles + if errFindInsertions := p.sess.Table("role"). + Where(strings.Join(where, " OR "), args...). + Find(&createdRoles); errFindInsertions != nil { + return nil, errFindInsertions + } + + return createdRoles, nil +} + +func (p *teamPermissionMigrator) bulkAssignRoles(rolesMap map[string]*accesscontrol.Role, assignments map[int64]map[string]struct{}) error { + if len(assignments) == 0 { + return nil + } + + ts := time.Now() + + roleAssignments := make([]accesscontrol.UserRole, 0, len(assignments)) + for userID, rolesByRoleKey := range assignments { + for key := range rolesByRoleKey { + role, ok := rolesMap[key] + if !ok { + return &ErrUnknownRole{key} + } + + roleAssignments = append(roleAssignments, accesscontrol.UserRole{ + OrgID: role.OrgID, + RoleID: role.ID, + UserID: userID, + Created: ts, + }) + } + } + + return batch(len(roleAssignments), batchSize, func(start, end int) error { + roleAssignmentsChunk := roleAssignments[start:end] + _, err := p.sess.Table("user_role").InsertMulti(roleAssignmentsChunk) + return err + }) +} + +// setRolePermissions sets the role permissions deleting any team related ones before inserting any. +func (p *teamPermissionMigrator) setRolePermissions(roleID int64, permissions []accesscontrol.Permission) error { + // First drop existing permissions + if _, errDeletingPerms := p.sess.Exec("DELETE FROM permission WHERE role_id = ? AND (action LIKE ? OR action LIKE ?)", roleID, "teams:%", "teams.permissions:%"); errDeletingPerms != nil { + return errDeletingPerms + } + + // Then insert new permissions + var newPermissions []accesscontrol.Permission + now := time.Now() + for _, permission := range permissions { + permission.RoleID = roleID + permission.Created = now + permission.Updated = now + newPermissions = append(newPermissions, permission) + } + + if _, errInsertPerms := p.sess.InsertMulti(&newPermissions); errInsertPerms != nil { + return errInsertPerms + } + + return nil +} + +// mapPermissionToFGAC translates the legacy membership (Member or Admin) into FGAC permissions +func (p *teamPermissionMigrator) mapPermissionToFGAC(permission models.PermissionType, teamID int64) []accesscontrol.Permission { + teamIDScope := accesscontrol.Scope("teams", "id", strconv.FormatInt(teamID, 10)) + switch permission { + case 0: + return []accesscontrol.Permission{{Action: "teams:read", Scope: teamIDScope}} + case models.PERMISSION_ADMIN: + return []accesscontrol.Permission{ + {Action: "teams:delete", Scope: teamIDScope}, + {Action: "teams:read", Scope: teamIDScope}, + {Action: "teams:write", Scope: teamIDScope}, + {Action: "teams.permissions:read", Scope: teamIDScope}, + {Action: "teams.permissions:write", Scope: teamIDScope}, + } + default: + return []accesscontrol.Permission{} + } +} + +func (p *teamPermissionMigrator) getUserRoleByOrgMapping() (map[int64]map[int64]string, error) { + var orgUsers []*models.OrgUserDTO + if err := p.sess.SQL(`SELECT * FROM org_user`).Cols("org_user.org_id", "org_user.user_id", "org_user.role").Find(&orgUsers); err != nil { + return nil, err + } + + userRolesByOrg := map[int64]map[int64]string{} + + // Loop through users and organise them by organization ID + for _, orgUser := range orgUsers { + orgRoles, initialized := userRolesByOrg[orgUser.OrgId] + if !initialized { + orgRoles = map[int64]string{} + } + + orgRoles[orgUser.UserId] = orgUser.Role + userRolesByOrg[orgUser.OrgId] = orgRoles + } + + return userRolesByOrg, nil +} + +// migrateMemberships generate managed permissions for users based on their memberships to teams +func (p *teamPermissionMigrator) migrateMemberships() error { + // Fetch user roles in each org + userRolesByOrg, err := p.getUserRoleByOrgMapping() + if err != nil { + return err + } + + // Fetch team memberships + teamMemberships := []*models.TeamMember{} + if err := p.sess.SQL("SELECT * FROM team_member").Find(&teamMemberships); err != nil { + return err + } + + // No need to create any roles if there is no team members + if len(teamMemberships) == 0 { + return nil + } + + // Loop through memberships and generate associated permissions + // Downgrade team permissions if needed - only admins or editors (when editorsCanAdmin option is enabled) + // can access team administration endpoints + userPermissionsByOrg, errGen := p.generateAssociatedPermissions(teamMemberships, userRolesByOrg) + if errGen != nil { + return errGen + } + + // Sort roles that: + // * need to be created and assigned (rolesToCreate, assignments) + // * are already created and assigned (rolesByOrg) + rolesToCreate, assignments, rolesByOrg, errOrganizeRoles := p.sortRolesToAssign(userPermissionsByOrg) + if errOrganizeRoles != nil { + return errOrganizeRoles + } + + // Create missing roles + createdRoles, errCreate := p.bulkCreateRoles(rolesToCreate) + if errCreate != nil { + return errCreate + } + + // Populate rolesMap with the newly created roles + for i := range createdRoles { + roleKey := p.getAssignmentKey(createdRoles[i].OrgID, createdRoles[i].Name) + rolesByOrg[roleKey] = createdRoles[i] + } + + // Assign newly created roles + if errAssign := p.bulkAssignRoles(rolesByOrg, assignments); errAssign != nil { + return errAssign + } + + // Set all roles teams related permissions + return p.setRolePermissionsForOrgs(userPermissionsByOrg, rolesByOrg) +} + +func (p *teamPermissionMigrator) setRolePermissionsForOrgs(userPermissionsByOrg map[int64]map[int64][]accesscontrol.Permission, rolesByOrg map[string]*accesscontrol.Role) error { + for orgID, userPermissions := range userPermissionsByOrg { + for userID, permissions := range userPermissions { + key := p.getAssignmentKey(orgID, fmt.Sprintf("managed:users:%d:permissions", userID)) + + role, ok := rolesByOrg[key] + if !ok { + return &ErrUnknownRole{key} + } + + if errSettingPerms := p.setRolePermissions(role.ID, permissions); errSettingPerms != nil { + return errSettingPerms + } + } + } + return nil +} + +func (p *teamPermissionMigrator) sortRolesToAssign(userPermissionsByOrg map[int64]map[int64][]accesscontrol.Permission) ([]*accesscontrol.Role, map[int64]map[string]struct{}, map[string]*accesscontrol.Role, error) { + var rolesToCreate []*accesscontrol.Role + + assignments := map[int64]map[string]struct{}{} + + rolesByOrg := map[string]*accesscontrol.Role{} + for orgID, userPermissions := range userPermissionsByOrg { + for userID := range userPermissions { + roleName := fmt.Sprintf("managed:users:%d:permissions", userID) + role, errFindingRoles := p.findRole(orgID, roleName) + if errFindingRoles != nil { + return nil, nil, nil, errFindingRoles + } + + roleKey := p.getAssignmentKey(orgID, roleName) + + if role.ID != 0 { + rolesByOrg[roleKey] = &role + } else { + roleToCreate := &accesscontrol.Role{ + Name: roleName, + OrgID: orgID, + } + rolesToCreate = append(rolesToCreate, roleToCreate) + + userAssignments, initialized := assignments[userID] + if !initialized { + userAssignments = map[string]struct{}{} + } + + userAssignments[roleKey] = struct{}{} + assignments[userID] = userAssignments + } + } + } + + return rolesToCreate, assignments, rolesByOrg, nil +} + +func (p *teamPermissionMigrator) generateAssociatedPermissions(teamMemberships []*models.TeamMember, + userRolesByOrg map[int64]map[int64]string) (map[int64]map[int64][]accesscontrol.Permission, error) { + userPermissionsByOrg := map[int64]map[int64][]accesscontrol.Permission{} + + for _, m := range teamMemberships { + // Downgrade team permissions if needed: + // only admins or editors (when editorsCanAdmin option is enabled) + // can access team administration endpoints + if m.Permission == models.PERMISSION_ADMIN { + if userRolesByOrg[m.OrgId][m.UserId] == string(models.ROLE_VIEWER) || (userRolesByOrg[m.OrgId][m.UserId] == string(models.ROLE_EDITOR) && !p.editorsCanAdmin) { + m.Permission = 0 + + if _, err := p.sess.Cols("permission").Where("org_id=? and team_id=? and user_id=?", m.OrgId, m.TeamId, m.UserId).Update(m); err != nil { + return nil, err + } + } + } + + userPermissions, initialized := userPermissionsByOrg[m.OrgId] + if !initialized { + userPermissions = map[int64][]accesscontrol.Permission{} + } + userPermissions[m.UserId] = append(userPermissions[m.UserId], p.mapPermissionToFGAC(m.Permission, m.TeamId)...) + userPermissionsByOrg[m.OrgId] = userPermissions + } + + return userPermissionsByOrg, nil +} diff --git a/pkg/services/sqlstore/migrations/accesscontrol/test/ac_test.go b/pkg/services/sqlstore/migrations/accesscontrol/test/ac_test.go new file mode 100644 index 00000000000..b79ba4c3d61 --- /dev/null +++ b/pkg/services/sqlstore/migrations/accesscontrol/test/ac_test.go @@ -0,0 +1,413 @@ +package test + +import ( + "fmt" + "testing" + "time" + + "xorm.io/xorm" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/sqlstore/migrations" + acmig "github.com/grafana/grafana/pkg/services/sqlstore/migrations/accesscontrol" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type rawPermission struct { + Action, Scope string +} + +// Setup users +var ( + now = time.Now() + + users = []models.User{ + { + Id: 1, + Email: "viewer1@example.org", + Name: "viewer1", + Login: "viewer1", + OrgId: 1, + Created: now, + Updated: now, + }, + { + Id: 2, + Email: "viewer2@example.org", + Name: "viewer2", + Login: "viewer2", + OrgId: 1, + Created: now, + Updated: now, + }, + { + Id: 3, + Email: "editor1@example.org", + Name: "editor1", + Login: "editor1", + OrgId: 1, + Created: now, + Updated: now, + }, + { + Id: 4, + Email: "admin1@example.org", + Name: "admin1", + Login: "admin1", + OrgId: 1, + Created: now, + Updated: now, + }, + { + Id: 5, + Email: "editor2@example.org", + Name: "editor2", + Login: "editor2", + OrgId: 2, + Created: now, + Updated: now, + }, + } +) + +func convertToRawPermissions(permissions []accesscontrol.Permission) []rawPermission { + raw := make([]rawPermission, len(permissions)) + for i, p := range permissions { + raw[i] = rawPermission{Action: p.Action, Scope: p.Scope} + } + return raw +} + +func TestMigrations(t *testing.T) { + // Run initial migration to have a working DB + x := setupTestDB(t) + + // Populate users and teams + setupTeams(t, x) + + // Create managed user roles with teams permissions (ex: teams:read and teams.permissions:read) + setupUnecessaryFGACPermissions(t, x) + + team1Scope := accesscontrol.Scope("teams", "id", "1") + team2Scope := accesscontrol.Scope("teams", "id", "2") + + type teamMigrationTestCase struct { + desc string + config *setting.Cfg + expectedRolePerms map[string][]rawPermission + } + testCases := []teamMigrationTestCase{ + { + desc: "with editors can admin", + config: &setting.Cfg{ + EditorsCanAdmin: true, + IsFeatureToggleEnabled: func(key string) bool { return key == "accesscontrol" }, + }, + expectedRolePerms: map[string][]rawPermission{ + "managed:users:1:permissions": {{Action: "teams:read", Scope: team1Scope}}, + "managed:users:2:permissions": {{Action: "teams:read", Scope: team1Scope}}, + "managed:users:3:permissions": { + {Action: "teams:read", Scope: team1Scope}, + {Action: "teams:delete", Scope: team1Scope}, + {Action: "teams:write", Scope: team1Scope}, + {Action: "teams.permissions:read", Scope: team1Scope}, + {Action: "teams.permissions:write", Scope: team1Scope}, + }, + "managed:users:4:permissions": { + {Action: "teams:read", Scope: team1Scope}, + {Action: "teams:delete", Scope: team1Scope}, + {Action: "teams:write", Scope: team1Scope}, + {Action: "teams.permissions:read", Scope: team1Scope}, + {Action: "teams.permissions:write", Scope: team1Scope}, + }, + "managed:users:5:permissions": { + {Action: "teams:read", Scope: team2Scope}, + {Action: "users:read", Scope: "users:*"}, + }, + }, + }, + { + desc: "without editors can admin", + config: &setting.Cfg{ + IsFeatureToggleEnabled: func(key string) bool { return key == "accesscontrol" }, + }, + expectedRolePerms: map[string][]rawPermission{ + "managed:users:1:permissions": {{Action: "teams:read", Scope: team1Scope}}, + "managed:users:2:permissions": {{Action: "teams:read", Scope: team1Scope}}, + "managed:users:3:permissions": {{Action: "teams:read", Scope: team1Scope}}, + "managed:users:4:permissions": { + {Action: "teams:read", Scope: team1Scope}, + {Action: "teams:delete", Scope: team1Scope}, + {Action: "teams:write", Scope: team1Scope}, + {Action: "teams.permissions:read", Scope: team1Scope}, + {Action: "teams.permissions:write", Scope: team1Scope}, + }, + "managed:users:5:permissions": { + {Action: "teams:read", Scope: team2Scope}, + {Action: "users:read", Scope: "users:*"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + // Remove migration + _, errDeleteMig := x.Exec("DELETE FROM migration_log WHERE migration_id = ?", acmig.TeamsMigrationID) + require.NoError(t, errDeleteMig) + + // Run accesscontrol migration (permissions insertion should not have conflicted) + acmigrator := migrator.NewMigrator(x, tc.config) + acmig.AddTeamMembershipMigrations(acmigrator) + + errRunningMig := acmigrator.Start() + require.NoError(t, errRunningMig) + + for _, user := range users { + // Check managed roles exist + roleName := fmt.Sprintf("managed:users:%d:permissions", user.Id) + role := accesscontrol.Role{} + hasRole, errManagedRoleSearch := x.Table("role").Where("org_id = ? AND name = ?", user.OrgId, roleName).Get(&role) + + require.NoError(t, errManagedRoleSearch) + assert.True(t, hasRole, "expected role to be granted to user", user, roleName) + + // Check permissions associated with each role + perms := []accesscontrol.Permission{} + countUserPermissions, errManagedPermsSearch := x.Table("permission").Where("role_id = ?", role.ID).FindAndCount(&perms) + + require.NoError(t, errManagedPermsSearch) + assert.Equal(t, int64(len(tc.expectedRolePerms[roleName])), countUserPermissions, "expected role to be tied to permissions", user, role) + + rawPerms := convertToRawPermissions(perms) + for _, perm := range rawPerms { + assert.Contains(t, tc.expectedRolePerms[roleName], perm) + } + + // Check assignment of the roles + assign := accesscontrol.UserRole{} + has, errAssignmentSearch := x.Table("user_role").Where("role_id = ? AND user_id = ?", role.ID, user.Id).Get(&assign) + require.NoError(t, errAssignmentSearch) + assert.True(t, has, "expected assignment of role to user", role, user) + } + }) + } +} + +func setupTestDB(t *testing.T) *xorm.Engine { + t.Helper() + testDB := sqlutil.SQLite3TestDB() + + const query = `select count(*) as count from migration_log` + result := struct{ Count int }{} + + x, err := xorm.NewEngine(testDB.DriverName, testDB.ConnStr) + require.NoError(t, err) + + err = migrator.NewDialect(x).CleanDB() + require.NoError(t, err) + + _, err = x.SQL(query).Get(&result) + require.Error(t, err) + + mg := migrator.NewMigrator(x, &setting.Cfg{ + IsFeatureToggleEnabled: func(key string) bool { return key == "accesscontrol" }, + }) + migrations := &migrations.OSSMigrations{} + migrations.AddMigration(mg) + + err = mg.Start() + require.NoError(t, err) + + return x +} + +func setupTeams(t *testing.T, x *xorm.Engine) { + t.Helper() + + usersCount, errInsertUsers := x.Insert(users) + require.NoError(t, errInsertUsers) + require.Equal(t, int64(5), usersCount, "needed 5 users for this test to run") + + orgUsers := []models.OrgUser{ + { + OrgId: 1, + UserId: 1, + Role: models.ROLE_VIEWER, + Created: now, + Updated: now, + }, + { + OrgId: 1, + UserId: 2, + Role: models.ROLE_VIEWER, + Created: now, + Updated: now, + }, + { + OrgId: 1, + UserId: 3, + Role: models.ROLE_EDITOR, + Created: now, + Updated: now, + }, + { + OrgId: 1, + UserId: 4, + Role: models.ROLE_ADMIN, + Created: now, + Updated: now, + }, + { + OrgId: 2, + UserId: 5, + Role: models.ROLE_EDITOR, + Created: now, + Updated: now, + }, + } + orgUsersCount, errInsertOrgUsers := x.Insert(orgUsers) + require.NoError(t, errInsertOrgUsers) + require.Equal(t, int64(5), orgUsersCount, "needed 5 users for this test to run") + + // Setup teams (and members) + teams := []models.Team{ + { + OrgId: 1, + Name: "teamOrg1", + Email: "teamorg1@example.org", + Created: now, + Updated: now, + }, + { + OrgId: 2, + Name: "teamOrg2", + Email: "teamorg2@example.org", + Created: now, + Updated: now, + }, + } + teamCount, errInsertTeams := x.Insert(teams) + require.NoError(t, errInsertTeams) + require.Equal(t, int64(2), teamCount, "needed 2 teams for this test to run") + + members := []models.TeamMember{ + { + // Can have viewer permissions + OrgId: 1, + TeamId: 1, + UserId: 1, + External: false, + Permission: 0, + Created: now, + Updated: now, + }, + { + // Cannot have admin permissions + OrgId: 1, + TeamId: 1, + UserId: 2, + External: false, + Permission: models.PERMISSION_ADMIN, + Created: now, + Updated: now, + }, + { + // Can have admin permissions + OrgId: 1, + TeamId: 1, + UserId: 3, + External: false, + Permission: models.PERMISSION_ADMIN, + Created: now, + Updated: now, + }, + { + // Can have admin permissions + OrgId: 1, + TeamId: 1, + UserId: 4, + External: false, + Permission: models.PERMISSION_ADMIN, + Created: now, + Updated: now, + }, + { + // Can have viewer permissions + OrgId: 2, + TeamId: 2, + UserId: 5, + External: false, + Permission: 0, + Created: now, + Updated: now, + }, + } + membersCount, err := x.Insert(members) + require.NoError(t, err) + require.Equal(t, int64(5), membersCount, "needed 5 members for this test to run") +} + +func setupUnecessaryFGACPermissions(t *testing.T, x *xorm.Engine) { + t.Helper() + + now := time.Now() + + role := accesscontrol.Role{ + // ID: 1, Not specifying this for pgsql to correctly increment sequence + OrgID: 2, + Version: 1, + UID: "user5managedpermissions", + Name: "managed:users:5:permissions", + Updated: now, + Created: now, + } + rolesCount, err := x.Insert(role) + require.NoError(t, err) + require.Equal(t, int64(1), rolesCount, "needed 1 role for this test to run") + + userRole := accesscontrol.UserRole{ + OrgID: 2, + RoleID: 1, + UserID: 5, + Created: now, + } + userRoleCount, err := x.Insert(userRole) + require.NoError(t, err) + require.Equal(t, int64(1), userRoleCount, "needed 1 assignment for this test to run") + + permissions := []accesscontrol.Permission{ + { + // Permission that shouldn't be removed + RoleID: 1, + Action: "users:read", + Scope: "users:*", + Updated: now, + Created: now, + }, + { + // Permission that should be recreated + RoleID: 1, + Action: "teams:read", + Scope: "teams:*", + Updated: now, + Created: now, + }, + { + // Permission that should be removed + RoleID: 1, + Action: "teams.permissions:read", + Scope: "teams:*", + Updated: now, + Created: now, + }, + } + permissionsCount, err := x.Insert(permissions) + require.NoError(t, err) + require.Equal(t, int64(3), permissionsCount, "needed 3 permissions for this test to run") +} diff --git a/pkg/services/sqlstore/migrations/accesscontrol/util_test.go b/pkg/services/sqlstore/migrations/accesscontrol/util_test.go new file mode 100644 index 00000000000..9f555ab1745 --- /dev/null +++ b/pkg/services/sqlstore/migrations/accesscontrol/util_test.go @@ -0,0 +1,42 @@ +package accesscontrol + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBatch(t *testing.T) { + type testCase struct { + desc string + batchSize int + count int + } + + testCases := []testCase{ + {desc: "empty", batchSize: 1, count: 0}, + {desc: "1 run of 5", batchSize: 5, count: 5}, + {desc: "10 runs of 5", batchSize: 5, count: 50}, + {desc: "unmatching end", batchSize: 10, count: 25}, + {desc: "batch bigger than count", batchSize: 500, count: 25}, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + items := make([]int, tc.count) + sum := 0 + + got := batch(len(items), tc.batchSize, func(start int, end int) error { + chunk := items[start:end] + for range chunk { + sum += 1 + } + + return nil + }) + + require.NoError(t, got) + require.Equal(t, tc.count, sum) + }) + } +} diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index cb1d9b1d269..0b766b0a470 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -3,6 +3,7 @@ package migrations import ( "os" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore/migrations/accesscontrol" "github.com/grafana/grafana/pkg/services/sqlstore/migrations/ualert" . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" @@ -56,8 +57,8 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { ualert.AddTablesMigrations(mg) ualert.AddDashAlertMigration(mg) addLibraryElementsMigrations(mg) - if mg.Cfg != nil { - if mg.Cfg.IsLiveConfigEnabled() { + if mg.Cfg != nil && mg.Cfg.IsFeatureToggleEnabled != nil { + if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagLiveConfig) { addLiveChannelMigrations(mg) } } @@ -67,6 +68,13 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { addKVStoreMigrations(mg) ualert.AddDashboardUIDPanelIDMigration(mg) accesscontrol.AddMigration(mg) + addQueryHistoryMigrations(mg) + + if mg.Cfg != nil && mg.Cfg.IsFeatureToggleEnabled != nil { + if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagAccesscontrol) { + accesscontrol.AddTeamMembershipMigrations(mg) + } + } } func addMigrationLogMigrations(mg *Migrator) { diff --git a/pkg/services/sqlstore/migrations/query_history_mig.go b/pkg/services/sqlstore/migrations/query_history_mig.go new file mode 100644 index 00000000000..9bc7097c06e --- /dev/null +++ b/pkg/services/sqlstore/migrations/query_history_mig.go @@ -0,0 +1,28 @@ +package migrations + +import ( + . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +) + +func addQueryHistoryMigrations(mg *Migrator) { + queryHistoryV1 := Table{ + Name: "query_history", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, Nullable: false, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: false}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "datasource_uid", Type: DB_NVarchar, Length: 40, Nullable: false}, + {Name: "created_by", Type: DB_Int, Nullable: false}, + {Name: "created_at", Type: DB_Int, Nullable: false}, + {Name: "comment", Type: DB_Text, Nullable: false}, + {Name: "queries", Type: DB_Text, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "created_by", "datasource_uid"}}, + }, + } + + mg.AddMigration("create query_history table v1", NewAddTableMigration(queryHistoryV1)) + + mg.AddMigration("add index query_history.org_id-created_by-datasource_uid", NewAddIndexMigration(queryHistoryV1, queryHistoryV1.Indices[0])) +} diff --git a/pkg/services/sqlstore/migrations/ualert/alert_rule.go b/pkg/services/sqlstore/migrations/ualert/alert_rule.go index c0e28048d73..fcc53586392 100644 --- a/pkg/services/sqlstore/migrations/ualert/alert_rule.go +++ b/pkg/services/sqlstore/migrations/ualert/alert_rule.go @@ -89,10 +89,6 @@ func addMigrationInfo(da *dashAlert) (map[string]string, map[string]string) { return lbls, annotations } -func getMigrationString(da dashAlert) string { - return fmt.Sprintf(`{"dashboardUid": "%v", "panelId": %v, "alertId": %v}`, da.DashboardUID, da.PanelId, da.Id) -} - func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string) (*alertRule, error) { lbls, annotations := addMigrationInfo(&da) lbls["alertname"] = da.Name diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go index 4114f84b480..6244a0f0fa0 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert.go @@ -22,7 +22,10 @@ import ( ) const GENERAL_FOLDER = "General Alerting" -const DASHBOARD_FOLDER = "Migrated %s" +const DASHBOARD_FOLDER = "%s Alerts - %s" + +// MaxFolderName is the maximum length of the folder name generated using DASHBOARD_FOLDER format +const MaxFolderName = 255 // FOLDER_CREATED_BY us used to track folders created by this migration // during alert migration cleanup. @@ -231,6 +234,7 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { if err != nil { return err } + mg.Logger.Info("alerts found to migrate", "alerts", len(dashAlerts)) // [orgID, dataSourceId] -> UID dsIDMap, err := m.slurpDSIDs() @@ -256,6 +260,9 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { return err } + // cache for folders created for dashboards that have custom permissions + folderCache := make(map[string]*dashboard) + for _, da := range dashAlerts { newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap) if err != nil { @@ -280,55 +287,65 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { } } - // get folder if exists - folder, err := m.getFolder(dash, da) - if err != nil { - return MigrationError{ - Err: err, - AlertId: da.Id, - } - } - + var folder *dashboard switch { case dash.HasAcl: - // create folder and assign the permissions of the dashboard (included default and inherited) - ptr, err := m.createFolder(dash.OrgId, fmt.Sprintf(DASHBOARD_FOLDER, getMigrationString(da))) - if err != nil { - return MigrationError{ - Err: fmt.Errorf("failed to create folder: %w", err), - AlertId: da.Id, + folderName := getAlertFolderNameFromDashboard(&dash) + f, ok := folderCache[folderName] + if !ok { + mg.Logger.Info("create a new folder for alerts that belongs to dashboard because it has custom permissions", "org", dash.OrgId, "dashboard_uid", dash.Uid, "folder", folderName) + // create folder and assign the permissions of the dashboard (included default and inherited) + f, err = m.createFolder(dash.OrgId, folderName) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to create folder: %w", err), + AlertId: da.Id, + } } - } - folder = *ptr - permissions, err := m.getACL(dash.OrgId, dash.Id) - if err != nil { - return MigrationError{ - Err: fmt.Errorf("failed to get dashboard %d under organisation %d permissions: %w", dash.Id, dash.OrgId, err), - AlertId: da.Id, + permissions, err := m.getACL(dash.OrgId, dash.Id) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to get dashboard %d under organisation %d permissions: %w", dash.Id, dash.OrgId, err), + AlertId: da.Id, + } } - } - err = m.setACL(folder.OrgId, folder.Id, permissions) - if err != nil { - return MigrationError{ - Err: fmt.Errorf("failed to set folder %d under organisation %d permissions: %w", folder.Id, folder.OrgId, err), - AlertId: da.Id, + err = m.setACL(f.OrgId, f.Id, permissions) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to set folder %d under organisation %d permissions: %w", folder.Id, folder.OrgId, err), + AlertId: da.Id, + } } + folderCache[folderName] = f } + folder = f case dash.FolderId > 0: - // link the new rule to the existing folder - default: - // get or create general folder - ptr, err := m.getOrCreateGeneralFolder(dash.OrgId) + // get folder if exists + f, err := m.getFolder(dash, da) if err != nil { return MigrationError{ - Err: fmt.Errorf("failed to get or create general folder under organisation %d: %w", dash.OrgId, err), + Err: err, AlertId: da.Id, } } + folder = &f + default: + f, ok := folderCache[GENERAL_FOLDER] + if !ok { + // get or create general folder + f, err = m.getOrCreateGeneralFolder(dash.OrgId) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to get or create general folder under organisation %d: %w", dash.OrgId, err), + AlertId: da.Id, + } + } + folderCache[GENERAL_FOLDER] = f + } // No need to assign default permissions to general folder // because they are included to the query result if it's a folder with no permissions // https://github.com/grafana/grafana/blob/076e2ce06a6ecf15804423fcc8dca1b620a321e5/pkg/services/sqlstore/dashboard_acl.go#L109 - folder = *ptr + folder = f } if folder.Uid == "" { @@ -476,37 +493,37 @@ func (m *migration) validateAlertmanagerConfig(orgID int64, config *PostableUser switch gr.Type { case "email": - _, err = channels.NewEmailNotifier(cfg, nil) // Email notifier already has a default template. + _, err = channels.NewEmailNotifier(cfg, nil, nil) // Email notifier already has a default template. case "pagerduty": - _, err = channels.NewPagerdutyNotifier(cfg, nil, decryptFunc) + _, err = channels.NewPagerdutyNotifier(cfg, nil, nil, decryptFunc) case "pushover": - _, err = channels.NewPushoverNotifier(cfg, nil, decryptFunc) + _, err = channels.NewPushoverNotifier(cfg, nil, nil, decryptFunc) case "slack": _, err = channels.NewSlackNotifier(cfg, nil, decryptFunc) case "telegram": - _, err = channels.NewTelegramNotifier(cfg, nil, decryptFunc) + _, err = channels.NewTelegramNotifier(cfg, nil, nil, decryptFunc) case "victorops": - _, err = channels.NewVictoropsNotifier(cfg, nil) + _, err = channels.NewVictoropsNotifier(cfg, nil, nil) case "teams": - _, err = channels.NewTeamsNotifier(cfg, nil) + _, err = channels.NewTeamsNotifier(cfg, nil, nil) case "dingding": - _, err = channels.NewDingDingNotifier(cfg, nil) + _, err = channels.NewDingDingNotifier(cfg, nil, nil) case "kafka": - _, err = channels.NewKafkaNotifier(cfg, nil) + _, err = channels.NewKafkaNotifier(cfg, nil, nil) case "webhook": - _, err = channels.NewWebHookNotifier(cfg, nil, decryptFunc) + _, err = channels.NewWebHookNotifier(cfg, nil, nil, decryptFunc) case "sensugo": - _, err = channels.NewSensuGoNotifier(cfg, nil, decryptFunc) + _, err = channels.NewSensuGoNotifier(cfg, nil, nil, decryptFunc) case "discord": - _, err = channels.NewDiscordNotifier(cfg, nil) + _, err = channels.NewDiscordNotifier(cfg, nil, nil) case "googlechat": - _, err = channels.NewGoogleChatNotifier(cfg, nil) + _, err = channels.NewGoogleChatNotifier(cfg, nil, nil) case "LINE": - _, err = channels.NewLineNotifier(cfg, nil, decryptFunc) + _, err = channels.NewLineNotifier(cfg, nil, nil, decryptFunc) case "threema": - _, err = channels.NewThreemaNotifier(cfg, nil, decryptFunc) + _, err = channels.NewThreemaNotifier(cfg, nil, nil, decryptFunc) case "opsgenie": - _, err = channels.NewOpsgenieNotifier(cfg, nil, decryptFunc) + _, err = channels.NewOpsgenieNotifier(cfg, nil, nil, decryptFunc) case "prometheus-alertmanager": _, err = channels.NewAlertmanagerNotifier(cfg, nil, decryptFunc) default: @@ -780,3 +797,14 @@ func CheckUnifiedAlertingEnabledByDefault(migrator *migrator.Migrator) error { migrator.Logger.Debug(fmt.Sprintf("Found %d legacy alerts in the database. Unified alerting enabled is %v", resp.Count, ualertEnabled)) return nil } + +// getAlertFolderNameFromDashboard generates a folder name for alerts that belong to a dashboard. Formats the string according to DASHBOARD_FOLDER format. +// If the resulting string exceeds the migrations.MaxTitleLength, the dashboard title is stripped to be at the maximum length +func getAlertFolderNameFromDashboard(dash *dashboard) string { + maxLen := MaxFolderName - len(fmt.Sprintf(DASHBOARD_FOLDER, "", dash.Uid)) + title := dash.Title + if len(title) > maxLen { + title = title[:maxLen] + } + return fmt.Sprintf(DASHBOARD_FOLDER, title, dash.Uid) // include UID to the name to avoid collision +} diff --git a/pkg/services/sqlstore/migrations/ualert/ualert_test.go b/pkg/services/sqlstore/migrations/ualert/ualert_test.go index 24a0632faea..967ff3d1a65 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert_test.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert_test.go @@ -184,3 +184,33 @@ func configFromReceivers(t *testing.T, receivers []*PostableGrafanaReceiver) *Po } const invalidUri = "�6�M��)uk譹1(�h`$�o�N>mĕ����cS2�dh![ę� ���`csB�!��OSxP�{�" + +func Test_getAlertFolderNameFromDashboard(t *testing.T) { + t.Run("should include full title", func(t *testing.T) { + dash := &dashboard{ + Uid: util.GenerateShortUID(), + Title: "TEST", + } + folder := getAlertFolderNameFromDashboard(dash) + require.Contains(t, folder, dash.Title) + require.Contains(t, folder, dash.Uid) + }) + t.Run("should cut title to the length", func(t *testing.T) { + title := "" + for { + title += util.GenerateShortUID() + if len(title) > MaxFolderName { + title = title[:MaxFolderName] + break + } + } + + dash := &dashboard{ + Uid: util.GenerateShortUID(), + Title: title, + } + folder := getAlertFolderNameFromDashboard(dash) + require.Len(t, folder, MaxFolderName) + require.Contains(t, folder, dash.Uid) + }) +} diff --git a/pkg/services/sqlstore/mockstore/mockstore.go b/pkg/services/sqlstore/mockstore/mockstore.go new file mode 100644 index 00000000000..7783611925d --- /dev/null +++ b/pkg/services/sqlstore/mockstore/mockstore.go @@ -0,0 +1,570 @@ +package mockstore + +import ( + "context" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/search" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +type SQLStoreMock struct { + SQLStore *sqlstore.SQLStore + ExpectedError error +} + +func NewSQLStoreMock() *SQLStoreMock { + return &SQLStoreMock{} +} + +func (m SQLStoreMock) DeleteExpiredSnapshots(ctx context.Context, cmd *models.DeleteExpiredSnapshotsCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) CreateDashboardSnapshot(ctx context.Context, cmd *models.CreateDashboardSnapshotCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteDashboardSnapshot(ctx context.Context, cmd *models.DeleteDashboardSnapshotCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDashboardSnapshot(query *models.GetDashboardSnapshotQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SearchDashboardSnapshots(query *models.GetDashboardSnapshotsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetOrgByName(name string) (*models.Org, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) CreateOrgWithMember(name string, userID int64) (models.Org, error) { + return models.Org{}, nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateOrg(ctx context.Context, cmd *models.UpdateOrgCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateOrgAddress(ctx context.Context, cmd *models.UpdateOrgAddressCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteOrg(ctx context.Context, cmd *models.DeleteOrgCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetProvisionedDataByDashboardID(dashboardID int64) (*models.DashboardProvisioning, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) GetProvisionedDataByDashboardUID(orgID int64, dashboardUID string) (*models.DashboardProvisioning, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) SaveProvisionedDashboard(cmd models.SaveDashboardCommand, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *models.DeleteOrphanedProvisionedDashboardsCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) CreateLoginAttempt(ctx context.Context, cmd *models.CreateLoginAttemptCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteOldLoginAttempts(ctx context.Context, cmd *models.DeleteOldLoginAttemptsCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) CloneUserToServiceAccount(ctx context.Context, siUser *models.SignedInUser) (*models.User, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) CreateServiceAccountForApikey(ctx context.Context, orgId int64, keyname string, role models.RoleType) (*models.User, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) CreateUser(ctx context.Context, cmd models.CreateUserCommand) (*models.User, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) GetUserById(ctx context.Context, query *models.GetUserByIdQuery) error { + return m.ExpectedError +} + +func (m SQLStoreMock) GetUserByLogin(ctx context.Context, query *models.GetUserByLoginQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetUserByEmail(ctx context.Context, query *models.GetUserByEmailQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateUser(ctx context.Context, cmd *models.UpdateUserCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) ChangeUserPassword(ctx context.Context, cmd *models.ChangeUserPasswordCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateUserLastSeenAt(ctx context.Context, cmd *models.UpdateUserLastSeenAtCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SetUsingOrg(ctx context.Context, cmd *models.SetUsingOrgCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetUserProfile(ctx context.Context, query *models.GetUserProfileQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetUserOrgList(ctx context.Context, query *models.GetUserOrgListQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetSignedInUserWithCacheCtx(ctx context.Context, query *models.GetSignedInUserQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetSignedInUser(ctx context.Context, query *models.GetSignedInUserQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) BatchDisableUsers(ctx context.Context, cmd *models.BatchDisableUsersCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteUser(ctx context.Context, cmd *models.DeleteUserCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateUserPermissions(userID int64, isAdmin bool) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SetUserHelpFlag(ctx context.Context, cmd *models.SetUserHelpFlagCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) CreateTeam(name string, email string, orgID int64) (models.Team, error) { + return models.Team{}, nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateTeam(ctx context.Context, cmd *models.UpdateTeamCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteTeam(ctx context.Context, cmd *models.DeleteTeamCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SearchTeams(ctx context.Context, query *models.SearchTeamsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetTeamById(ctx context.Context, query *models.GetTeamByIdQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetTeamsByUser(ctx context.Context, query *models.GetTeamsByUserQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) AddTeamMember(userID int64, orgID int64, teamID int64, isExternal bool, permission models.PermissionType) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateTeamMember(ctx context.Context, cmd *models.UpdateTeamMemberCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) IsTeamMember(orgId int64, teamId int64, userId int64) (bool, error) { + return false, nil // TODO: Implement +} + +func (m SQLStoreMock) RemoveTeamMember(ctx context.Context, cmd *models.RemoveTeamMemberCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetTeamMembers(ctx context.Context, query *models.GetTeamMembersQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) NewSession(ctx context.Context) *sqlstore.DBSession { + return nil // TODO: Implement +} + +func (m SQLStoreMock) WithDbSession(ctx context.Context, callback sqlstore.DBTransactionFunc) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetPreferencesWithDefaults(ctx context.Context, query *models.GetPreferencesWithDefaultsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetPreferences(ctx context.Context, query *models.GetPreferencesQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SavePreferences(ctx context.Context, cmd *models.SavePreferencesCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetPluginSettings(ctx context.Context, orgID int64) ([]*models.PluginSettingInfoDTO, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) GetPluginSettingById(ctx context.Context, query *models.GetPluginSettingByIdQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdatePluginSetting(ctx context.Context, cmd *models.UpdatePluginSettingCmd) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdatePluginSettingVersion(ctx context.Context, cmd *models.UpdatePluginSettingVersionCmd) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) IsStarredByUserCtx(ctx context.Context, query *models.IsStarredByUserQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) StarDashboard(ctx context.Context, cmd *models.StarDashboardCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UnstarDashboard(ctx context.Context, cmd *models.UnstarDashboardCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetUserStars(ctx context.Context, query *models.GetUserStarsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetOrgQuotaByTarget(ctx context.Context, query *models.GetOrgQuotaByTargetQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetOrgQuotas(ctx context.Context, query *models.GetOrgQuotasQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateOrgQuota(ctx context.Context, cmd *models.UpdateOrgQuotaCmd) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetUserQuotaByTarget(ctx context.Context, query *models.GetUserQuotaByTargetQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetUserQuotas(ctx context.Context, query *models.GetUserQuotasQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateUserQuota(ctx context.Context, cmd *models.UpdateUserQuotaCmd) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetGlobalQuotaByTarget(ctx context.Context, query *models.GetGlobalQuotaByTargetQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) WithTransactionalDbSession(ctx context.Context, callback sqlstore.DBTransactionFunc) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) InTransaction(ctx context.Context, fn func(ctx context.Context) error) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDashboardVersion(ctx context.Context, query *models.GetDashboardVersionQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDashboardVersions(ctx context.Context, query *models.GetDashboardVersionsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteExpiredVersions(ctx context.Context, cmd *models.DeleteExpiredVersionsCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateDashboardACL(ctx context.Context, dashboardID int64, items []*models.DashboardAcl) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateDashboardACLCtx(ctx context.Context, dashboardID int64, items []*models.DashboardAcl) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDashboardAclInfoList(ctx context.Context, query *models.GetDashboardAclInfoListQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) CreatePlaylist(ctx context.Context, cmd *models.CreatePlaylistCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdatePlaylist(ctx context.Context, cmd *models.UpdatePlaylistCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetPlaylist(ctx context.Context, query *models.GetPlaylistByIdQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeletePlaylist(ctx context.Context, cmd *models.DeletePlaylistCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SearchPlaylists(ctx context.Context, query *models.GetPlaylistsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetPlaylistItem(ctx context.Context, query *models.GetPlaylistItemsByIdQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAlertById(ctx context.Context, query *models.GetAlertByIdQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAllAlertQueryHandler(ctx context.Context, query *models.GetAllAlertsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) HandleAlertsQuery(ctx context.Context, query *models.GetAlertsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SaveAlerts(ctx context.Context, dashID int64, alerts []*models.Alert) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SetAlertState(ctx context.Context, cmd *models.SetAlertStateCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) PauseAlert(ctx context.Context, cmd *models.PauseAlertCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) PauseAllAlerts(ctx context.Context, cmd *models.PauseAllAlertCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAlertStatesForDashboard(ctx context.Context, query *models.GetAlertStatesForDashboardQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) AddOrgUser(ctx context.Context, cmd *models.AddOrgUserCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateOrgUser(ctx context.Context, cmd *models.UpdateOrgUserCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetOrgUsers(ctx context.Context, query *models.GetOrgUsersQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SearchOrgUsers(ctx context.Context, query *models.SearchOrgUsersQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) RemoveOrgUser(ctx context.Context, cmd *models.RemoveOrgUserCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SaveDashboard(cmd models.SaveDashboardCommand) (*models.Dashboard, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) GetDashboard(id int64, orgID int64, uid string, slug string) (*models.Dashboard, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) GetFolderByTitle(orgID int64, title string) (*models.Dashboard, error) { + return nil, nil // TODO: Implement +} + +func (m SQLStoreMock) SearchDashboards(ctx context.Context, query *search.FindPersistedDashboardsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteDashboard(ctx context.Context, cmd *models.DeleteDashboardCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDashboards(ctx context.Context, query *models.GetDashboardsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDashboardUIDById(ctx context.Context, query *models.GetDashboardRefByIdQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) ValidateDashboardBeforeSave(dashboard *models.Dashboard, overwrite bool) (bool, error) { + return false, nil // TODO: Implement +} + +func (m SQLStoreMock) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDataSources(ctx context.Context, query *models.GetDataSourcesQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDataSourcesByType(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) Migrate() error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) Sync() error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) Reset() error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) Quote(value string) string { + return "" // TODO: Implement +} + +func (m SQLStoreMock) DeleteAlertNotification(ctx context.Context, cmd *models.DeleteAlertNotificationCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteAlertNotificationWithUid(ctx context.Context, cmd *models.DeleteAlertNotificationWithUidCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAlertNotifications(ctx context.Context, query *models.GetAlertNotificationsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAlertNotificationUidWithId(ctx context.Context, query *models.GetAlertNotificationUidQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAlertNotificationsWithUid(ctx context.Context, query *models.GetAlertNotificationsWithUidQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAllAlertNotifications(ctx context.Context, query *models.GetAllAlertNotificationsQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAlertNotificationsWithUidToSend(ctx context.Context, query *models.GetAlertNotificationsWithUidToSendQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) CreateAlertNotificationCommand(ctx context.Context, cmd *models.CreateAlertNotificationCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateAlertNotification(ctx context.Context, cmd *models.UpdateAlertNotificationCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateAlertNotificationWithUid(ctx context.Context, cmd *models.UpdateAlertNotificationWithUidCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *models.SetAlertNotificationStateToCompleteCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *models.SetAlertNotificationStateToPendingCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetOrCreateAlertNotificationState(ctx context.Context, cmd *models.GetOrCreateNotificationStateQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetAPIKeys(ctx context.Context, query *models.GetApiKeysQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetNonServiceAccountAPIKeys(ctx context.Context) []*models.ApiKey { + return nil // TODO: Implement +} + +func (m SQLStoreMock) DeleteApiKey(ctx context.Context, cmd *models.DeleteApiKeyCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) AddAPIKey(ctx context.Context, cmd *models.AddApiKeyCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateApikeyServiceAccount(ctx context.Context, apikeyId int64, saccountId int64) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetApiKeyById(ctx context.Context, query *models.GetApiKeyByIdQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetApiKeyByName(ctx context.Context, query *models.GetApiKeyByNameQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateTempUserStatus(ctx context.Context, cmd *models.UpdateTempUserStatusCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) CreateTempUser(ctx context.Context, cmd *models.CreateTempUserCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) UpdateTempUserWithEmailSent(ctx context.Context, cmd *models.UpdateTempUserWithEmailSentCommand) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetTempUsersQuery(ctx context.Context, query *models.GetTempUsersQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) GetTempUserByCode(ctx context.Context, query *models.GetTempUserByCodeQuery) error { + return nil // TODO: Implement +} + +func (m SQLStoreMock) ExpireOldUserInvites(ctx context.Context, cmd *models.ExpireTempUsersCommand) error { + return nil // TODO: Implement +} diff --git a/pkg/services/sqlstore/org.go b/pkg/services/sqlstore/org.go index ae011aec032..cf90e4b3b6b 100644 --- a/pkg/services/sqlstore/org.go +++ b/pkg/services/sqlstore/org.go @@ -15,14 +15,14 @@ import ( // MainOrgName is the name of the main organization. const MainOrgName = "Main Org." -func init() { +func (ss *SQLStore) addOrgQueryAndCommandHandlers() { bus.AddHandler("sql", GetOrgById) bus.AddHandler("sql", CreateOrg) - bus.AddHandler("sql", UpdateOrg) - bus.AddHandler("sql", UpdateOrgAddress) + bus.AddHandler("sql", ss.UpdateOrg) + bus.AddHandler("sql", ss.UpdateOrgAddress) bus.AddHandler("sql", GetOrgByName) bus.AddHandler("sql", SearchOrgs) - bus.AddHandler("sql", DeleteOrg) + bus.AddHandler("sql", ss.DeleteOrg) } func SearchOrgs(ctx context.Context, query *models.SearchOrgsQuery) error { @@ -164,8 +164,8 @@ func CreateOrg(ctx context.Context, cmd *models.CreateOrgCommand) error { return nil } -func UpdateOrg(ctx context.Context, cmd *models.UpdateOrgCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) UpdateOrg(ctx context.Context, cmd *models.UpdateOrgCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { if isNameTaken, err := isOrgNameTaken(cmd.Name, cmd.OrgId, sess); err != nil { return err } else if isNameTaken { @@ -197,8 +197,8 @@ func UpdateOrg(ctx context.Context, cmd *models.UpdateOrgCommand) error { }) } -func UpdateOrgAddress(ctx context.Context, cmd *models.UpdateOrgAddressCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) UpdateOrgAddress(ctx context.Context, cmd *models.UpdateOrgAddressCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { org := models.Org{ Address1: cmd.Address1, Address2: cmd.Address2, @@ -224,8 +224,8 @@ func UpdateOrgAddress(ctx context.Context, cmd *models.UpdateOrgAddressCommand) }) } -func DeleteOrg(ctx context.Context, cmd *models.DeleteOrgCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) DeleteOrg(ctx context.Context, cmd *models.DeleteOrgCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { if res, err := sess.Query("SELECT 1 from org WHERE id=?", cmd.Id); err != nil { return err } else if len(res) != 1 { diff --git a/pkg/services/sqlstore/org_test.go b/pkg/services/sqlstore/org_test.go index e58cc4f6b52..2d3c5ffb103 100644 --- a/pkg/services/sqlstore/org_test.go +++ b/pkg/services/sqlstore/org_test.go @@ -87,9 +87,9 @@ func TestAccountDataAccess(t *testing.T) { q1 := models.GetUserOrgListQuery{UserId: ac1.Id} q2 := models.GetUserOrgListQuery{UserId: ac2.Id} - err = GetUserOrgList(context.Background(), &q1) + err = sqlStore.GetUserOrgList(context.Background(), &q1) require.NoError(t, err) - err = GetUserOrgList(context.Background(), &q2) + err = sqlStore.GetUserOrgList(context.Background(), &q2) require.NoError(t, err) require.Equal(t, q1.Result[0].OrgId, q2.Result[0].OrgId) @@ -195,7 +195,7 @@ func TestAccountDataAccess(t *testing.T) { t.Run("Can get logged in user projection", func(t *testing.T) { query := models.GetSignedInUserQuery{UserId: ac2.Id} - err := GetSignedInUser(context.Background(), &query) + err := sqlStore.GetSignedInUser(context.Background(), &query) require.NoError(t, err) require.Equal(t, query.Result.Email, "ac2@test.com") @@ -209,7 +209,7 @@ func TestAccountDataAccess(t *testing.T) { t.Run("Can get user organizations", func(t *testing.T) { query := models.GetUserOrgListQuery{UserId: ac2.Id} - err := GetUserOrgList(context.Background(), &query) + err := sqlStore.GetUserOrgList(context.Background(), &query) require.NoError(t, err) require.Equal(t, len(query.Result), 2) @@ -256,7 +256,7 @@ func TestAccountDataAccess(t *testing.T) { t.Run("SignedInUserQuery with a different org", func(t *testing.T) { query := models.GetSignedInUserQuery{UserId: ac2.Id} - err := GetSignedInUser(context.Background(), &query) + err := sqlStore.GetSignedInUser(context.Background(), &query) require.NoError(t, err) require.Equal(t, query.Result.OrgId, ac1.OrgId) @@ -273,7 +273,7 @@ func TestAccountDataAccess(t *testing.T) { require.NoError(t, err) query := models.GetSignedInUserQuery{UserId: ac2.Id} - err = GetSignedInUser(context.Background(), &query) + err = sqlStore.GetSignedInUser(context.Background(), &query) require.NoError(t, err) require.Equal(t, query.Result.OrgId, ac2.OrgId) @@ -282,7 +282,7 @@ func TestAccountDataAccess(t *testing.T) { t.Run("Removing user from org should delete user completely if in no other org", func(t *testing.T) { // make sure ac2 has no org - err := DeleteOrg(context.Background(), &models.DeleteOrgCommand{Id: ac2.OrgId}) + err := sqlStore.DeleteOrg(context.Background(), &models.DeleteOrgCommand{Id: ac2.OrgId}) require.NoError(t, err) // remove ac2 user from ac1 org @@ -291,7 +291,7 @@ func TestAccountDataAccess(t *testing.T) { require.NoError(t, err) require.True(t, remCmd.UserWasDeleted) - err = GetSignedInUser(context.Background(), &models.GetSignedInUserQuery{UserId: ac2.Id}) + err = sqlStore.GetSignedInUser(context.Background(), &models.GetSignedInUserQuery{UserId: ac2.Id}) require.Equal(t, err, models.ErrUserNotFound) }) diff --git a/pkg/services/sqlstore/org_users.go b/pkg/services/sqlstore/org_users.go index 898ace7dff1..f0817f16d27 100644 --- a/pkg/services/sqlstore/org_users.go +++ b/pkg/services/sqlstore/org_users.go @@ -8,6 +8,8 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/util" ) @@ -107,10 +109,24 @@ func (ss *SQLStore) GetOrgUsers(ctx context.Context, query *models.GetOrgUsersQu whereConditions = append(whereConditions, "org_user.org_id = ?") whereParams = append(whereParams, query.OrgId) + if query.UserID != 0 { + whereConditions = append(whereConditions, "org_user.user_id = ?") + whereParams = append(whereParams, query.UserID) + } + // TODO: add to chore, for cleaning up after we have created // service accounts table in the modelling whereConditions = append(whereConditions, fmt.Sprintf("%s.is_service_account = %t", x.Dialect().Quote("user"), query.IsServiceAccount)) + if ss.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagAccesscontrol) { + q, args, err := accesscontrol.Filter(ctx, "org_user.user_id", "users", "org.users:read", query.User) + if err != nil { + return err + } + whereConditions = append(whereConditions, q) + whereParams = append(whereParams, args...) + } + if query.Query != "" { queryWithWildcards := "%" + query.Query + "%" whereConditions = append(whereConditions, "(email "+dialect.LikeStr()+" ? OR name "+dialect.LikeStr()+" ? OR login "+dialect.LikeStr()+" ?)") @@ -165,6 +181,15 @@ func (ss *SQLStore) SearchOrgUsers(ctx context.Context, query *models.SearchOrgU // service accounts table in the modelling whereConditions = append(whereConditions, fmt.Sprintf("%s.is_service_account = %t", x.Dialect().Quote("user"), query.IsServiceAccount)) + if ss.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagAccesscontrol) { + q, args, err := accesscontrol.Filter(ctx, "org_user.user_id", "users", "org.users:read", query.User) + if err != nil { + return err + } + whereConditions = append(whereConditions, q) + whereParams = append(whereParams, args...) + } + if query.Query != "" { queryWithWildcards := "%" + query.Query + "%" whereConditions = append(whereConditions, "(email "+dialect.LikeStr()+" ? OR name "+dialect.LikeStr()+" ? OR login "+dialect.LikeStr()+" ?)") diff --git a/pkg/services/sqlstore/org_users_test.go b/pkg/services/sqlstore/org_users_test.go new file mode 100644 index 00000000000..141870d05e3 --- /dev/null +++ b/pkg/services/sqlstore/org_users_test.go @@ -0,0 +1,177 @@ +package sqlstore + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/models" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/featuremgmt" +) + +type getOrgUsersTestCase struct { + desc string + query *models.GetOrgUsersQuery + expectedNumUsers int +} + +func TestSQLStore_GetOrgUsers(t *testing.T) { + tests := []getOrgUsersTestCase{ + { + desc: "should return all users", + query: &models.GetOrgUsersQuery{ + OrgId: 1, + User: &models.SignedInUser{ + OrgId: 1, + Permissions: map[int64]map[string][]string{1: {ac.ActionOrgUsersRead: {ac.ScopeUsersAll}}}, + }, + }, + expectedNumUsers: 10, + }, + { + desc: "should return no users", + query: &models.GetOrgUsersQuery{ + OrgId: 1, + User: &models.SignedInUser{ + OrgId: 1, + Permissions: map[int64]map[string][]string{1: {ac.ActionOrgUsersRead: {""}}}, + }, + }, + expectedNumUsers: 0, + }, + { + desc: "should return some users", + query: &models.GetOrgUsersQuery{ + OrgId: 1, + User: &models.SignedInUser{ + OrgId: 1, + Permissions: map[int64]map[string][]string{1: {ac.ActionOrgUsersRead: { + "users:id:1", + "users:id:5", + "users:id:9", + }}}, + }, + }, + expectedNumUsers: 3, + }, + } + + store := InitTestDB(t) + store.Cfg.IsFeatureToggleEnabled = featuremgmt.WithFeatures(featuremgmt.FlagAccesscontrol).IsEnabled + seedOrgUsers(t, store, 10) + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + err := store.GetOrgUsers(context.Background(), tt.query) + require.NoError(t, err) + require.Len(t, tt.query.Result, tt.expectedNumUsers) + + if !hasWildcardScope(tt.query.User, ac.ActionOrgUsersRead) { + for _, u := range tt.query.Result { + assert.Contains(t, tt.query.User.Permissions[tt.query.User.OrgId][ac.ActionOrgUsersRead], fmt.Sprintf("users:id:%d", u.UserId)) + } + } + }) + } +} + +type searchOrgUsersTestCase struct { + desc string + query *models.SearchOrgUsersQuery + expectedNumUsers int +} + +func TestSQLStore_SearchOrgUsers(t *testing.T) { + tests := []searchOrgUsersTestCase{ + { + desc: "should return all users", + query: &models.SearchOrgUsersQuery{ + OrgID: 1, + User: &models.SignedInUser{ + OrgId: 1, + Permissions: map[int64]map[string][]string{1: {ac.ActionOrgUsersRead: {ac.ScopeUsersAll}}}, + }, + }, + expectedNumUsers: 10, + }, + { + desc: "should return no users", + query: &models.SearchOrgUsersQuery{ + OrgID: 1, + User: &models.SignedInUser{ + OrgId: 1, + Permissions: map[int64]map[string][]string{1: {ac.ActionOrgUsersRead: {""}}}, + }, + }, + expectedNumUsers: 0, + }, + { + desc: "should return some users", + query: &models.SearchOrgUsersQuery{ + OrgID: 1, + User: &models.SignedInUser{ + OrgId: 1, + Permissions: map[int64]map[string][]string{1: {ac.ActionOrgUsersRead: { + "users:id:1", + "users:id:5", + "users:id:9", + }}}, + }, + }, + expectedNumUsers: 3, + }, + } + + store := InitTestDB(t) + store.Cfg.IsFeatureToggleEnabled = featuremgmt.WithFeatures(featuremgmt.FlagAccesscontrol).IsEnabled + seedOrgUsers(t, store, 10) + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + err := store.SearchOrgUsers(context.Background(), tt.query) + require.NoError(t, err) + assert.Len(t, tt.query.Result.OrgUsers, tt.expectedNumUsers) + + if !hasWildcardScope(tt.query.User, ac.ActionOrgUsersRead) { + for _, u := range tt.query.Result.OrgUsers { + assert.Contains(t, tt.query.User.Permissions[tt.query.User.OrgId][ac.ActionOrgUsersRead], fmt.Sprintf("users:id:%d", u.UserId)) + } + } + }) + } +} + +func seedOrgUsers(t *testing.T, store *SQLStore, numUsers int) { + t.Helper() + // Seed users + for i := 1; i <= numUsers; i++ { + user, err := store.CreateUser(context.Background(), models.CreateUserCommand{ + Login: fmt.Sprintf("user-%d", i), + OrgId: 1, + }) + require.NoError(t, err) + + if i != 1 { + err = store.AddOrgUser(context.Background(), &models.AddOrgUserCommand{ + Role: "Viewer", + OrgId: 1, + UserId: user.Id, + }) + require.NoError(t, err) + } + } +} + +func hasWildcardScope(user *models.SignedInUser, action string) bool { + for _, scope := range user.Permissions[user.OrgId][action] { + if strings.HasSuffix(scope, ":*") { + return true + } + } + return false +} diff --git a/pkg/services/sqlstore/session.go b/pkg/services/sqlstore/session.go index a739b385801..5545525d0b8 100644 --- a/pkg/services/sqlstore/session.go +++ b/pkg/services/sqlstore/session.go @@ -12,7 +12,7 @@ type DBSession struct { events []interface{} } -type dbTransactionFunc func(sess *DBSession) error +type DBTransactionFunc func(sess *DBSession) error func (sess *DBSession) publishAfterCommit(msg interface{}) { sess.events = append(sess.events, msg) @@ -55,11 +55,11 @@ func startSession(ctx context.Context, engine *xorm.Engine, beginTran bool) (*DB } // WithDbSession calls the callback with a session. -func (ss *SQLStore) WithDbSession(ctx context.Context, callback dbTransactionFunc) error { +func (ss *SQLStore) WithDbSession(ctx context.Context, callback DBTransactionFunc) error { return withDbSession(ctx, ss.engine, callback) } -func withDbSession(ctx context.Context, engine *xorm.Engine, callback dbTransactionFunc) error { +func withDbSession(ctx context.Context, engine *xorm.Engine, callback DBTransactionFunc) error { sess := &DBSession{Session: engine.NewSession()} sess.Session = sess.Session.Context(ctx) defer sess.Close() diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 734b1f23653..2cddc4c7519 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -12,21 +12,24 @@ import ( "time" "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + "xorm.io/xorm" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/annotations" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore/migrations" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util/errutil" - _ "github.com/lib/pq" - "xorm.io/xorm" ) var ( @@ -50,14 +53,16 @@ type SQLStore struct { Dialect migrator.Dialect skipEnsureDefaultOrgAndUser bool migrations registry.DatabaseMigrator + tracer tracing.Tracer } -func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bus.Bus, migrations registry.DatabaseMigrator) (*SQLStore, error) { +func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bus.Bus, migrations registry.DatabaseMigrator, tracer tracing.Tracer, +) (*SQLStore, error) { // This change will make xorm use an empty default schema for postgres and // by that mimic the functionality of how it was functioning before // xorm's changes above. xorm.DefaultPostgresSchema = "" - s, err := newSQLStore(cfg, cacheService, bus, nil, migrations) + s, err := newSQLStore(cfg, cacheService, bus, nil, migrations, tracer) if err != nil { return nil, err } @@ -69,7 +74,7 @@ func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, bus if err := s.Reset(); err != nil { return nil, err } - + s.tracer = tracer return s, nil } @@ -78,7 +83,7 @@ func ProvideServiceForTests(migrations registry.DatabaseMigrator) (*SQLStore, er } func newSQLStore(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bus.Bus, engine *xorm.Engine, - migrations registry.DatabaseMigrator, opts ...InitTestDBOpt) (*SQLStore, error) { + migrations registry.DatabaseMigrator, tracer tracing.Tracer, opts ...InitTestDBOpt) (*SQLStore, error) { ss := &SQLStore{ Cfg: cfg, Bus: bus, @@ -86,6 +91,7 @@ func newSQLStore(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bu log: log.New("sqlstore"), skipEnsureDefaultOrgAndUser: false, migrations: migrations, + tracer: tracer, } for _, opt := range opts { if !opt.EnsureDefaultOrgAndUser { @@ -122,6 +128,10 @@ func newSQLStore(cfg *setting.Cfg, cacheService *localcache.CacheService, bus bu ss.addDashboardVersionQueryAndCommandHandlers() ss.addAPIKeysQueryAndCommandHandlers() ss.addPlaylistQueryAndCommandHandlers() + ss.addLoginAttemptQueryAndCommandHandlers() + ss.addTeamQueryAndCommandHandlers() + ss.addDashboardProvisioningQueryAndCommandHandlers() + ss.addOrgQueryAndCommandHandlers() // if err := ss.Reset(); err != nil { // return nil, err @@ -321,8 +331,8 @@ func (ss *SQLStore) initEngine(engine *xorm.Engine) error { return err } - if ss.Cfg.IsDatabaseMetricsEnabled() { - ss.dbCfg.Type = WrapDatabaseDriverWithHooks(ss.dbCfg.Type) + if ss.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagDatabaseMetrics) { + ss.dbCfg.Type = WrapDatabaseDriverWithHooks(ss.dbCfg.Type, ss.tracer) } sqlog.Info("Connecting to DB", "dbtype", ss.dbCfg.Type) @@ -372,7 +382,8 @@ func (ss *SQLStore) initEngine(engine *xorm.Engine) error { if !debugSQL { engine.SetLogger(&xorm.DiscardLogger{}) } else { - engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm"))) + // add stack to database calls to be able to see what repository initiated queries. Top 7 items from the stack as they are likely in the xorm library. + engine.SetLogger(NewXormLogger(log.LvlInfo, log.WithSuffix(log.New("sqlstore.xorm"), log.CallerContextKey, log.StackCaller(log.DefaultCallerDepth)))) engine.ShowSQL(true) engine.ShowExecTime(true) } @@ -486,6 +497,7 @@ func initTestDB(migration registry.DatabaseMigrator, opts ...InitTestDBOpt) (*SQ // set test db config cfg := setting.NewCfg() + cfg.IsFeatureToggleEnabled = func(key string) bool { return false } sec, err := cfg.Raw.NewSection("database") if err != nil { return nil, err @@ -526,7 +538,11 @@ func initTestDB(migration registry.DatabaseMigrator, opts ...InitTestDBOpt) (*SQ engine.DatabaseTZ = time.UTC engine.TZLocation = time.UTC - testSQLStore, err = newSQLStore(cfg, localcache.New(5*time.Minute, 10*time.Minute), bus.GetBus(), engine, migration, opts...) + tracer, err := tracing.InitializeTracerForTest() + if err != nil { + return nil, err + } + testSQLStore, err = newSQLStore(cfg, localcache.New(5*time.Minute, 10*time.Minute), bus.GetBus(), engine, migration, tracer, opts...) if err != nil { return nil, err } diff --git a/pkg/services/sqlstore/store.go b/pkg/services/sqlstore/store.go new file mode 100644 index 00000000000..ddb8595582d --- /dev/null +++ b/pkg/services/sqlstore/store.go @@ -0,0 +1,149 @@ +package sqlstore + +import ( + "context" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/search" +) + +type Store interface { + DeleteExpiredSnapshots(ctx context.Context, cmd *models.DeleteExpiredSnapshotsCommand) error + CreateDashboardSnapshot(ctx context.Context, cmd *models.CreateDashboardSnapshotCommand) error + DeleteDashboardSnapshot(ctx context.Context, cmd *models.DeleteDashboardSnapshotCommand) error + GetDashboardSnapshot(query *models.GetDashboardSnapshotQuery) error + SearchDashboardSnapshots(query *models.GetDashboardSnapshotsQuery) error + GetOrgByName(name string) (*models.Org, error) + CreateOrgWithMember(name string, userID int64) (models.Org, error) + UpdateOrg(ctx context.Context, cmd *models.UpdateOrgCommand) error + UpdateOrgAddress(ctx context.Context, cmd *models.UpdateOrgAddressCommand) error + DeleteOrg(ctx context.Context, cmd *models.DeleteOrgCommand) error + GetProvisionedDataByDashboardID(dashboardID int64) (*models.DashboardProvisioning, error) + GetProvisionedDataByDashboardUID(orgID int64, dashboardUID string) (*models.DashboardProvisioning, error) + SaveProvisionedDashboard(cmd models.SaveDashboardCommand, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) + GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) + DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *models.DeleteOrphanedProvisionedDashboardsCommand) error + CreateLoginAttempt(ctx context.Context, cmd *models.CreateLoginAttemptCommand) error + DeleteOldLoginAttempts(ctx context.Context, cmd *models.DeleteOldLoginAttemptsCommand) error + CloneUserToServiceAccount(ctx context.Context, siUser *models.SignedInUser) (*models.User, error) + CreateServiceAccountForApikey(ctx context.Context, orgId int64, keyname string, role models.RoleType) (*models.User, error) + CreateUser(ctx context.Context, cmd models.CreateUserCommand) (*models.User, error) + GetUserById(ctx context.Context, query *models.GetUserByIdQuery) error + GetUserByLogin(ctx context.Context, query *models.GetUserByLoginQuery) error + GetUserByEmail(ctx context.Context, query *models.GetUserByEmailQuery) error + UpdateUser(ctx context.Context, cmd *models.UpdateUserCommand) error + ChangeUserPassword(ctx context.Context, cmd *models.ChangeUserPasswordCommand) error + UpdateUserLastSeenAt(ctx context.Context, cmd *models.UpdateUserLastSeenAtCommand) error + SetUsingOrg(ctx context.Context, cmd *models.SetUsingOrgCommand) error + GetUserProfile(ctx context.Context, query *models.GetUserProfileQuery) error + GetUserOrgList(ctx context.Context, query *models.GetUserOrgListQuery) error + GetSignedInUserWithCacheCtx(ctx context.Context, query *models.GetSignedInUserQuery) error + GetSignedInUser(ctx context.Context, query *models.GetSignedInUserQuery) error + BatchDisableUsers(ctx context.Context, cmd *models.BatchDisableUsersCommand) error + DeleteUser(ctx context.Context, cmd *models.DeleteUserCommand) error + UpdateUserPermissions(userID int64, isAdmin bool) error + SetUserHelpFlag(ctx context.Context, cmd *models.SetUserHelpFlagCommand) error + CreateTeam(name, email string, orgID int64) (models.Team, error) + UpdateTeam(ctx context.Context, cmd *models.UpdateTeamCommand) error + DeleteTeam(ctx context.Context, cmd *models.DeleteTeamCommand) error + SearchTeams(ctx context.Context, query *models.SearchTeamsQuery) error + GetTeamById(ctx context.Context, query *models.GetTeamByIdQuery) error + GetTeamsByUser(ctx context.Context, query *models.GetTeamsByUserQuery) error + AddTeamMember(userID, orgID, teamID int64, isExternal bool, permission models.PermissionType) error + UpdateTeamMember(ctx context.Context, cmd *models.UpdateTeamMemberCommand) error + IsTeamMember(orgId int64, teamId int64, userId int64) (bool, error) + RemoveTeamMember(ctx context.Context, cmd *models.RemoveTeamMemberCommand) error + GetTeamMembers(ctx context.Context, query *models.GetTeamMembersQuery) error + NewSession(ctx context.Context) *DBSession + WithDbSession(ctx context.Context, callback DBTransactionFunc) error + GetPreferencesWithDefaults(ctx context.Context, query *models.GetPreferencesWithDefaultsQuery) error + GetPreferences(ctx context.Context, query *models.GetPreferencesQuery) error + SavePreferences(ctx context.Context, cmd *models.SavePreferencesCommand) error + GetPluginSettings(ctx context.Context, orgID int64) ([]*models.PluginSettingInfoDTO, error) + GetPluginSettingById(ctx context.Context, query *models.GetPluginSettingByIdQuery) error + UpdatePluginSetting(ctx context.Context, cmd *models.UpdatePluginSettingCmd) error + UpdatePluginSettingVersion(ctx context.Context, cmd *models.UpdatePluginSettingVersionCmd) error + IsStarredByUserCtx(ctx context.Context, query *models.IsStarredByUserQuery) error + StarDashboard(ctx context.Context, cmd *models.StarDashboardCommand) error + UnstarDashboard(ctx context.Context, cmd *models.UnstarDashboardCommand) error + GetUserStars(ctx context.Context, query *models.GetUserStarsQuery) error + GetOrgQuotaByTarget(ctx context.Context, query *models.GetOrgQuotaByTargetQuery) error + GetOrgQuotas(ctx context.Context, query *models.GetOrgQuotasQuery) error + UpdateOrgQuota(ctx context.Context, cmd *models.UpdateOrgQuotaCmd) error + GetUserQuotaByTarget(ctx context.Context, query *models.GetUserQuotaByTargetQuery) error + GetUserQuotas(ctx context.Context, query *models.GetUserQuotasQuery) error + UpdateUserQuota(ctx context.Context, cmd *models.UpdateUserQuotaCmd) error + GetGlobalQuotaByTarget(ctx context.Context, query *models.GetGlobalQuotaByTargetQuery) error + WithTransactionalDbSession(ctx context.Context, callback DBTransactionFunc) error + InTransaction(ctx context.Context, fn func(ctx context.Context) error) error + GetDashboardVersion(ctx context.Context, query *models.GetDashboardVersionQuery) error + GetDashboardVersions(ctx context.Context, query *models.GetDashboardVersionsQuery) error + DeleteExpiredVersions(ctx context.Context, cmd *models.DeleteExpiredVersionsCommand) error + UpdateDashboardACL(ctx context.Context, dashboardID int64, items []*models.DashboardAcl) error + UpdateDashboardACLCtx(ctx context.Context, dashboardID int64, items []*models.DashboardAcl) error + GetDashboardAclInfoList(ctx context.Context, query *models.GetDashboardAclInfoListQuery) error + CreatePlaylist(ctx context.Context, cmd *models.CreatePlaylistCommand) error + UpdatePlaylist(ctx context.Context, cmd *models.UpdatePlaylistCommand) error + GetPlaylist(ctx context.Context, query *models.GetPlaylistByIdQuery) error + DeletePlaylist(ctx context.Context, cmd *models.DeletePlaylistCommand) error + SearchPlaylists(ctx context.Context, query *models.GetPlaylistsQuery) error + GetPlaylistItem(ctx context.Context, query *models.GetPlaylistItemsByIdQuery) error + GetAlertById(ctx context.Context, query *models.GetAlertByIdQuery) error + GetAllAlertQueryHandler(ctx context.Context, query *models.GetAllAlertsQuery) error + HandleAlertsQuery(ctx context.Context, query *models.GetAlertsQuery) error + SaveAlerts(ctx context.Context, dashID int64, alerts []*models.Alert) error + SetAlertState(ctx context.Context, cmd *models.SetAlertStateCommand) error + PauseAlert(ctx context.Context, cmd *models.PauseAlertCommand) error + PauseAllAlerts(ctx context.Context, cmd *models.PauseAllAlertCommand) error + GetAlertStatesForDashboard(ctx context.Context, query *models.GetAlertStatesForDashboardQuery) error + AddOrgUser(ctx context.Context, cmd *models.AddOrgUserCommand) error + UpdateOrgUser(ctx context.Context, cmd *models.UpdateOrgUserCommand) error + GetOrgUsers(ctx context.Context, query *models.GetOrgUsersQuery) error + SearchOrgUsers(ctx context.Context, query *models.SearchOrgUsersQuery) error + RemoveOrgUser(ctx context.Context, cmd *models.RemoveOrgUserCommand) error + SaveDashboard(cmd models.SaveDashboardCommand) (*models.Dashboard, error) + GetDashboard(id, orgID int64, uid, slug string) (*models.Dashboard, error) + GetFolderByTitle(orgID int64, title string) (*models.Dashboard, error) + SearchDashboards(ctx context.Context, query *search.FindPersistedDashboardsQuery) error + DeleteDashboard(ctx context.Context, cmd *models.DeleteDashboardCommand) error + GetDashboards(ctx context.Context, query *models.GetDashboardsQuery) error + GetDashboardUIDById(ctx context.Context, query *models.GetDashboardRefByIdQuery) error + ValidateDashboardBeforeSave(dashboard *models.Dashboard, overwrite bool) (bool, error) + GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error + GetDataSources(ctx context.Context, query *models.GetDataSourcesQuery) error + GetDataSourcesByType(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error + GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error + DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error + AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error + UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error + Migrate() error + Sync() error + Reset() error + Quote(value string) string + DeleteAlertNotification(ctx context.Context, cmd *models.DeleteAlertNotificationCommand) error + DeleteAlertNotificationWithUid(ctx context.Context, cmd *models.DeleteAlertNotificationWithUidCommand) error + GetAlertNotifications(ctx context.Context, query *models.GetAlertNotificationsQuery) error + GetAlertNotificationUidWithId(ctx context.Context, query *models.GetAlertNotificationUidQuery) error + GetAlertNotificationsWithUid(ctx context.Context, query *models.GetAlertNotificationsWithUidQuery) error + GetAllAlertNotifications(ctx context.Context, query *models.GetAllAlertNotificationsQuery) error + GetAlertNotificationsWithUidToSend(ctx context.Context, query *models.GetAlertNotificationsWithUidToSendQuery) error + CreateAlertNotificationCommand(ctx context.Context, cmd *models.CreateAlertNotificationCommand) error + UpdateAlertNotification(ctx context.Context, cmd *models.UpdateAlertNotificationCommand) error + UpdateAlertNotificationWithUid(ctx context.Context, cmd *models.UpdateAlertNotificationWithUidCommand) error + SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *models.SetAlertNotificationStateToCompleteCommand) error + SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *models.SetAlertNotificationStateToPendingCommand) error + GetOrCreateAlertNotificationState(ctx context.Context, cmd *models.GetOrCreateNotificationStateQuery) error + GetAPIKeys(ctx context.Context, query *models.GetApiKeysQuery) error + GetNonServiceAccountAPIKeys(ctx context.Context) []*models.ApiKey + DeleteApiKey(ctx context.Context, cmd *models.DeleteApiKeyCommand) error + AddAPIKey(ctx context.Context, cmd *models.AddApiKeyCommand) error + UpdateApikeyServiceAccount(ctx context.Context, apikeyId int64, saccountId int64) error + GetApiKeyById(ctx context.Context, query *models.GetApiKeyByIdQuery) error + GetApiKeyByName(ctx context.Context, query *models.GetApiKeyByNameQuery) error + UpdateTempUserStatus(ctx context.Context, cmd *models.UpdateTempUserStatusCommand) error + CreateTempUser(ctx context.Context, cmd *models.CreateTempUserCommand) error + UpdateTempUserWithEmailSent(ctx context.Context, cmd *models.UpdateTempUserWithEmailSentCommand) error + GetTempUsersQuery(ctx context.Context, query *models.GetTempUsersQuery) error + GetTempUserByCode(ctx context.Context, query *models.GetTempUserByCodeQuery) error + ExpireOldUserInvites(ctx context.Context, cmd *models.ExpireTempUsersCommand) error +} diff --git a/pkg/services/sqlstore/team.go b/pkg/services/sqlstore/team.go index ecf14d26ff0..5dd220e251a 100644 --- a/pkg/services/sqlstore/team.go +++ b/pkg/services/sqlstore/team.go @@ -9,21 +9,33 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" ) -func init() { - bus.AddHandler("sql", UpdateTeam) - bus.AddHandler("sql", DeleteTeam) - bus.AddHandler("sql", SearchTeams) - bus.AddHandler("sql", GetTeamById) - bus.AddHandler("sql", GetTeamsByUser) +func (ss *SQLStore) addTeamQueryAndCommandHandlers() { + bus.AddHandler("sql", ss.UpdateTeam) + bus.AddHandler("sql", ss.DeleteTeam) + bus.AddHandler("sql", ss.SearchTeams) + bus.AddHandler("sql", ss.GetTeamById) + bus.AddHandler("sql", ss.GetTeamsByUser) - bus.AddHandler("sql", UpdateTeamMember) - bus.AddHandler("sql", RemoveTeamMember) - bus.AddHandler("sql", GetTeamMembers) + bus.AddHandler("sql", ss.UpdateTeamMember) + bus.AddHandler("sql", ss.RemoveTeamMember) + bus.AddHandler("sql", ss.GetTeamMembers) bus.AddHandler("sql", IsAdminOfTeams) } +type TeamStore interface { + UpdateTeam(ctx context.Context, cmd *models.UpdateTeamCommand) error + DeleteTeam(ctx context.Context, cmd *models.DeleteTeamCommand) error + SearchTeams(ctx context.Context, query *models.SearchTeamsQuery) error + GetTeamById(ctx context.Context, query *models.GetTeamByIdQuery) error + UpdateTeamMember(ctx context.Context, cmd *models.UpdateTeamMemberCommand) error + RemoveTeamMember(ctx context.Context, cmd *models.RemoveTeamMemberCommand) error + GetTeamMembers(ctx context.Context, cmd *models.GetTeamMembersQuery) error + AddOrUpdateTeamMember(userID, orgID, teamID int64, isExternal bool, permission models.PermissionType) error +} + func getFilteredUsers(signedInUser *models.SignedInUser, hiddenUsers map[string]struct{}) []string { filteredUsers := make([]string, 0, len(hiddenUsers)) if signedInUser == nil || signedInUser.IsGrafanaAdmin { @@ -95,8 +107,8 @@ func (ss *SQLStore) CreateTeam(name, email string, orgID int64) (models.Team, er return team, err } -func UpdateTeam(ctx context.Context, cmd *models.UpdateTeamCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) UpdateTeam(ctx context.Context, cmd *models.UpdateTeamCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { if isNameTaken, err := isTeamNameTaken(cmd.OrgId, cmd.Name, cmd.Id, sess); err != nil { return err } else if isNameTaken { @@ -126,8 +138,8 @@ func UpdateTeam(ctx context.Context, cmd *models.UpdateTeamCommand) error { } // DeleteTeam will delete a team, its member and any permissions connected to the team -func DeleteTeam(ctx context.Context, cmd *models.DeleteTeamCommand) error { - return inTransaction(func(sess *DBSession) error { +func (ss *SQLStore) DeleteTeam(ctx context.Context, cmd *models.DeleteTeamCommand) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { if _, err := teamExists(cmd.OrgId, cmd.Id, sess); err != nil { return err } @@ -136,6 +148,7 @@ func DeleteTeam(ctx context.Context, cmd *models.DeleteTeamCommand) error { "DELETE FROM team_member WHERE org_id=? and team_id = ?", "DELETE FROM team WHERE org_id=? and id = ?", "DELETE FROM dashboard_acl WHERE org_id=? and team_id = ?", + "DELETE FROM team_role WHERE org_id=? and team_id = ?", } for _, sql := range deletes { @@ -144,7 +157,10 @@ func DeleteTeam(ctx context.Context, cmd *models.DeleteTeamCommand) error { return err } } - return nil + + _, err := sess.Exec("DELETE FROM permission WHERE scope=?", ac.Scope("teams", "id", fmt.Sprint(cmd.Id))) + + return err }) } @@ -172,7 +188,7 @@ func isTeamNameTaken(orgId int64, name string, existingId int64, sess *DBSession return false, nil } -func SearchTeams(ctx context.Context, query *models.SearchTeamsQuery) error { +func (ss *SQLStore) SearchTeams(ctx context.Context, query *models.SearchTeamsQuery) error { query.Result = models.SearchTeamQueryResult{ Teams: make([]*models.TeamDTO, 0), } @@ -235,7 +251,7 @@ func SearchTeams(ctx context.Context, query *models.SearchTeamsQuery) error { return err } -func GetTeamById(ctx context.Context, query *models.GetTeamByIdQuery) error { +func (ss *SQLStore) GetTeamById(ctx context.Context, query *models.GetTeamByIdQuery) error { var sql bytes.Buffer params := make([]interface{}, 0) @@ -264,45 +280,31 @@ func GetTeamById(ctx context.Context, query *models.GetTeamByIdQuery) error { } // GetTeamsByUser is used by the Guardian when checking a users' permissions -func GetTeamsByUser(ctx context.Context, query *models.GetTeamsByUserQuery) error { - query.Result = make([]*models.TeamDTO, 0) +func (ss *SQLStore) GetTeamsByUser(ctx context.Context, query *models.GetTeamsByUserQuery) error { + return ss.WithDbSession(ctx, func(sess *DBSession) error { + query.Result = make([]*models.TeamDTO, 0) - var sql bytes.Buffer + var sql bytes.Buffer - sql.WriteString(getTeamSelectSQLBase([]string{})) - sql.WriteString(` INNER JOIN team_member on team.id = team_member.team_id`) - sql.WriteString(` WHERE team.org_id = ? and team_member.user_id = ?`) + sql.WriteString(getTeamSelectSQLBase([]string{})) + sql.WriteString(` INNER JOIN team_member on team.id = team_member.team_id`) + sql.WriteString(` WHERE team.org_id = ? and team_member.user_id = ?`) - err := x.SQL(sql.String(), query.OrgId, query.UserId).Find(&query.Result) - return err + err := sess.SQL(sql.String(), query.OrgId, query.UserId).Find(&query.Result) + return err + }) } // AddTeamMember adds a user to a team func (ss *SQLStore) AddTeamMember(userID, orgID, teamID int64, isExternal bool, permission models.PermissionType) error { return ss.WithTransactionalDbSession(context.Background(), func(sess *DBSession) error { - if res, err := sess.Query("SELECT 1 from team_member WHERE org_id=? and team_id=? and user_id=?", - orgID, teamID, userID); err != nil { + if isMember, err := isTeamMember(sess, orgID, teamID, userID); err != nil { return err - } else if len(res) == 1 { + } else if isMember { return models.ErrTeamMemberAlreadyAdded } - if _, err := teamExists(orgID, teamID, sess); err != nil { - return err - } - - entity := models.TeamMember{ - OrgId: orgID, - TeamId: teamID, - UserId: userID, - External: isExternal, - Created: time.Now(), - Updated: time.Now(), - Permission: permission, - } - - _, err := sess.Insert(&entity) - return err + return addTeamMember(sess, orgID, teamID, userID, isExternal, permission) }) } @@ -322,59 +324,127 @@ func getTeamMember(sess *DBSession, orgId int64, teamId int64, userId int64) (mo } // UpdateTeamMember updates a team member -func UpdateTeamMember(ctx context.Context, cmd *models.UpdateTeamMemberCommand) error { +func (ss *SQLStore) UpdateTeamMember(ctx context.Context, cmd *models.UpdateTeamMemberCommand) error { return inTransaction(func(sess *DBSession) error { - member, err := getTeamMember(sess, cmd.OrgId, cmd.TeamId, cmd.UserId) - if err != nil { - return err - } - - if cmd.ProtectLastAdmin { - _, err := isLastAdmin(sess, cmd.OrgId, cmd.TeamId, cmd.UserId) - if err != nil { - return err - } - } - - if cmd.Permission != models.PERMISSION_ADMIN { // make sure we don't get invalid permission levels in store - cmd.Permission = 0 - } - - member.Permission = cmd.Permission - _, err = sess.Cols("permission").Where("org_id=? and team_id=? and user_id=?", cmd.OrgId, cmd.TeamId, cmd.UserId).Update(member) - - return err + return updateTeamMember(sess, cmd.OrgId, cmd.TeamId, cmd.UserId, cmd.Permission) }) } -// RemoveTeamMember removes a member from a team -func RemoveTeamMember(ctx context.Context, cmd *models.RemoveTeamMemberCommand) error { - return inTransaction(func(sess *DBSession) error { - if _, err := teamExists(cmd.OrgId, cmd.TeamId, sess); err != nil { - return err - } +func (ss *SQLStore) IsTeamMember(orgId int64, teamId int64, userId int64) (bool, error) { + var isMember bool - if cmd.ProtectLastAdmin { - _, err := isLastAdmin(sess, cmd.OrgId, cmd.TeamId, cmd.UserId) - if err != nil { - return err - } - } + err := ss.WithTransactionalDbSession(context.Background(), func(sess *DBSession) error { + var err error + isMember, err = isTeamMember(sess, orgId, teamId, userId) + return err + }) - var rawSQL = "DELETE FROM team_member WHERE org_id=? and team_id=? and user_id=?" - res, err := sess.Exec(rawSQL, cmd.OrgId, cmd.TeamId, cmd.UserId) + return isMember, err +} + +func isTeamMember(sess *DBSession, orgId int64, teamId int64, userId int64) (bool, error) { + if res, err := sess.Query("SELECT 1 FROM team_member WHERE org_id=? and team_id=? and user_id=?", orgId, teamId, userId); err != nil { + return false, err + } else if len(res) != 1 { + return false, nil + } + + return true, nil +} + +// AddOrUpdateTeamMemberHook is called from team resource permission service +// it adds user to a team or updates user permissions in a team within the given transaction session +func AddOrUpdateTeamMemberHook(sess *DBSession, userID, orgID, teamID int64, isExternal bool, permission models.PermissionType) error { + isMember, err := isTeamMember(sess, orgID, teamID, userID) + if err != nil { + return err + } + + if isMember { + err = updateTeamMember(sess, orgID, teamID, userID, permission) + } else { + err = addTeamMember(sess, orgID, teamID, userID, isExternal, permission) + } + + return err +} + +func addTeamMember(sess *DBSession, orgID, teamID, userID int64, isExternal bool, permission models.PermissionType) error { + if _, err := teamExists(orgID, teamID, sess); err != nil { + return err + } + + entity := models.TeamMember{ + OrgId: orgID, + TeamId: teamID, + UserId: userID, + External: isExternal, + Created: time.Now(), + Updated: time.Now(), + Permission: permission, + } + + _, err := sess.Insert(&entity) + return err +} + +func updateTeamMember(sess *DBSession, orgID, teamID, userID int64, permission models.PermissionType) error { + member, err := getTeamMember(sess, orgID, teamID, userID) + if err != nil { + return err + } + + if permission != models.PERMISSION_ADMIN { + permission = 0 // make sure we don't get invalid permission levels in store + + // protect the last team admin + _, err := isLastAdmin(sess, orgID, teamID, userID) if err != nil { return err } - rows, err := res.RowsAffected() - if rows == 0 { - return models.ErrTeamMemberNotFound - } + } - return err + member.Permission = permission + _, err = sess.Cols("permission").Where("org_id=? and team_id=? and user_id=?", orgID, teamID, userID).Update(member) + return err +} + +// RemoveTeamMember removes a member from a team +func (ss *SQLStore) RemoveTeamMember(ctx context.Context, cmd *models.RemoveTeamMemberCommand) error { + return inTransaction(func(sess *DBSession) error { + return removeTeamMember(sess, cmd) }) } +// RemoveTeamMemberHook is called from team resource permission service +// it removes a member from a team within the given transaction session +func RemoveTeamMemberHook(sess *DBSession, cmd *models.RemoveTeamMemberCommand) error { + return removeTeamMember(sess, cmd) +} + +func removeTeamMember(sess *DBSession, cmd *models.RemoveTeamMemberCommand) error { + if _, err := teamExists(cmd.OrgId, cmd.TeamId, sess); err != nil { + return err + } + + _, err := isLastAdmin(sess, cmd.OrgId, cmd.TeamId, cmd.UserId) + if err != nil { + return err + } + + var rawSQL = "DELETE FROM team_member WHERE org_id=? and team_id=? and user_id=?" + res, err := sess.Exec(rawSQL, cmd.OrgId, cmd.TeamId, cmd.UserId) + if err != nil { + return err + } + rows, err := res.RowsAffected() + if rows == 0 { + return models.ErrTeamMemberNotFound + } + + return err +} + func isLastAdmin(sess *DBSession, orgId int64, teamId int64, userId int64) (bool, error) { rawSQL := "SELECT user_id FROM team_member WHERE org_id=? and team_id=? and permission=?" userIds := []*int64{} @@ -399,7 +469,7 @@ func isLastAdmin(sess *DBSession, orgId int64, teamId int64, userId int64) (bool } // GetTeamMembers return a list of members for the specified team -func GetTeamMembers(ctx context.Context, query *models.GetTeamMembersQuery) error { +func (ss *SQLStore) GetTeamMembers(ctx context.Context, query *models.GetTeamMembersQuery) error { query.Result = make([]*models.TeamMemberDTO, 0) sess := x.Table("team_member") sess.Join("INNER", x.Dialect().Quote("user"), fmt.Sprintf("team_member.user_id=%s.id", x.Dialect().Quote("user"))) diff --git a/pkg/services/sqlstore/team_test.go b/pkg/services/sqlstore/team_test.go index 170e0e43248..2d8f0b6defc 100644 --- a/pkg/services/sqlstore/team_test.go +++ b/pkg/services/sqlstore/team_test.go @@ -44,7 +44,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { t.Run("Should be able to create teams and add users", func(t *testing.T) { query := &models.SearchTeamsQuery{OrgId: testOrgID, Name: "group1 name", Page: 1, Limit: 10} - err = SearchTeams(context.Background(), query) + err = sqlStore.SearchTeams(context.Background(), query) require.NoError(t, err) require.Equal(t, query.Page, 1) @@ -60,7 +60,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) q1 := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team1.Id} - err = GetTeamMembers(context.Background(), q1) + err = sqlStore.GetTeamMembers(context.Background(), q1) require.NoError(t, err) require.Equal(t, len(q1.Result), 2) require.Equal(t, q1.Result[0].TeamId, team1.Id) @@ -72,7 +72,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.Equal(t, q1.Result[1].External, true) q2 := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team1.Id, External: true} - err = GetTeamMembers(context.Background(), q2) + err = sqlStore.GetTeamMembers(context.Background(), q2) require.NoError(t, err) require.Equal(t, len(q2.Result), 1) require.Equal(t, q2.Result[0].TeamId, team1.Id) @@ -80,13 +80,13 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.Equal(t, q2.Result[0].OrgId, testOrgID) require.Equal(t, q2.Result[0].External, true) - err = SearchTeams(context.Background(), query) + err = sqlStore.SearchTeams(context.Background(), query) require.NoError(t, err) team1 = query.Result.Teams[0] require.EqualValues(t, team1.MemberCount, 2) getTeamQuery := &models.GetTeamByIdQuery{OrgId: testOrgID, Id: team1.Id} - err = GetTeamById(context.Background(), getTeamQuery) + err = sqlStore.GetTeamById(context.Background(), getTeamQuery) require.NoError(t, err) team1 = getTeamQuery.Result require.Equal(t, team1.Name, "group1 name") @@ -101,7 +101,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { userId := userIds[1] teamQuery := &models.SearchTeamsQuery{OrgId: testOrgID, Name: "group1 name", Page: 1, Limit: 10} - err = SearchTeams(context.Background(), teamQuery) + err = sqlStore.SearchTeams(context.Background(), teamQuery) require.NoError(t, err) require.Equal(t, teamQuery.Page, 1) @@ -111,7 +111,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) memberQuery := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team1.Id, External: true} - err = GetTeamMembers(context.Background(), memberQuery) + err = sqlStore.GetTeamMembers(context.Background(), memberQuery) require.NoError(t, err) require.Equal(t, len(memberQuery.Result), 1) require.Equal(t, memberQuery.Result[0].TeamId, team1.Id) @@ -127,11 +127,11 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) qBeforeUpdate := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team.Id} - err = GetTeamMembers(context.Background(), qBeforeUpdate) + err = sqlStore.GetTeamMembers(context.Background(), qBeforeUpdate) require.NoError(t, err) require.EqualValues(t, qBeforeUpdate.Result[0].Permission, 0) - err = UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ + err = sqlStore.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ UserId: userId, OrgId: testOrgID, TeamId: team.Id, @@ -141,7 +141,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) qAfterUpdate := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team.Id} - err = GetTeamMembers(context.Background(), qAfterUpdate) + err = sqlStore.GetTeamMembers(context.Background(), qAfterUpdate) require.NoError(t, err) require.Equal(t, qAfterUpdate.Result[0].Permission, models.PERMISSION_ADMIN) }) @@ -155,12 +155,12 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) qBeforeUpdate := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team.Id} - err = GetTeamMembers(context.Background(), qBeforeUpdate) + err = sqlStore.GetTeamMembers(context.Background(), qBeforeUpdate) require.NoError(t, err) require.EqualValues(t, qBeforeUpdate.Result[0].Permission, 0) invalidPermissionLevel := models.PERMISSION_EDIT - err = UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ + err = sqlStore.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ UserId: userID, OrgId: testOrgID, TeamId: team.Id, @@ -170,7 +170,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) qAfterUpdate := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team.Id} - err = GetTeamMembers(context.Background(), qAfterUpdate) + err = sqlStore.GetTeamMembers(context.Background(), qAfterUpdate) require.NoError(t, err) require.EqualValues(t, qAfterUpdate.Result[0].Permission, 0) }) @@ -178,7 +178,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { t.Run("Shouldn't be able to update a user not in the team.", func(t *testing.T) { sqlStore = InitTestDB(t) setup() - err = UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ + err = sqlStore.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{ UserId: 1, OrgId: testOrgID, TeamId: team1.Id, @@ -190,13 +190,13 @@ func TestTeamCommandsAndQueries(t *testing.T) { t.Run("Should be able to search for teams", func(t *testing.T) { query := &models.SearchTeamsQuery{OrgId: testOrgID, Query: "group", Page: 1} - err = SearchTeams(context.Background(), query) + err = sqlStore.SearchTeams(context.Background(), query) require.NoError(t, err) require.Equal(t, len(query.Result.Teams), 2) require.EqualValues(t, query.Result.TotalCount, 2) query2 := &models.SearchTeamsQuery{OrgId: testOrgID, Query: ""} - err = SearchTeams(context.Background(), query2) + err = sqlStore.SearchTeams(context.Background(), query2) require.NoError(t, err) require.Equal(t, len(query2.Result.Teams), 2) }) @@ -209,7 +209,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) query := &models.GetTeamsByUserQuery{OrgId: testOrgID, UserId: userIds[0]} - err = GetTeamsByUser(context.Background(), query) + err = sqlStore.GetTeamsByUser(context.Background(), query) require.NoError(t, err) require.Equal(t, len(query.Result), 1) require.Equal(t, query.Result[0].Name, "group2 name") @@ -220,33 +220,33 @@ func TestTeamCommandsAndQueries(t *testing.T) { err = sqlStore.AddTeamMember(userIds[0], testOrgID, team1.Id, false, 0) require.NoError(t, err) - err = RemoveTeamMember(context.Background(), &models.RemoveTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0]}) + err = sqlStore.RemoveTeamMember(context.Background(), &models.RemoveTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0]}) require.NoError(t, err) q2 := &models.GetTeamMembersQuery{OrgId: testOrgID, TeamId: team1.Id} - err = GetTeamMembers(context.Background(), q2) + err = sqlStore.GetTeamMembers(context.Background(), q2) require.NoError(t, err) require.Equal(t, len(q2.Result), 0) }) - t.Run("When ProtectLastAdmin is set to true", func(t *testing.T) { + t.Run("Should never remove the last admin of a team", func(t *testing.T) { err = sqlStore.AddTeamMember(userIds[0], testOrgID, team1.Id, false, models.PERMISSION_ADMIN) require.NoError(t, err) t.Run("A user should not be able to remove the last admin", func(t *testing.T) { - err = RemoveTeamMember(context.Background(), &models.RemoveTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0], ProtectLastAdmin: true}) + err = sqlStore.RemoveTeamMember(context.Background(), &models.RemoveTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0]}) require.Equal(t, err, models.ErrLastTeamAdmin) }) t.Run("A user should be able to remove an admin if there are other admins", func(t *testing.T) { err = sqlStore.AddTeamMember(userIds[1], testOrgID, team1.Id, false, models.PERMISSION_ADMIN) require.NoError(t, err) - err = RemoveTeamMember(context.Background(), &models.RemoveTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0], ProtectLastAdmin: true}) + err = sqlStore.RemoveTeamMember(context.Background(), &models.RemoveTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[1]}) require.NoError(t, err) }) t.Run("A user should not be able to remove the admin permission for the last admin", func(t *testing.T) { - err = UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0], Permission: 0, ProtectLastAdmin: true}) + err = sqlStore.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0], Permission: 0}) require.Error(t, err, models.ErrLastTeamAdmin) }) @@ -259,7 +259,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { err = sqlStore.AddTeamMember(userIds[1], testOrgID, team1.Id, false, models.PERMISSION_ADMIN) require.NoError(t, err) - err = UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0], Permission: 0, ProtectLastAdmin: true}) + err = sqlStore.UpdateTeamMember(context.Background(), &models.UpdateTeamMemberCommand{OrgId: testOrgID, TeamId: team1.Id, UserId: userIds[0], Permission: 0}) require.NoError(t, err) }) }) @@ -274,11 +274,11 @@ func TestTeamCommandsAndQueries(t *testing.T) { DashboardID: 1, OrgID: testOrgID, Permission: models.PERMISSION_EDIT, TeamID: groupId, }) require.NoError(t, err) - err = DeleteTeam(context.Background(), &models.DeleteTeamCommand{OrgId: testOrgID, Id: groupId}) + err = sqlStore.DeleteTeam(context.Background(), &models.DeleteTeamCommand{OrgId: testOrgID, Id: groupId}) require.NoError(t, err) query := &models.GetTeamByIdQuery{OrgId: testOrgID, Id: groupId} - err = GetTeamById(context.Background(), query) + err = sqlStore.GetTeamById(context.Background(), query) require.Equal(t, err, models.ErrTeamNotFound) permQuery := &models.GetDashboardAclInfoListQuery{DashboardID: 1, OrgID: testOrgID} @@ -323,21 +323,21 @@ func TestTeamCommandsAndQueries(t *testing.T) { require.NoError(t, err) searchQuery := &models.SearchTeamsQuery{OrgId: testOrgID, Page: 1, Limit: 10, SignedInUser: signedInUser, HiddenUsers: hiddenUsers} - err = SearchTeams(context.Background(), searchQuery) + err = sqlStore.SearchTeams(context.Background(), searchQuery) require.NoError(t, err) require.Equal(t, len(searchQuery.Result.Teams), 2) team1 := searchQuery.Result.Teams[0] require.EqualValues(t, team1.MemberCount, 2) searchQueryFilteredByUser := &models.SearchTeamsQuery{OrgId: testOrgID, Page: 1, Limit: 10, UserIdFilter: userIds[0], SignedInUser: signedInUser, HiddenUsers: hiddenUsers} - err = SearchTeams(context.Background(), searchQueryFilteredByUser) + err = sqlStore.SearchTeams(context.Background(), searchQueryFilteredByUser) require.NoError(t, err) require.Equal(t, len(searchQueryFilteredByUser.Result.Teams), 1) team1 = searchQuery.Result.Teams[0] require.EqualValues(t, team1.MemberCount, 2) getTeamQuery := &models.GetTeamByIdQuery{OrgId: testOrgID, Id: teamId, SignedInUser: signedInUser, HiddenUsers: hiddenUsers} - err = GetTeamById(context.Background(), getTeamQuery) + err = sqlStore.GetTeamById(context.Background(), getTeamQuery) require.NoError(t, err) require.EqualValues(t, getTeamQuery.Result.MemberCount, 2) }) diff --git a/pkg/services/sqlstore/transactions.go b/pkg/services/sqlstore/transactions.go index 559560082eb..56444c1ab70 100644 --- a/pkg/services/sqlstore/transactions.go +++ b/pkg/services/sqlstore/transactions.go @@ -15,7 +15,7 @@ import ( var tsclogger = log.New("sqlstore.transactions") // WithTransactionalDbSession calls the callback with a session within a transaction. -func (ss *SQLStore) WithTransactionalDbSession(ctx context.Context, callback dbTransactionFunc) error { +func (ss *SQLStore) WithTransactionalDbSession(ctx context.Context, callback DBTransactionFunc) error { return inTransactionWithRetryCtx(ctx, ss.engine, callback, 0) } @@ -30,11 +30,11 @@ func (ss *SQLStore) inTransactionWithRetry(ctx context.Context, fn func(ctx cont }, retry) } -func inTransactionWithRetry(callback dbTransactionFunc, retry int) error { +func inTransactionWithRetry(callback DBTransactionFunc, retry int) error { return inTransactionWithRetryCtx(context.Background(), x, callback, retry) } -func inTransactionWithRetryCtx(ctx context.Context, engine *xorm.Engine, callback dbTransactionFunc, retry int) error { +func inTransactionWithRetryCtx(ctx context.Context, engine *xorm.Engine, callback DBTransactionFunc, retry int) error { sess, err := startSession(ctx, engine, true) if err != nil { return err @@ -77,10 +77,10 @@ func inTransactionWithRetryCtx(ctx context.Context, engine *xorm.Engine, callbac return nil } -func inTransaction(callback dbTransactionFunc) error { +func inTransaction(callback DBTransactionFunc) error { return inTransactionWithRetry(callback, 0) } -func inTransactionCtx(ctx context.Context, callback dbTransactionFunc) error { +func inTransactionCtx(ctx context.Context, callback DBTransactionFunc) error { return inTransactionWithRetryCtx(ctx, x, callback, 0) } diff --git a/pkg/services/sqlstore/user.go b/pkg/services/sqlstore/user.go index a9b9bf78897..0e09553c1ef 100644 --- a/pkg/services/sqlstore/user.go +++ b/pkg/services/sqlstore/user.go @@ -20,7 +20,7 @@ import ( func (ss *SQLStore) addUserQueryAndCommandHandlers() { ss.Bus.AddHandler(ss.GetSignedInUserWithCacheCtx) - bus.AddHandler("sql", GetUserById) + bus.AddHandler("sql", ss.GetUserById) bus.AddHandler("sql", ss.UpdateUser) bus.AddHandler("sql", ss.ChangeUserPassword) bus.AddHandler("sql", ss.GetUserByLogin) @@ -29,7 +29,7 @@ func (ss *SQLStore) addUserQueryAndCommandHandlers() { bus.AddHandler("sql", ss.UpdateUserLastSeenAt) bus.AddHandler("sql", ss.GetUserProfile) bus.AddHandler("sql", SearchUsers) - bus.AddHandler("sql", GetUserOrgList) + bus.AddHandler("sql", ss.GetUserOrgList) bus.AddHandler("sql", DisableUser) bus.AddHandler("sql", ss.BatchDisableUsers) bus.AddHandler("sql", ss.DeleteUser) @@ -320,7 +320,7 @@ func (ss *SQLStore) CreateUser(ctx context.Context, cmd models.CreateUserCommand return user, err } -func GetUserById(ctx context.Context, query *models.GetUserByIdQuery) error { +func (ss *SQLStore) GetUserById(ctx context.Context, query *models.GetUserByIdQuery) error { return withDbSession(ctx, x, func(sess *DBSession) error { user := new(models.User) has, err := sess.ID(query.Id).Get(user) @@ -444,7 +444,7 @@ func (ss *SQLStore) UpdateUserLastSeenAt(ctx context.Context, cmd *models.Update func (ss *SQLStore) SetUsingOrg(ctx context.Context, cmd *models.SetUsingOrgCommand) error { getOrgsForUserCmd := &models.GetUserOrgListQuery{UserId: cmd.UserId} - if err := GetUserOrgList(ctx, getOrgsForUserCmd); err != nil { + if err := ss.GetUserOrgList(ctx, getOrgsForUserCmd); err != nil { return err } @@ -522,7 +522,7 @@ func (o byOrgName) Less(i, j int) bool { return o[i].Name < o[j].Name } -func GetUserOrgList(ctx context.Context, query *models.GetUserOrgListQuery) error { +func (ss *SQLStore) GetUserOrgList(ctx context.Context, query *models.GetUserOrgListQuery) error { query.Result = make([]*models.UserOrgDTO, 0) sess := x.Table("org_user") sess.Join("INNER", "org", "org_user.org_id=org.id") @@ -546,7 +546,7 @@ func (ss *SQLStore) GetSignedInUserWithCacheCtx(ctx context.Context, query *mode return nil } - err := GetSignedInUser(ctx, query) + err := ss.GetSignedInUser(ctx, query) if err != nil { return err } @@ -556,7 +556,7 @@ func (ss *SQLStore) GetSignedInUserWithCacheCtx(ctx context.Context, query *mode return nil } -func GetSignedInUser(ctx context.Context, query *models.GetSignedInUserQuery) error { +func (ss *SQLStore) GetSignedInUser(ctx context.Context, query *models.GetSignedInUserQuery) error { orgId := "u.org_id" if query.OrgId > 0 { orgId = strconv.FormatInt(query.OrgId, 10) @@ -603,7 +603,7 @@ func GetSignedInUser(ctx context.Context, query *models.GetSignedInUserQuery) er } getTeamsByUserQuery := &models.GetTeamsByUserQuery{OrgId: user.OrgId, UserId: user.UserId} - err = GetTeamsByUser(ctx, getTeamsByUserQuery) + err = ss.GetTeamsByUser(ctx, getTeamsByUserQuery) if err != nil { return err } diff --git a/pkg/services/sqlstore/user_test.go b/pkg/services/sqlstore/user_test.go index 4a18372ec89..84f787afbbc 100644 --- a/pkg/services/sqlstore/user_test.go +++ b/pkg/services/sqlstore/user_test.go @@ -27,7 +27,7 @@ func TestUserDataAccess(t *testing.T) { require.NoError(t, err) query := models.GetUserByIdQuery{Id: user.Id} - err = GetUserById(context.Background(), &query) + err = ss.GetUserById(context.Background(), &query) require.Nil(t, err) require.Equal(t, query.Result.Email, "usertest@test.com") @@ -37,7 +37,7 @@ func TestUserDataAccess(t *testing.T) { require.False(t, query.Result.IsDisabled) query = models.GetUserByIdQuery{Id: user.Id} - err = GetUserById(context.Background(), &query) + err = ss.GetUserById(context.Background(), &query) require.Nil(t, err) require.Equal(t, query.Result.Email, "usertest@test.com") @@ -60,7 +60,7 @@ func TestUserDataAccess(t *testing.T) { require.Nil(t, err) query := models.GetUserByIdQuery{Id: user.Id} - err = GetUserById(context.Background(), &query) + err = ss.GetUserById(context.Background(), &query) require.Nil(t, err) require.Equal(t, query.Result.Email, "usertest@test.com") @@ -94,7 +94,7 @@ func TestUserDataAccess(t *testing.T) { require.Nil(t, err) query := models.GetUserByIdQuery{Id: user.Id} - err = GetUserById(context.Background(), &query) + err = ss.GetUserById(context.Background(), &query) require.Nil(t, err) require.Equal(t, query.Result.Email, "usertest@test.com") @@ -469,7 +469,7 @@ func TestUserDataAccess(t *testing.T) { require.Equal(t, updatePermsError, models.ErrLastGrafanaAdmin) query := models.GetUserByIdQuery{Id: user.Id} - getUserError := GetUserById(context.Background(), &query) + getUserError := ss.GetUserById(context.Background(), &query) require.Nil(t, getUserError) require.True(t, query.Result.IsAdmin) diff --git a/pkg/services/teamguardian/database/database.go b/pkg/services/teamguardian/database/database.go index 3ffba1e0acf..cdcbedf30ae 100644 --- a/pkg/services/teamguardian/database/database.go +++ b/pkg/services/teamguardian/database/database.go @@ -7,14 +7,16 @@ import ( "github.com/grafana/grafana/pkg/services/sqlstore" ) -type TeamGuardianStoreImpl struct{} +type TeamGuardianStoreImpl struct { + sqlStore *sqlstore.SQLStore +} -func ProvideTeamGuardianStore() *TeamGuardianStoreImpl { - return &TeamGuardianStoreImpl{} +func ProvideTeamGuardianStore(sqlStore *sqlstore.SQLStore) *TeamGuardianStoreImpl { + return &TeamGuardianStoreImpl{sqlStore: sqlStore} } func (t *TeamGuardianStoreImpl) GetTeamMembers(ctx context.Context, query models.GetTeamMembersQuery) ([]*models.TeamMemberDTO, error) { - if err := sqlstore.GetTeamMembers(ctx, &query); err != nil { + if err := t.sqlStore.GetTeamMembers(ctx, &query); err != nil { return nil, err } diff --git a/pkg/services/teamguardian/manager/service_test.go b/pkg/services/teamguardian/manager/service_test.go index feb1878bec9..0ff8d3c3372 100644 --- a/pkg/services/teamguardian/manager/service_test.go +++ b/pkg/services/teamguardian/manager/service_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/teamguardian/database" "github.com/stretchr/testify/mock" @@ -32,9 +33,11 @@ func TestUpdateTeam(t *testing.T) { t.Run("Given an editor and a team he isn't a member of", func(t *testing.T) { t.Run("Should not be able to update the team", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) ctx := context.Background() store.On("GetTeamMembers", ctx, mock.Anything).Return([]*models.TeamMemberDTO{}, nil).Once() - err := teamGuardianService.CanAdmin(ctx, testTeam.OrgId, testTeam.Id, &editor) + err = teamGuardianService.CanAdmin(ctx, testTeam.OrgId, testTeam.Id, &editor) require.Equal(t, models.ErrNotAllowedToUpdateTeam, err) }) }) diff --git a/pkg/services/thumbs/crawler.go b/pkg/services/thumbs/crawler.go index 884712c81c9..110372c6909 100644 --- a/pkg/services/thumbs/crawler.go +++ b/pkg/services/thumbs/crawler.go @@ -139,9 +139,15 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rend r.mode = mode r.opts = rendering.Opts{ - OrgID: c.OrgId, - UserID: c.UserId, - OrgRole: c.OrgRole, + AuthOpts: rendering.AuthOpts{ + OrgID: c.OrgId, + UserID: c.UserId, + OrgRole: c.OrgRole, + }, + TimeoutOpts: rendering.TimeoutOpts{ + Timeout: 10 * time.Second, + RequestTimeoutMultiplier: 3, + }, Theme: theme, ConcurrentLimit: 10, } @@ -204,14 +210,11 @@ func (r *simpleCrawler) walk() { Width: 320, Height: 240, Path: panelURL, - OrgID: r.opts.OrgID, - UserID: r.opts.UserID, - ConcurrentLimit: r.opts.ConcurrentLimit, - OrgRole: r.opts.OrgRole, + AuthOpts: r.opts.AuthOpts, + TimeoutOpts: r.opts.TimeoutOpts, Theme: r.opts.Theme, - Timeout: 10 * time.Second, DeviceScaleFactor: -5, // negative numbers will render larger then scale down - }) + }, nil) if err != nil { tlog.Warn("error getting image", "err", err) r.status.Errors++ diff --git a/pkg/services/thumbs/service.go b/pkg/services/thumbs/service.go index 6bd41dd44d8..62104f2e60e 100644 --- a/pkg/services/thumbs/service.go +++ b/pkg/services/thumbs/service.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/guardian" "github.com/grafana/grafana/pkg/services/live" "github.com/grafana/grafana/pkg/services/rendering" @@ -39,8 +40,8 @@ type Service interface { CrawlerStatus(c *models.ReqContext) response.Response } -func ProvideService(cfg *setting.Cfg, renderService rendering.Service, gl *live.GrafanaLive) Service { - if !cfg.IsDashboardPreviesEnabled() { +func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive) Service { + if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) { return &dummyService{} } diff --git a/pkg/services/updatechecker/grafana.go b/pkg/services/updatechecker/grafana.go new file mode 100644 index 00000000000..777fcd07a19 --- /dev/null +++ b/pkg/services/updatechecker/grafana.go @@ -0,0 +1,115 @@ +package updatechecker + +import ( + "context" + "encoding/json" + "io/ioutil" + "net/http" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-version" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/setting" +) + +type GrafanaService struct { + hasUpdate bool + latestVersion string + + enabled bool + grafanaVersion string + httpClient http.Client + mutex sync.RWMutex + log log.Logger +} + +func ProvideGrafanaService(cfg *setting.Cfg) *GrafanaService { + return &GrafanaService{ + enabled: cfg.CheckForUpdates, + grafanaVersion: cfg.BuildVersion, + httpClient: http.Client{Timeout: 10 * time.Second}, + log: log.New("grafana.update.checker"), + } +} + +func (s *GrafanaService) IsDisabled() bool { + return !s.enabled +} + +func (s *GrafanaService) Run(ctx context.Context) error { + s.checkForUpdates() + + ticker := time.NewTicker(time.Minute * 10) + run := true + + for run { + select { + case <-ticker.C: + s.checkForUpdates() + case <-ctx.Done(): + run = false + } + } + + return ctx.Err() +} + +func (s *GrafanaService) checkForUpdates() { + resp, err := s.httpClient.Get("https://raw.githubusercontent.com/grafana/grafana/main/latest.json") + if err != nil { + s.log.Debug("Failed to get latest.json repo from github.com", "error", err) + return + } + defer func() { + if err := resp.Body.Close(); err != nil { + s.log.Warn("Failed to close response body", "err", err) + } + }() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + s.log.Debug("Update check failed, reading response from github.com", "error", err) + return + } + + type latestJSON struct { + Stable string `json:"stable"` + Testing string `json:"testing"` + } + var latest latestJSON + err = json.Unmarshal(body, &latest) + if err != nil { + s.log.Debug("Failed to unmarshal latest.json", "error", err) + return + } + + s.mutex.Lock() + defer s.mutex.Unlock() + if strings.Contains(s.grafanaVersion, "-") { + s.latestVersion = latest.Testing + s.hasUpdate = !strings.HasPrefix(s.grafanaVersion, latest.Testing) + } else { + s.latestVersion = latest.Stable + s.hasUpdate = latest.Stable != s.grafanaVersion + } + + currVersion, err1 := version.NewVersion(s.grafanaVersion) + latestVersion, err2 := version.NewVersion(s.latestVersion) + if err1 == nil && err2 == nil { + s.hasUpdate = currVersion.LessThan(latestVersion) + } +} + +func (s *GrafanaService) UpdateAvailable() bool { + s.mutex.RLock() + defer s.mutex.RUnlock() + return s.hasUpdate +} + +func (s *GrafanaService) LatestVersion() string { + s.mutex.RLock() + defer s.mutex.RUnlock() + return s.latestVersion +} diff --git a/pkg/services/updatechecker/grafana_update_checker.go b/pkg/services/updatechecker/grafana_update_checker.go deleted file mode 100644 index fd2fa3fac82..00000000000 --- a/pkg/services/updatechecker/grafana_update_checker.go +++ /dev/null @@ -1,120 +0,0 @@ -package updatechecker - -import ( - "context" - "encoding/json" - "io/ioutil" - "net/http" - "strings" - "sync" - "time" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/setting" - "github.com/hashicorp/go-version" -) - -var ( - httpClient = http.Client{Timeout: 10 * time.Second} - logger = log.New("update.checker") -) - -type latestJSON struct { - Stable string `json:"stable"` - Testing string `json:"testing"` -} - -type Service struct { - cfg *setting.Cfg - - hasUpdate bool - latestVersion string - mutex sync.RWMutex -} - -func ProvideService(cfg *setting.Cfg) *Service { - s := newUpdateChecker(cfg) - - return s -} - -func newUpdateChecker(cfg *setting.Cfg) *Service { - return &Service{ - cfg: cfg, - } -} - -func (s *Service) IsDisabled() bool { - return !s.cfg.CheckForUpdates -} - -func (s *Service) Run(ctx context.Context) error { - s.checkForUpdates() - - ticker := time.NewTicker(time.Minute * 10) - run := true - - for run { - select { - case <-ticker.C: - s.checkForUpdates() - case <-ctx.Done(): - run = false - } - } - - return ctx.Err() -} - -func (s *Service) checkForUpdates() { - resp, err := httpClient.Get("https://raw.githubusercontent.com/grafana/grafana/main/latest.json") - if err != nil { - logger.Debug("Failed to get latest.json repo from github.com", "error", err) - return - } - defer func() { - if err := resp.Body.Close(); err != nil { - logger.Warn("Failed to close response body", "err", err) - } - }() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logger.Debug("Update check failed, reading response from github.com", "error", err) - return - } - - var latest latestJSON - err = json.Unmarshal(body, &latest) - if err != nil { - logger.Debug("Failed to unmarshal latest.json", "error", err) - return - } - - s.mutex.Lock() - defer s.mutex.Unlock() - if strings.Contains(s.cfg.BuildVersion, "-") { - s.latestVersion = latest.Testing - s.hasUpdate = !strings.HasPrefix(s.cfg.BuildVersion, latest.Testing) - } else { - s.latestVersion = latest.Stable - s.hasUpdate = latest.Stable != s.cfg.BuildVersion - } - - currVersion, err1 := version.NewVersion(s.cfg.BuildVersion) - latestVersion, err2 := version.NewVersion(s.latestVersion) - if err1 == nil && err2 == nil { - s.hasUpdate = currVersion.LessThan(latestVersion) - } -} - -func (s *Service) GrafanaUpdateAvailable() bool { - s.mutex.RLock() - defer s.mutex.RUnlock() - return s.hasUpdate -} - -func (s *Service) LatestGrafanaVersion() string { - s.mutex.RLock() - defer s.mutex.RUnlock() - return s.latestVersion -} diff --git a/pkg/services/updatechecker/plugins.go b/pkg/services/updatechecker/plugins.go new file mode 100644 index 00000000000..618597acda6 --- /dev/null +++ b/pkg/services/updatechecker/plugins.go @@ -0,0 +1,167 @@ +package updatechecker + +import ( + "context" + "encoding/json" + "io/ioutil" + "net/http" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-version" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" +) + +type PluginsService struct { + availableUpdates map[string]string + + enabled bool + grafanaVersion string + pluginStore plugins.Store + httpClient httpClient + mutex sync.RWMutex + log log.Logger +} + +func ProvidePluginsService(cfg *setting.Cfg, pluginStore plugins.Store) *PluginsService { + return &PluginsService{ + enabled: cfg.CheckForUpdates, + grafanaVersion: cfg.BuildVersion, + httpClient: &http.Client{Timeout: 10 * time.Second}, + log: log.New("plugins.update.checker"), + pluginStore: pluginStore, + availableUpdates: make(map[string]string), + } +} + +type httpClient interface { + Get(url string) (resp *http.Response, err error) +} + +func (s *PluginsService) IsDisabled() bool { + return !s.enabled +} + +func (s *PluginsService) Run(ctx context.Context) error { + s.checkForUpdates(ctx) + + ticker := time.NewTicker(time.Minute * 10) + run := true + + for run { + select { + case <-ticker.C: + s.checkForUpdates(ctx) + case <-ctx.Done(): + run = false + } + } + + return ctx.Err() +} + +func (s *PluginsService) HasUpdate(ctx context.Context, pluginID string) (string, bool) { + s.mutex.RLock() + updateVers, updateAvailable := s.availableUpdates[pluginID] + s.mutex.RUnlock() + if updateAvailable { + // check if plugin has already been updated since the last invocation of `checkForUpdates` + plugin, exists := s.pluginStore.Plugin(ctx, pluginID) + if !exists { + return "", false + } + + if canUpdate(plugin.Info.Version, updateVers) { + return updateVers, true + } + } + + return "", false +} + +func (s *PluginsService) checkForUpdates(ctx context.Context) { + s.log.Debug("Checking for updates") + + localPlugins := s.pluginsEligibleForVersionCheck(ctx) + resp, err := s.httpClient.Get("https://grafana.com/api/plugins/versioncheck?slugIn=" + + s.pluginIDsCSV(localPlugins) + "&grafanaVersion=" + s.grafanaVersion) + if err != nil { + s.log.Debug("Failed to get plugins repo from grafana.com", "error", err.Error()) + return + } + defer func() { + if err := resp.Body.Close(); err != nil { + s.log.Warn("Failed to close response body", "err", err) + } + }() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + s.log.Debug("Update check failed, reading response from grafana.com", "error", err.Error()) + return + } + + type gcomPlugin struct { + Slug string `json:"slug"` + Version string `json:"version"` + } + var gcomPlugins []gcomPlugin + err = json.Unmarshal(body, &gcomPlugins) + if err != nil { + s.log.Debug("Failed to unmarshal plugin repo, reading response from grafana.com", "error", err.Error()) + return + } + + availableUpdates := map[string]string{} + for _, gcomP := range gcomPlugins { + if localP, exists := localPlugins[gcomP.Slug]; exists { + if canUpdate(localP.Info.Version, gcomP.Version) { + availableUpdates[localP.ID] = gcomP.Version + } + } + } + + if len(availableUpdates) > 0 { + s.mutex.Lock() + s.availableUpdates = availableUpdates + s.mutex.Unlock() + } +} + +func canUpdate(v1, v2 string) bool { + ver1, err1 := version.NewVersion(v1) + if err1 != nil { + return false + } + ver2, err2 := version.NewVersion(v2) + if err2 != nil { + return false + } + + return ver1.LessThan(ver2) +} + +func (s *PluginsService) pluginIDsCSV(m map[string]plugins.PluginDTO) string { + var ids []string + for pluginID := range m { + ids = append(ids, pluginID) + } + + return strings.Join(ids, ",") +} + +func (s *PluginsService) pluginsEligibleForVersionCheck(ctx context.Context) map[string]plugins.PluginDTO { + result := make(map[string]plugins.PluginDTO) + for _, p := range s.pluginStore.Plugins(ctx) { + if p.IsCorePlugin() { + continue + } + result[p.ID] = p + } + + return result +} diff --git a/pkg/services/updatechecker/plugins_test.go b/pkg/services/updatechecker/plugins_test.go new file mode 100644 index 00000000000..1f89033f083 --- /dev/null +++ b/pkg/services/updatechecker/plugins_test.go @@ -0,0 +1,225 @@ +package updatechecker + +import ( + "context" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/plugins" +) + +func TestPluginUpdateChecker_HasUpdate(t *testing.T) { + t.Run("update is available", func(t *testing.T) { + svc := PluginsService{ + availableUpdates: map[string]string{ + "test-ds": "1.0.0", + }, + pluginStore: fakePluginStore{ + plugins: map[string]plugins.PluginDTO{ + "test-ds": { + JSONData: plugins.JSONData{ + Info: plugins.Info{Version: "0.9.0"}, + }, + }, + }, + }, + } + + update, exists := svc.HasUpdate(context.Background(), "test-ds") + require.True(t, exists) + require.Equal(t, "1.0.0", update) + }) + + t.Run("update is not available", func(t *testing.T) { + svc := PluginsService{ + availableUpdates: map[string]string{ + "test-panel": "0.9.0", + "test-app": "0.0.1", + }, + pluginStore: fakePluginStore{ + plugins: map[string]plugins.PluginDTO{ + "test-ds": { + JSONData: plugins.JSONData{ + Info: plugins.Info{Version: "0.9.0"}, + }, + }, + "test-panel": { + JSONData: plugins.JSONData{ + Info: plugins.Info{Version: "0.9.0"}, + }, + }, + "test-app": { + JSONData: plugins.JSONData{ + Info: plugins.Info{Version: "0.9.0"}, + }, + }, + }, + }, + } + + update, exists := svc.HasUpdate(context.Background(), "test-ds") + require.False(t, exists) + require.Empty(t, update) + + update, exists = svc.HasUpdate(context.Background(), "test-panel") + require.False(t, exists) + require.Empty(t, update) + + update, exists = svc.HasUpdate(context.Background(), "test-app") + require.False(t, exists) + require.Empty(t, update) + }) + + t.Run("update is available but plugin is not in store", func(t *testing.T) { + svc := PluginsService{ + availableUpdates: map[string]string{ + "test-panel": "0.9.0", + }, + pluginStore: fakePluginStore{ + plugins: map[string]plugins.PluginDTO{ + "test-ds": { + JSONData: plugins.JSONData{ + Info: plugins.Info{Version: "1.0.0"}, + }, + }, + }, + }, + } + + update, exists := svc.HasUpdate(context.Background(), "test-panel") + require.False(t, exists) + require.Empty(t, update) + + update, exists = svc.HasUpdate(context.Background(), "test-ds") + require.False(t, exists) + require.Empty(t, update) + }) +} + +func TestPluginUpdateChecker_checkForUpdates(t *testing.T) { + t.Run("update is available", func(t *testing.T) { + jsonResp := `[ + { + "slug": "test-ds", + "version": "1.0.12" + }, + { + "slug": "test-panel", + "version": "2.5.7" + }, + { + "slug": "test-core-panel", + "version": "1.0.0" + } + ]` + + svc := PluginsService{ + availableUpdates: map[string]string{ + "test-app": "1.0.0", + }, + pluginStore: fakePluginStore{ + plugins: map[string]plugins.PluginDTO{ + "test-ds": { + JSONData: plugins.JSONData{ + ID: "test-ds", + Info: plugins.Info{Version: "0.9.0"}, + }, + }, + "test-app": { + JSONData: plugins.JSONData{ + ID: "test-app", + Info: plugins.Info{Version: "0.5.0"}, + }, + }, + "test-panel": { + JSONData: plugins.JSONData{ + ID: "test-panel", + Info: plugins.Info{Version: "2.5.7"}, + }, + }, + "test-core-panel": { + Class: plugins.Core, + JSONData: plugins.JSONData{ + ID: "test-core-panel", + Info: plugins.Info{Version: "0.0.1"}, + }, + }, + }, + }, + httpClient: &fakeHTTPClient{ + fakeResp: jsonResp, + }, + log: &fakeLogger{}, + } + + svc.checkForUpdates(context.Background()) + + require.Equal(t, 1, len(svc.availableUpdates)) + + require.Equal(t, "1.0.12", svc.availableUpdates["test-ds"]) + update, exists := svc.HasUpdate(context.Background(), "test-ds") + require.True(t, exists) + require.Equal(t, "1.0.12", update) + + require.Empty(t, svc.availableUpdates["test-app"]) + update, exists = svc.HasUpdate(context.Background(), "test-app") + require.False(t, exists) + require.Empty(t, update) + + require.Empty(t, svc.availableUpdates["test-panel"]) + update, exists = svc.HasUpdate(context.Background(), "test-panel") + require.False(t, exists) + require.Empty(t, update) + + require.Empty(t, svc.availableUpdates["test-core-panel"]) + }) +} + +type fakeHTTPClient struct { + fakeResp string + + requestURL string +} + +func (c *fakeHTTPClient) Get(url string) (*http.Response, error) { + c.requestURL = url + + resp := &http.Response{ + Body: ioutil.NopCloser(strings.NewReader(c.fakeResp)), + } + + return resp, nil +} + +type fakePluginStore struct { + plugins.Store + + plugins map[string]plugins.PluginDTO +} + +func (pr fakePluginStore) Plugin(_ context.Context, pluginID string) (plugins.PluginDTO, bool) { + p, exists := pr.plugins[pluginID] + + return p, exists +} + +func (pr fakePluginStore) Plugins(_ context.Context, _ ...plugins.Type) []plugins.PluginDTO { + var result []plugins.PluginDTO + for _, p := range pr.plugins { + result = append(result, p) + } + return result +} + +type fakeLogger struct { + log.Logger +} + +func (l *fakeLogger) Debug(_ string, _ ...interface{}) {} + +func (l *fakeLogger) Warn(_ string, _ ...interface{}) {} diff --git a/pkg/setting/provider.go b/pkg/setting/provider.go index ee72b162801..459c8e8e42b 100644 --- a/pkg/setting/provider.go +++ b/pkg/setting/provider.go @@ -132,7 +132,7 @@ func (o *OSSImpl) Section(section string) Section { func (OSSImpl) RegisterReloadHandler(string, ReloadHandler) {} func (o OSSImpl) IsFeatureToggleEnabled(name string) bool { - return o.Cfg.FeatureToggles[name] + return o.Cfg.IsFeatureToggleEnabled(name) } type keyValImpl struct { diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 1b5db0e641a..713bfc31d3f 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -319,6 +319,7 @@ type Cfg struct { JWTAuthCacheTTL time.Duration JWTAuthKeyFile string JWTAuthJWKSetFile string + JWTAuthAutoSignUp bool // Dataproxy SendUserHeader bool @@ -341,8 +342,10 @@ type Cfg struct { ApiKeyMaxSecondsToLive int64 - // Use to enable new features which may still be in alpha/beta stage. - FeatureToggles map[string]bool + // Check if a feature toggle is enabled + // @deprecated + IsFeatureToggleEnabled func(key string) bool // filled in dynamically + AnonymousEnabled bool AnonymousOrgName string AnonymousOrgRole string @@ -426,41 +429,9 @@ type Cfg struct { // Unified Alerting UnifiedAlerting UnifiedAlertingSettings -} -// IsLiveConfigEnabled returns true if live should be able to save configs to SQL tables -func (cfg Cfg) IsLiveConfigEnabled() bool { - return cfg.FeatureToggles["live-config"] -} - -// IsLiveConfigEnabled returns true if live should be able to save configs to SQL tables -func (cfg Cfg) IsDashboardPreviesEnabled() bool { - return cfg.FeatureToggles["dashboardPreviews"] -} - -// IsTrimDefaultsEnabled returns whether the standalone trim dashboard default feature is enabled. -func (cfg Cfg) IsTrimDefaultsEnabled() bool { - return cfg.FeatureToggles["trimDefaults"] -} - -// IsDatabaseMetricsEnabled returns whether the database instrumentation feature is enabled. -func (cfg Cfg) IsDatabaseMetricsEnabled() bool { - return cfg.FeatureToggles["database_metrics"] -} - -// IsHTTPRequestHistogramDisabled returns whether the request historgrams is disabled. -// This feature toggle will be removed in Grafana 8.x but gives the operator -// some graceperiod to update all the monitoring tools. -func (cfg Cfg) IsHTTPRequestHistogramDisabled() bool { - return cfg.FeatureToggles["disable_http_request_histogram"] -} - -func (cfg Cfg) IsNewNavigationEnabled() bool { - return cfg.FeatureToggles["newNavigation"] -} - -func (cfg Cfg) IsServiceAccountEnabled() bool { - return cfg.FeatureToggles["service-accounts"] + // Query history + QueryHistoryEnabled bool } type CommandLineArgs struct { @@ -504,7 +475,6 @@ func RedactedValue(key, value string) string { "ACCOUNT_KEY", "ENCRYPTION_KEY", "VAULT_TOKEN", - "AWSKMS_.*_TOKEN", } { if match, err := regexp.MatchString(pattern, uppercased); match && err == nil { return RedactedPassword @@ -973,6 +943,9 @@ func (cfg *Cfg) Load(args CommandLineArgs) error { explore := iniFile.Section("explore") ExploreEnabled = explore.Key("enabled").MustBool(true) + queryHistory := iniFile.Section("query_history") + cfg.QueryHistoryEnabled = queryHistory.Key("enabled").MustBool(false) + panelsSection := iniFile.Section("panels") cfg.DisableSanitizeHtml = panelsSection.Key("disable_sanitize_html").MustBool(false) @@ -1308,6 +1281,7 @@ func readAuthSettings(iniFile *ini.File, cfg *Cfg) (err error) { cfg.JWTAuthCacheTTL = authJWT.Key("cache_ttl").MustDuration(time.Minute * 60) cfg.JWTAuthKeyFile = valueAsString(authJWT, "key_file", "") cfg.JWTAuthJWKSetFile = valueAsString(authJWT, "jwk_set_file", "") + cfg.JWTAuthAutoSignUp = authJWT.Key("auto_sign_up").MustBool(false) authProxy := iniFile.Section("auth.proxy") AuthProxyEnabled = authProxy.Key("enabled").MustBool(false) diff --git a/pkg/setting/setting_feature_toggles.go b/pkg/setting/setting_feature_toggles.go index 4c103b31030..abef83d7c48 100644 --- a/pkg/setting/setting_feature_toggles.go +++ b/pkg/setting/setting_feature_toggles.go @@ -3,42 +3,23 @@ package setting import ( "strconv" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/grafana/grafana/pkg/util" "gopkg.in/ini.v1" ) -var ( - featureToggleInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "feature_toggles_info", - Help: "info metric that exposes what feature toggles are enabled or not", - Namespace: "grafana", - }, []string{"name"}) - - defaultFeatureToggles = map[string]bool{ - "recordedQueries": false, - "accesscontrol": false, - "service-accounts": false, - "httpclientprovider_azure_auth": false, - } -) - +// @deprecated -- should use `featuremgmt.FeatureToggles` func (cfg *Cfg) readFeatureToggles(iniFile *ini.File) error { - toggles, err := overrideDefaultWithConfiguration(iniFile, defaultFeatureToggles) + section := iniFile.Section("feature_toggles") + toggles, err := ReadFeatureTogglesFromInitFile(section) if err != nil { return err } - - cfg.FeatureToggles = toggles - + cfg.IsFeatureToggleEnabled = func(key string) bool { return toggles[key] } return nil } -func overrideDefaultWithConfiguration(iniFile *ini.File, featureToggles map[string]bool) (map[string]bool, error) { - // Read and populate feature toggles list - featureTogglesSection := iniFile.Section("feature_toggles") +func ReadFeatureTogglesFromInitFile(featureTogglesSection *ini.Section) (map[string]bool, error) { + featureToggles := make(map[string]bool, 10) // parse the comma separated list in `enable`. featuresTogglesStr := valueAsString(featureTogglesSection, "enable", "") @@ -60,15 +41,5 @@ func overrideDefaultWithConfiguration(iniFile *ini.File, featureToggles map[stri featureToggles[v.Name()] = b } - - // track if feature toggles are enabled or not using an info metric - for k, v := range featureToggles { - if v { - featureToggleInfo.WithLabelValues(k).Set(1) - } else { - featureToggleInfo.WithLabelValues(k).Set(0) - } - } - return featureToggles, nil } diff --git a/pkg/setting/setting_feature_toggles_test.go b/pkg/setting/setting_feature_toggles_test.go index 92ec0f03095..b0c3730bcad 100644 --- a/pkg/setting/setting_feature_toggles_test.go +++ b/pkg/setting/setting_feature_toggles_test.go @@ -14,7 +14,6 @@ func TestFeatureToggles(t *testing.T) { conf map[string]string err error expectedToggles map[string]bool - defaultToggles map[string]bool }{ { name: "can parse feature toggles passed in the `enable` array", @@ -58,18 +57,6 @@ func TestFeatureToggles(t *testing.T) { expectedToggles: map[string]bool{}, err: strconv.ErrSyntax, }, - { - name: "should override default feature toggles", - defaultToggles: map[string]bool{ - "feature1": true, - }, - conf: map[string]string{ - "feature1": "false", - }, - expectedToggles: map[string]bool{ - "feature1": false, - }, - }, } for _, tc := range testCases { @@ -81,12 +68,7 @@ func TestFeatureToggles(t *testing.T) { require.ErrorIs(t, err, nil) } - dt := map[string]bool{} - if len(tc.defaultToggles) > 0 { - dt = tc.defaultToggles - } - - featureToggles, err := overrideDefaultWithConfiguration(f, dt) + featureToggles, err := ReadFeatureTogglesFromInitFile(toggles) require.ErrorIs(t, err, tc.err) if err == nil { diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go index d1332086ec3..b4a74ff4f47 100644 --- a/pkg/setting/setting_unified_alerting.go +++ b/pkg/setting/setting_unified_alerting.go @@ -79,7 +79,7 @@ func (cfg *Cfg) readUnifiedAlertingEnabledSetting(section *ini.Section) (*bool, // the unified alerting is not enabled by default. First, check the feature flag if err != nil { // TODO: Remove in Grafana v9 - if cfg.FeatureToggles["ngalert"] { + if cfg.IsFeatureToggleEnabled("ngalert") { cfg.Logger.Warn("ngalert feature flag is deprecated: use unified alerting enabled setting instead") enabled = true // feature flag overrides the legacy alerting setting. diff --git a/pkg/setting/setting_unified_alerting_test.go b/pkg/setting/setting_unified_alerting_test.go index d890700032c..63c9d790238 100644 --- a/pkg/setting/setting_unified_alerting_test.go +++ b/pkg/setting/setting_unified_alerting_test.go @@ -143,6 +143,7 @@ func TestUnifiedAlertingSettings(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { f := ini.Empty() cfg := NewCfg() + cfg.IsFeatureToggleEnabled = func(key string) bool { return false } unifiedAlertingSec, err := f.NewSection("unified_alerting") require.NoError(t, err) for k, v := range tc.unifiedAlertingOptions { diff --git a/pkg/tests/api/alerting/api_admin_configuration_test.go b/pkg/tests/api/alerting/api_admin_configuration_test.go index d6d5d1e3e00..24a32a45a39 100644 --- a/pkg/tests/api/alerting/api_admin_configuration_test.go +++ b/pkg/tests/api/alerting/api_admin_configuration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -21,6 +22,8 @@ import ( func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) { const disableOrgID int64 = 3 + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -64,7 +67,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) { resp := getRequest(t, alertsURL, http.StatusNotFound) // nolint b, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) - require.JSONEq(t, string(b), "{\"message\": \"API error\",\"error\": \"no admin configuration available\"}") + require.JSONEq(t, string(b), "{\"message\": \"no admin configuration available\",\"error\": \"no admin configuration available\"}") } // Now, lets re-set external Alertmanagers for main organisation. diff --git a/pkg/tests/api/alerting/api_alertmanager_configuration_test.go b/pkg/tests/api/alerting/api_alertmanager_configuration_test.go index 96628bc1bae..59e8e1c2895 100644 --- a/pkg/tests/api/alerting/api_alertmanager_configuration_test.go +++ b/pkg/tests/api/alerting/api_alertmanager_configuration_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/tests/testinfra" @@ -17,6 +18,9 @@ import ( ) func TestAlertmanagerConfigurationIsTransactional(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -88,7 +92,7 @@ func TestAlertmanagerConfigurationIsTransactional(t *testing.T) { } ` resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint - require.JSONEq(t, `{"message": "API error","error":"failed to save and apply Alertmanager configuration: failed to build integration map: the receiver is invalid: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API"}`, getBody(t, resp.Body)) + require.JSONEq(t, `{"message": "failed to save and apply Alertmanager configuration: failed to build integration map: the receiver is invalid: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API","error":"failed to save and apply Alertmanager configuration: failed to build integration map: the receiver is invalid: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API"}`, getBody(t, resp.Body)) resp = getRequest(t, alertConfigURL, http.StatusOK) // nolint require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body)) @@ -127,6 +131,9 @@ func TestAlertmanagerConfigurationIsTransactional(t *testing.T) { } func TestAlertmanagerConfigurationPersistSecrets(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -208,7 +215,7 @@ func TestAlertmanagerConfigurationPersistSecrets(t *testing.T) { ` resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint - require.JSONEq(t, `{"message": "API error","error": "unknown receiver: invalid"}`, getBody(t, resp.Body)) + require.JSONEq(t, `{"message": "unknown receiver: invalid","error": "unknown receiver: invalid"}`, getBody(t, resp.Body)) } // The secure settings must be present diff --git a/pkg/tests/api/alerting/api_alertmanager_test.go b/pkg/tests/api/alerting/api_alertmanager_test.go index da8b2f1a3ca..9a631b5539f 100644 --- a/pkg/tests/api/alerting/api_alertmanager_test.go +++ b/pkg/tests/api/alerting/api_alertmanager_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" @@ -28,6 +29,9 @@ import ( ) func TestAMConfigAccess(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -96,7 +100,7 @@ func TestAMConfigAccess(t *testing.T) { desc: "viewer request should fail", url: "http://viewer:viewer@%s/api/alertmanager/grafana/config/api/v1/alerts", expStatus: http.StatusForbidden, - expBody: `{"message": "API error","error": "permission denied"}`, + expBody: `{"message": "permission denied","error": "permission denied"}`, }, { desc: "editor request should succeed", @@ -166,7 +170,7 @@ func TestAMConfigAccess(t *testing.T) { desc: "viewer request should fail", url: "http://viewer:viewer@%s/api/alertmanager/grafana/config/api/v1/alerts", expStatus: http.StatusForbidden, - expBody: `{"message": "API error","error": "permission denied"}`, + expBody: `{"message": "permission denied","error": "permission denied"}`, }, { desc: "editor request should succeed", @@ -229,7 +233,7 @@ func TestAMConfigAccess(t *testing.T) { desc: "viewer request should fail", url: "http://viewer:viewer@%s/api/alertmanager/grafana/api/v2/silences", expStatus: http.StatusForbidden, - expBody: `{"message": "API error","error": "permission denied"}`, + expBody: `{"message": "permission denied","error": "permission denied"}`, }, { desc: "editor request should succeed", @@ -314,7 +318,7 @@ func TestAMConfigAccess(t *testing.T) { }) var silences apimodels.GettableSilences - err := json.Unmarshal(blob, &silences) + err = json.Unmarshal(blob, &silences) require.NoError(t, err) assert.Len(t, silences, 2) silenceIDs := make([]string, 0, len(silences)) @@ -335,7 +339,7 @@ func TestAMConfigAccess(t *testing.T) { desc: "viewer request should fail", url: "http://viewer:viewer@%s/api/alertmanager/grafana/api/v2/silence/%s", expStatus: http.StatusForbidden, - expBody: `{"message": "API error","error": "permission denied"}`, + expBody: `{"message": "permission denied","error": "permission denied"}`, }, { desc: "editor request should succeed", @@ -387,6 +391,9 @@ func TestAMConfigAccess(t *testing.T) { } func TestAlertAndGroupsQuery(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -552,6 +559,8 @@ func TestAlertAndGroupsQuery(t *testing.T) { } func TestRulerAccess(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, @@ -566,7 +575,7 @@ func TestRulerAccess(t *testing.T) { store.Bus = bus.GetBus() // Create the namespace we'll save our alerts to. - _, err := createFolder(t, store, 0, "default") + _, err = createFolder(t, store, 0, "default") require.NoError(t, err) // Create a users to make authenticated requests @@ -603,7 +612,7 @@ func TestRulerAccess(t *testing.T) { desc: "viewer request should fail", url: "http://viewer:viewer@%s/api/ruler/grafana/api/v1/rules/default", expStatus: http.StatusForbidden, - expectedResponse: `{"message": "API error","error":"user does not have permissions to edit the namespace: user does not have permissions to edit the namespace"}`, + expectedResponse: `{"message": "user does not have permissions to edit the namespace: user does not have permissions to edit the namespace","error":"user does not have permissions to edit the namespace: user does not have permissions to edit the namespace"}`, }, { desc: "editor request should succeed", @@ -679,6 +688,8 @@ func TestRulerAccess(t *testing.T) { } func TestDeleteFolderWithRules(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, @@ -839,6 +850,8 @@ func TestDeleteFolderWithRules(t *testing.T) { } func TestAlertRuleCRUD(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, @@ -858,7 +871,7 @@ func TestAlertRuleCRUD(t *testing.T) { }) // Create the namespace we'll save our alerts to. - _, err := createFolder(t, store, 0, "default") + _, err = createFolder(t, store, 0, "default") require.NoError(t, err) interval, err := model.ParseDuration("1m") @@ -890,7 +903,7 @@ func TestAlertRuleCRUD(t *testing.T) { Data: []ngmodels.AlertQuery{}, }, }, - expectedResponse: `{"message": "API error","error":"failed to update rule group: invalid alert rule: no queries or expressions are found"}`, + expectedResponse: `{"message": "failed to update rule group: invalid alert rule: no queries or expressions are found","error":"failed to update rule group: invalid alert rule: no queries or expressions are found"}`, }, { desc: "alert rule with empty title", @@ -920,7 +933,7 @@ func TestAlertRuleCRUD(t *testing.T) { }, }, }, - expectedResponse: `{"message": "API error","error":"failed to update rule group: invalid alert rule: title is empty"}`, + expectedResponse: `{"message": "failed to update rule group: invalid alert rule: title is empty","error":"failed to update rule group: invalid alert rule: title is empty"}`, }, { desc: "alert rule with too long name", @@ -950,7 +963,7 @@ func TestAlertRuleCRUD(t *testing.T) { }, }, }, - expectedResponse: `{"message": "API error","error":"failed to update rule group: invalid alert rule: name length should not be greater than 190"}`, + expectedResponse: `{"message": "failed to update rule group: invalid alert rule: name length should not be greater than 190","error":"failed to update rule group: invalid alert rule: name length should not be greater than 190"}`, }, { desc: "alert rule with too long rulegroup", @@ -980,7 +993,7 @@ func TestAlertRuleCRUD(t *testing.T) { }, }, }, - expectedResponse: `{"message": "API error","error":"failed to update rule group: invalid alert rule: rule group name length should not be greater than 190"}`, + expectedResponse: `{"message": "failed to update rule group: invalid alert rule: rule group name length should not be greater than 190","error":"failed to update rule group: invalid alert rule: rule group name length should not be greater than 190"}`, }, { desc: "alert rule with invalid interval", @@ -1011,7 +1024,7 @@ func TestAlertRuleCRUD(t *testing.T) { }, }, }, - expectedResponse: `{"message": "API error","error":"failed to update rule group: invalid alert rule: interval (1s) should be non-zero and divided exactly by scheduler interval: 10s"}`, + expectedResponse: `{"message": "failed to update rule group: invalid alert rule: interval (1s) should be non-zero and divided exactly by scheduler interval: 10s","error":"failed to update rule group: invalid alert rule: interval (1s) should be non-zero and divided exactly by scheduler interval: 10s"}`, }, { desc: "alert rule with unknown datasource", @@ -1041,7 +1054,7 @@ func TestAlertRuleCRUD(t *testing.T) { }, }, }, - expectedResponse: `{"message": "API error","error":"failed to validate alert rule \"AlwaysFiring\": invalid query A: data source not found: unknown"}`, + expectedResponse: `{"message": "failed to validate alert rule \"AlwaysFiring\": invalid query A: data source not found: unknown","error":"failed to validate alert rule \"AlwaysFiring\": invalid query A: data source not found: unknown"}`, }, { desc: "alert rule with invalid condition", @@ -1071,7 +1084,7 @@ func TestAlertRuleCRUD(t *testing.T) { }, }, }, - expectedResponse: `{"message": "API error","error":"failed to validate alert rule \"AlwaysFiring\": condition B not found in any query or expression: it should be one of: [A]"}`, + expectedResponse: `{"message": "failed to validate alert rule \"AlwaysFiring\": condition B not found in any query or expression: it should be one of: [A]","error":"failed to validate alert rule \"AlwaysFiring\": condition B not found in any query or expression: it should be one of: [A]"}`, }, } @@ -1361,7 +1374,7 @@ func TestAlertRuleCRUD(t *testing.T) { require.NoError(t, err) assert.Equal(t, http.StatusNotFound, resp.StatusCode) - require.JSONEq(t, `{"message": "API error","error":"failed to update rule group: failed to get alert rule unknown: could not find alert rule"}`, string(b)) + require.JSONEq(t, `{"message": "failed to update rule group: failed to get alert rule unknown: could not find alert rule","error":"failed to update rule group: failed to get alert rule unknown: could not find alert rule"}`, string(b)) // let's make sure that rule definitions are not affected by the failed POST request. u = fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr) @@ -1480,7 +1493,7 @@ func TestAlertRuleCRUD(t *testing.T) { require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - require.JSONEq(t, fmt.Sprintf(`{"message": "API error","error":"failed to validate alert rule \"AlwaysAlerting\": conflicting UID \"%s\" found"}`, ruleUID), string(b)) + require.JSONEq(t, fmt.Sprintf(`{"message": "failed to validate alert rule \"AlwaysAlerting\": conflicting UID \"%s\" found","error":"failed to validate alert rule \"AlwaysAlerting\": conflicting UID \"%s\" found"}`, ruleUID, ruleUID), string(b)) // let's make sure that rule definitions are not affected by the failed POST request. u = fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr) @@ -1885,7 +1898,7 @@ func TestAlertRuleCRUD(t *testing.T) { require.NoError(t, err) require.Equal(t, http.StatusNotFound, resp.StatusCode) - require.JSONEq(t, `{"message": "API error","error":"failed to delete rule group: rule group not found under this namespace"}`, string(b)) + require.JSONEq(t, `{"message": "failed to delete rule group: rule group not found under this namespace","error":"failed to delete rule group: rule group not found under this namespace"}`, string(b)) }) t.Run("succeed if the rule group name does exist", func(t *testing.T) { @@ -1969,6 +1982,8 @@ func TestAlertmanagerStatus(t *testing.T) { } func TestQuota(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, @@ -1982,7 +1997,7 @@ func TestQuota(t *testing.T) { store.Bus = bus.GetBus() // Create the namespace we'll save our alerts to. - _, err := createFolder(t, store, 0, "default") + _, err = createFolder(t, store, 0, "default") require.NoError(t, err) // Create a user to make authenticated requests @@ -2093,7 +2108,7 @@ func TestQuota(t *testing.T) { b, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, resp.StatusCode) - require.JSONEq(t, `{"message": "API error","error":"quota reached"}`, string(b)) + require.JSONEq(t, `{"message": "quota reached","error":"quota reached"}`, string(b)) }) t.Run("when quota limit exceed updating existing rule should succeed", func(t *testing.T) { @@ -2212,6 +2227,8 @@ func TestQuota(t *testing.T) { } func TestEval(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, @@ -2231,7 +2248,7 @@ func TestEval(t *testing.T) { }) // Create the namespace we'll save our alerts to. - _, err := createFolder(t, store, 0, "default") + _, err = createFolder(t, store, 0, "default") require.NoError(t, err) // test eval conditions @@ -2388,7 +2405,7 @@ func TestEval(t *testing.T) { } `, expectedStatusCode: http.StatusBadRequest, - expectedResponse: `{"message": "API error","error":"invalid condition: condition B not found in any query or expression: it should be one of: [A]"}`, + expectedResponse: `{"message": "invalid condition: condition B not found in any query or expression: it should be one of: [A]","error":"invalid condition: condition B not found in any query or expression: it should be one of: [A]"}`, }, { desc: "unknown query datasource", @@ -2413,7 +2430,7 @@ func TestEval(t *testing.T) { } `, expectedStatusCode: http.StatusBadRequest, - expectedResponse: `{"message": "API error","error":"invalid condition: invalid query A: data source not found: unknown"}`, + expectedResponse: `{"message": "invalid condition: invalid query A: data source not found: unknown","error":"invalid condition: invalid query A: data source not found: unknown"}`, }, } @@ -2569,7 +2586,7 @@ func TestEval(t *testing.T) { } `, expectedStatusCode: http.StatusBadRequest, - expectedResponse: `{"message": "API error","error":"invalid queries or expressions: invalid query A: data source not found: unknown"}`, + expectedResponse: `{"message": "invalid queries or expressions: invalid query A: data source not found: unknown","error":"invalid queries or expressions: invalid query A: data source not found: unknown"}`, }, } diff --git a/pkg/tests/api/alerting/api_available_channel_test.go b/pkg/tests/api/alerting/api_available_channel_test.go index 003984ca1b1..dbd01ad238b 100644 --- a/pkg/tests/api/alerting/api_available_channel_test.go +++ b/pkg/tests/api/alerting/api_available_channel_test.go @@ -9,11 +9,15 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tests/testinfra" ) func TestAvailableChannels(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, diff --git a/pkg/tests/api/alerting/api_notification_channel_test.go b/pkg/tests/api/alerting/api_notification_channel_test.go index faa36379668..e519221ca75 100644 --- a/pkg/tests/api/alerting/api_notification_channel_test.go +++ b/pkg/tests/api/alerting/api_notification_channel_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -32,16 +33,18 @@ import ( func TestTestReceivers(t *testing.T) { t.Run("assert no receivers returns 400 Bad Request", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", @@ -50,8 +53,8 @@ func TestTestReceivers(t *testing.T) { testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) // nolint resp := postRequest(t, testReceiversURL, `{ - "receivers": [] -}`, http.StatusBadRequest) + "receivers": [] + }`, http.StatusBadRequest) t.Cleanup(func() { err := resp.Body.Close() require.NoError(t, err) @@ -63,47 +66,45 @@ func TestTestReceivers(t *testing.T) { }) t.Run("assert working receiver returns OK", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", }) - oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") mockEmails := &mockEmailHandler{} - bus.AddHandler("", mockEmails.sendEmailCommandHandlerSync) - t.Cleanup(func() { - bus.AddHandler("", oldEmailBus) - }) + env.NotificationService.EmailHandlerSync = mockEmails.sendEmailCommandHandlerSync testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) // nolint resp := postRequest(t, testReceiversURL, `{ - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "uid": "", - "name": "receiver-1", - "type": "email", - "disableResolveMessage": false, - "settings": { - "addresses":"example@email.com" - }, - "secureFields": {} - } - ] - }] -}`, http.StatusOK) + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "uid": "", + "name": "receiver-1", + "type": "email", + "disableResolveMessage": false, + "settings": { + "addresses":"example@email.com" + }, + "secureFields": {} + } + ] + }] + }`, http.StatusOK) t.Cleanup(func() { err := resp.Body.Close() require.NoError(t, err) @@ -118,28 +119,28 @@ func TestTestReceivers(t *testing.T) { require.Len(t, result.Receivers[0].Configs, 1) expectedJSON := fmt.Sprintf(`{ - "alert": { - "annotations": { - "summary": "Notification test", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - }, - "labels": { - "alertname": "TestAlert", - "instance": "Grafana" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-1", - "uid": "%s", - "status": "ok" + "alert": { + "annotations": { + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "alertname": "TestAlert", + "instance": "Grafana" } - ] - }], - "notified_at": "%s" -}`, + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-1", + "uid": "%s", + "status": "ok" + } + ] + }], + "notified_at": "%s" + }`, result.Receivers[0].Configs[0].UID, result.NotifiedAt.Format(time.RFC3339Nano)) require.JSONEq(t, expectedJSON, string(b)) @@ -149,45 +150,43 @@ func TestTestReceivers(t *testing.T) { }) t.Run("assert invalid receiver returns 400 Bad Request", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", }) - oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") mockEmails := &mockEmailHandler{} - bus.AddHandler("", mockEmails.sendEmailCommandHandlerSync) - t.Cleanup(func() { - bus.AddHandler("", oldEmailBus) - }) + env.NotificationService.EmailHandlerSync = mockEmails.sendEmailCommandHandlerSync testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) // nolint resp := postRequest(t, testReceiversURL, `{ - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "uid": "", - "name": "receiver-1", - "type": "email", - "disableResolveMessage": false, - "settings": {}, - "secureFields": {} - } - ] - }] -}`, http.StatusBadRequest) + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "uid": "", + "name": "receiver-1", + "type": "email", + "disableResolveMessage": false, + "settings": {}, + "secureFields": {} + } + ] + }] + }`, http.StatusBadRequest) t.Cleanup(func() { require.NoError(t, resp.Body.Close()) }) @@ -201,77 +200,75 @@ func TestTestReceivers(t *testing.T) { require.Len(t, result.Receivers[0].Configs, 1) expectedJSON := fmt.Sprintf(`{ - "alert": { - "annotations": { - "summary": "Notification test", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - }, - "labels": { - "alertname": "TestAlert", - "instance": "Grafana" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-1", - "uid": "%s", - "status": "failed", - "error": "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings" + "alert": { + "annotations": { + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "alertname": "TestAlert", + "instance": "Grafana" } - ] - }], - "notified_at": "%s" -}`, + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-1", + "uid": "%s", + "status": "failed", + "error": "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings" + } + ] + }], + "notified_at": "%s" + }`, result.Receivers[0].Configs[0].UID, result.NotifiedAt.Format(time.RFC3339Nano)) require.JSONEq(t, expectedJSON, string(b)) }) t.Run("assert timed out receiver returns 408 Request Timeout", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", }) - oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") mockEmails := &mockEmailHandlerWithTimeout{ timeout: 5 * time.Second, } - bus.AddHandler("", mockEmails.sendEmailCommandHandlerSync) - t.Cleanup(func() { - bus.AddHandler("", oldEmailBus) - }) + env.NotificationService.EmailHandlerSync = mockEmails.sendEmailCommandHandlerSync testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) req, err := http.NewRequest(http.MethodPost, testReceiversURL, strings.NewReader(`{ - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "uid": "", - "name": "receiver-1", - "type": "email", - "disableResolveMessage": false, - "settings": { - "addresses":"example@email.com" - }, - "secureFields": {} - } - ] - }] -}`)) + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "uid": "", + "name": "receiver-1", + "type": "email", + "disableResolveMessage": false, + "settings": { + "addresses":"example@email.com" + }, + "secureFields": {} + } + ] + }] + }`)) require.NoError(t, err) req.Header.Set("Content-Type", "application/json") req.Header.Set("Request-Timeout", "1") @@ -292,89 +289,87 @@ func TestTestReceivers(t *testing.T) { require.Len(t, result.Receivers[0].Configs, 1) expectedJSON := fmt.Sprintf(`{ - "alert": { - "annotations": { - "summary": "Notification test", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - }, - "labels": { - "alertname": "TestAlert", - "instance": "Grafana" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-1", - "uid": "%s", - "status": "failed", - "error": "the receiver timed out: context deadline exceeded" + "alert": { + "annotations": { + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "alertname": "TestAlert", + "instance": "Grafana" } - ] - }], - "notified_at": "%s" -}`, + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-1", + "uid": "%s", + "status": "failed", + "error": "the receiver timed out: context deadline exceeded" + } + ] + }], + "notified_at": "%s" + }`, result.Receivers[0].Configs[0].UID, result.NotifiedAt.Format(time.RFC3339Nano)) require.JSONEq(t, expectedJSON, string(b)) }) t.Run("assert multiple different errors returns 207 Multi Status", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", }) - oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") mockEmails := &mockEmailHandlerWithTimeout{ timeout: 5 * time.Second, } - bus.AddHandler("", mockEmails.sendEmailCommandHandlerSync) - t.Cleanup(func() { - bus.AddHandler("", oldEmailBus) - }) + env.NotificationService.EmailHandlerSync = mockEmails.sendEmailCommandHandlerSync testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) req, err := http.NewRequest(http.MethodPost, testReceiversURL, strings.NewReader(`{ - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "uid": "", - "name": "receiver-1", - "type": "email", - "disableResolveMessage": false, - "settings": {}, - "secureFields": {} - } - ] - }, { - "name":"receiver-2", - "grafana_managed_receiver_configs": [ - { - "uid": "", - "name": "receiver-2", - "type": "email", - "disableResolveMessage": false, - "settings": { - "addresses":"example@email.com" - }, - "secureFields": {} - } - ] - }] -}`)) + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "uid": "", + "name": "receiver-1", + "type": "email", + "disableResolveMessage": false, + "settings": {}, + "secureFields": {} + } + ] + }, { + "name":"receiver-2", + "grafana_managed_receiver_configs": [ + { + "uid": "", + "name": "receiver-2", + "type": "email", + "disableResolveMessage": false, + "settings": { + "addresses":"example@email.com" + }, + "secureFields": {} + } + ] + }] + }`)) require.NoError(t, err) req.Header.Set("Content-Type", "application/json") req.Header.Set("Request-Timeout", "1") @@ -396,39 +391,39 @@ func TestTestReceivers(t *testing.T) { require.Len(t, result.Receivers[1].Configs, 1) expectedJSON := fmt.Sprintf(`{ - "alert": { - "annotations": { - "summary": "Notification test", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + "alert": { + "annotations": { + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "alertname": "TestAlert", + "instance": "Grafana" + } }, - "labels": { - "alertname": "TestAlert", - "instance": "Grafana" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-1", - "uid": "%s", - "status": "failed", - "error": "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings" - } - ] - }, { - "name":"receiver-2", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-2", - "uid": "%s", - "status": "failed", - "error": "the receiver timed out: context deadline exceeded" - } - ] - }], - "notified_at": "%s" -}`, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-1", + "uid": "%s", + "status": "failed", + "error": "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings" + } + ] + }, { + "name":"receiver-2", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-2", + "uid": "%s", + "status": "failed", + "error": "the receiver timed out: context deadline exceeded" + } + ] + }], + "notified_at": "%s" + }`, result.Receivers[0].Configs[0].UID, result.Receivers[1].Configs[0].UID, result.NotifiedAt.Format(time.RFC3339Nano)) @@ -438,56 +433,54 @@ func TestTestReceivers(t *testing.T) { func TestTestReceiversAlertCustomization(t *testing.T) { t.Run("assert custom annotations and labels are sent", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", }) - oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") mockEmails := &mockEmailHandler{} - bus.AddHandler("", mockEmails.sendEmailCommandHandlerSync) - t.Cleanup(func() { - bus.AddHandler("", oldEmailBus) - }) + env.NotificationService.EmailHandlerSync = mockEmails.sendEmailCommandHandlerSync testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) // nolint resp := postRequest(t, testReceiversURL, `{ - "alert": { - "annotations": { - "annotation1": "value1", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - }, - "labels": { - "label1": "value1" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "uid":"", - "name":"receiver-1", - "type":"email", - "disableResolveMessage":false, - "settings":{ - "addresses":"example@email.com" - }, - "secureFields":{} + "alert": { + "annotations": { + "annotation1": "value1", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "label1": "value1" } - ] - }] -}`, http.StatusOK) + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "uid":"", + "name":"receiver-1", + "type":"email", + "disableResolveMessage":false, + "settings":{ + "addresses":"example@email.com" + }, + "secureFields":{} + } + ] + }] + }`, http.StatusOK) t.Cleanup(func() { err := resp.Body.Close() require.NoError(t, err) @@ -502,30 +495,30 @@ func TestTestReceiversAlertCustomization(t *testing.T) { require.Len(t, result.Receivers[0].Configs, 1) expectedJSON := fmt.Sprintf(`{ - "alert": { - "annotations": { - "annotation1": "value1", - "summary": "Notification test", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - }, - "labels": { - "alertname": "TestAlert", - "instance": "Grafana", - "label1": "value1" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-1", - "uid": "%s", - "status": "ok" + "alert": { + "annotations": { + "annotation1": "value1", + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "alertname": "TestAlert", + "instance": "Grafana", + "label1": "value1" } - ] - }], - "notified_at": "%s" -}`, + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-1", + "uid": "%s", + "status": "ok" + } + ] + }], + "notified_at": "%s" + }`, result.Receivers[0].Configs[0].UID, result.NotifiedAt.Format(time.RFC3339Nano)) require.JSONEq(t, expectedJSON, string(b)) @@ -535,53 +528,51 @@ func TestTestReceiversAlertCustomization(t *testing.T) { }) t.Run("assert custom annotations can replace default annotations", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", }) - oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") mockEmails := &mockEmailHandler{} - bus.AddHandler("", mockEmails.sendEmailCommandHandlerSync) - t.Cleanup(func() { - bus.AddHandler("", oldEmailBus) - }) + env.NotificationService.EmailHandlerSync = mockEmails.sendEmailCommandHandlerSync testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) // nolint resp := postRequest(t, testReceiversURL, `{ - "alert": { - "annotations": { - "summary": "This is a custom annotation", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "uid":"", - "name":"receiver-1", - "type":"email", - "disableResolveMessage":false, - "settings":{ - "addresses":"example@email.com" - }, - "secureFields":{} + "alert": { + "annotations": { + "summary": "This is a custom annotation", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" } - ] - }] -}`, http.StatusOK) + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "uid":"", + "name":"receiver-1", + "type":"email", + "disableResolveMessage":false, + "settings":{ + "addresses":"example@email.com" + }, + "secureFields":{} + } + ] + }] + }`, http.StatusOK) t.Cleanup(func() { err := resp.Body.Close() require.NoError(t, err) @@ -596,28 +587,28 @@ func TestTestReceiversAlertCustomization(t *testing.T) { require.Len(t, result.Receivers[0].Configs, 1) expectedJSON := fmt.Sprintf(`{ - "alert": { - "annotations": { - "summary": "This is a custom annotation", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - }, - "labels": { - "alertname": "TestAlert", - "instance": "Grafana" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-1", - "uid": "%s", - "status": "ok" + "alert": { + "annotations": { + "summary": "This is a custom annotation", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "alertname": "TestAlert", + "instance": "Grafana" } - ] - }], - "notified_at": "%s" -}`, + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-1", + "uid": "%s", + "status": "ok" + } + ] + }], + "notified_at": "%s" + }`, result.Receivers[0].Configs[0].UID, result.NotifiedAt.Format(time.RFC3339Nano)) require.JSONEq(t, expectedJSON, string(b)) @@ -627,52 +618,50 @@ func TestTestReceiversAlertCustomization(t *testing.T) { }) t.Run("assert custom labels can replace default label", func(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, }) - grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) - store.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() - createUser(t, store, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Login: "grafana", Password: "password", }) - oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") mockEmails := &mockEmailHandler{} - bus.AddHandler("", mockEmails.sendEmailCommandHandlerSync) - t.Cleanup(func() { - bus.AddHandler("", oldEmailBus) - }) + env.NotificationService.EmailHandlerSync = mockEmails.sendEmailCommandHandlerSync testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr) // nolint resp := postRequest(t, testReceiversURL, `{ - "alert": { - "labels": { - "alertname": "This is a custom label" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "uid":"", - "name":"receiver-1", - "type":"email", - "disableResolveMessage":false, - "settings":{ - "addresses":"example@email.com" - }, - "secureFields":{} + "alert": { + "labels": { + "alertname": "This is a custom label" } - ] - }] -}`, http.StatusOK) + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "uid":"", + "name":"receiver-1", + "type":"email", + "disableResolveMessage":false, + "settings":{ + "addresses":"example@email.com" + }, + "secureFields":{} + } + ] + }] + }`, http.StatusOK) t.Cleanup(func() { err := resp.Body.Close() require.NoError(t, err) @@ -687,28 +676,28 @@ func TestTestReceiversAlertCustomization(t *testing.T) { require.Len(t, result.Receivers[0].Configs, 1) expectedJSON := fmt.Sprintf(`{ - "alert": { - "annotations": { - "summary": "Notification test", - "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" - }, - "labels": { - "alertname": "This is a custom label", - "instance": "Grafana" - } - }, - "receivers": [{ - "name":"receiver-1", - "grafana_managed_receiver_configs": [ - { - "name": "receiver-1", - "uid": "%s", - "status": "ok" + "alert": { + "annotations": { + "summary": "Notification test", + "__value_string__": "[ metric='foo' labels={instance=bar} value=10 ]" + }, + "labels": { + "alertname": "This is a custom label", + "instance": "Grafana" } - ] - }], - "notified_at": "%s" -}`, + }, + "receivers": [{ + "name":"receiver-1", + "grafana_managed_receiver_configs": [ + { + "name": "receiver-1", + "uid": "%s", + "status": "ok" + } + ] + }], + "notified_at": "%s" + }`, result.Receivers[0].Configs[0].UID, result.NotifiedAt.Format(time.RFC3339Nano)) require.JSONEq(t, expectedJSON, string(b)) @@ -719,14 +708,17 @@ func TestTestReceiversAlertCustomization(t *testing.T) { } func TestNotificationChannels(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, DisableAnonymous: true, }) - grafanaListedAddr, s := testinfra.StartGrafana(t, dir, path) - s.Bus = bus.GetBus() + grafanaListedAddr, env := testinfra.StartGrafanaEnv(t, dir, path) + env.SQLStore.Bus = bus.GetBus() mockChannel := newMockNotificationChannel(t, grafanaListedAddr) amConfig := getAlertmanagerConfig(mockChannel.server.Addr) @@ -737,13 +729,11 @@ func TestNotificationChannels(t *testing.T) { channels.TelegramAPIURL, channels.PushoverEndpoint, channels.GetBoundary, channels.LineNotifyURL, channels.ThreemaGwBaseURL originalTemplate := channels.DefaultTemplateString - originalEmailBus := bus.GetHandlerCtx("SendEmailCommandSync") t.Cleanup(func() { channels.SlackAPIEndpoint, channels.PagerdutyEventAPIURL, channels.TelegramAPIURL, channels.PushoverEndpoint, channels.GetBoundary, channels.LineNotifyURL, channels.ThreemaGwBaseURL = os, opa, ot, opu, ogb, ol, oth channels.DefaultTemplateString = originalTemplate - bus.AddHandler("", originalEmailBus) }) channels.DefaultTemplateString = channels.TemplateForTestsString channels.SlackAPIEndpoint = fmt.Sprintf("http://%s/slack_recvX/slack_testX", mockChannel.server.Addr) @@ -753,10 +743,19 @@ func TestNotificationChannels(t *testing.T) { channels.LineNotifyURL = fmt.Sprintf("http://%s/line_recv/line_test", mockChannel.server.Addr) channels.ThreemaGwBaseURL = fmt.Sprintf("http://%s/threema_recv/threema_test", mockChannel.server.Addr) channels.GetBoundary = func() string { return "abcd" } - bus.AddHandler("", mockEmail.sendEmailCommandHandlerSync) + + env.NotificationService.EmailHandlerSync = mockEmail.sendEmailCommandHandlerSync + // As we are using a NotificationService mock here, but he test expects real NotificationService - + // we try to issue a real POST request here + env.NotificationService.WebhookHandler = func(_ context.Context, cmd *models.SendWebhookSync) error { + if res, err := http.Post(cmd.Url, "", strings.NewReader(cmd.Body)); err == nil { + _ = res.Body.Close() + } + return nil + } // Create a user to make authenticated requests - createUser(t, s, models.CreateUserCommand{ + createUser(t, env.SQLStore, models.CreateUserCommand{ DefaultOrgRole: string(models.ROLE_EDITOR), Password: "password", Login: "grafana", @@ -773,7 +772,7 @@ func TestNotificationChannels(t *testing.T) { { // Create the namespace we'll save our alerts to. - _, err := createFolder(t, s, 0, "default") + _, err := createFolder(t, env.SQLStore, 0, "default") require.NoError(t, err) // Post the alertmanager config. diff --git a/pkg/tests/api/alerting/api_prometheus_test.go b/pkg/tests/api/alerting/api_prometheus_test.go index 8a9c5085ebe..29a29f0551a 100644 --- a/pkg/tests/api/alerting/api_prometheus_test.go +++ b/pkg/tests/api/alerting/api_prometheus_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -21,6 +22,9 @@ import ( ) func TestPrometheusRules(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -32,7 +36,7 @@ func TestPrometheusRules(t *testing.T) { store.Bus = bus.GetBus() // Create the namespace under default organisation (orgID = 1) where we'll save our alerts to. - _, err := createFolder(t, store, 0, "default") + _, err = createFolder(t, store, 0, "default") require.NoError(t, err) // Create a user to make authenticated requests @@ -202,7 +206,7 @@ func TestPrometheusRules(t *testing.T) { require.NoError(t, err) assert.Equal(t, 400, resp.StatusCode) - require.JSONEq(t, `{"message": "API error","error":"failed to update rule group: invalid alert rule: cannot have Panel ID without a Dashboard UID"}`, string(b)) + require.JSONEq(t, `{"message": "failed to update rule group: invalid alert rule: cannot have Panel ID without a Dashboard UID","error":"failed to update rule group: invalid alert rule: cannot have Panel ID without a Dashboard UID"}`, string(b)) } // Now, let's see how this looks like. @@ -320,6 +324,8 @@ func TestPrometheusRules(t *testing.T) { } func TestPrometheusRulesFilterByDashboard(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ EnableFeatureToggles: []string{"ngalert"}, DisableAnonymous: true, @@ -594,7 +600,7 @@ func TestPrometheusRulesFilterByDashboard(t *testing.T) { require.Equal(t, http.StatusBadRequest, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) - require.JSONEq(t, `{"message": "API error","error":"invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax"}`, string(b)) + require.JSONEq(t, `{"message": "invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax","error":"invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax"}`, string(b)) } // Now, let's check a panel_id without dashboard_uid returns a 400 Bad Request response @@ -610,11 +616,14 @@ func TestPrometheusRulesFilterByDashboard(t *testing.T) { require.Equal(t, http.StatusBadRequest, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) - require.JSONEq(t, `{"message": "API error","error":"panel_id must be set with dashboard_uid"}`, string(b)) + require.JSONEq(t, `{"message": "panel_id must be set with dashboard_uid","error":"panel_id must be set with dashboard_uid"}`, string(b)) } } func TestPrometheusRulesPermissions(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -633,7 +642,7 @@ func TestPrometheusRulesPermissions(t *testing.T) { }) // Create a namespace under default organisation (orgID = 1) where we'll save some alerts. - _, err := createFolder(t, store, 0, "folder1") + _, err = createFolder(t, store, 0, "folder1") require.NoError(t, err) // Create another namespace under default organisation (orgID = 1) where we'll save some alerts. diff --git a/pkg/tests/api/alerting/api_ruler_test.go b/pkg/tests/api/alerting/api_ruler_test.go index e8f6ee78595..d4710d8360e 100644 --- a/pkg/tests/api/alerting/api_ruler_test.go +++ b/pkg/tests/api/alerting/api_ruler_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/models" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -22,6 +23,9 @@ import ( func TestAlertRulePermissions(t *testing.T) { // Setup Grafana and its Database + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, EnableUnifiedAlerting: true, @@ -40,7 +44,7 @@ func TestAlertRulePermissions(t *testing.T) { }) // Create the namespace we'll save our alerts to. - _, err := createFolder(t, store, 0, "folder1") + _, err = createFolder(t, store, 0, "folder1") require.NoError(t, err) _, err = createFolder(t, store, 0, "folder2") @@ -324,6 +328,8 @@ func createRule(t *testing.T, grafanaListedAddr string, folder string, user, pas } func TestAlertRuleConflictingTitle(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) // Setup Grafana and its Database dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ DisableLegacyAlerting: true, @@ -338,7 +344,7 @@ func TestAlertRuleConflictingTitle(t *testing.T) { store.Bus = bus.GetBus() // Create the namespace we'll save our alerts to. - _, err := createFolder(t, store, 0, "folder1") + _, err = createFolder(t, store, 0, "folder1") require.NoError(t, err) _, err = createFolder(t, store, 0, "folder2") require.NoError(t, err) @@ -422,7 +428,7 @@ func TestAlertRuleConflictingTitle(t *testing.T) { require.NoError(t, err) assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) - require.JSONEq(t, `{"message": "API error","error":"failed to update rule group: a conflicting alert rule is found: rule title under the same organisation and folder should be unique"}`, string(b)) + require.JSONEq(t, `{"message": "failed to update rule group: a conflicting alert rule is found: rule title under the same organisation and folder should be unique","error":"failed to update rule group: a conflicting alert rule is found: rule title under the same organisation and folder should be unique"}`, string(b)) }) t.Run("trying to create alert with same title under another folder should succeed", func(t *testing.T) { @@ -448,6 +454,9 @@ func TestAlertRuleConflictingTitle(t *testing.T) { } func TestRulerRulesFilterByDashboard(t *testing.T) { + _, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ EnableFeatureToggles: []string{"ngalert"}, DisableAnonymous: true, @@ -773,7 +782,7 @@ func TestRulerRulesFilterByDashboard(t *testing.T) { require.Equal(t, http.StatusBadRequest, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) - require.JSONEq(t, `{"message": "API error","error":"invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax"}`, string(b)) + require.JSONEq(t, `{"message": "invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax","error":"invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax"}`, string(b)) } // Now, let's check a panel_id without dashboard_uid returns a 400 Bad Request response @@ -789,6 +798,6 @@ func TestRulerRulesFilterByDashboard(t *testing.T) { require.Equal(t, http.StatusBadRequest, resp.StatusCode) b, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) - require.JSONEq(t, `{"message": "API error","error":"panel_id must be set with dashboard_uid"}`, string(b)) + require.JSONEq(t, `{"message": "panel_id must be set with dashboard_uid","error":"panel_id must be set with dashboard_uid"}`, string(b)) } } diff --git a/pkg/tests/api/dashboards/api_dashboards_test.go b/pkg/tests/api/dashboards/api_dashboards_test.go index 9d0aa581e42..7570afe0c5a 100644 --- a/pkg/tests/api/dashboards/api_dashboards_test.go +++ b/pkg/tests/api/dashboards/api_dashboards_test.go @@ -12,10 +12,10 @@ import ( "path/filepath" "testing" - "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/dashboardimport" "github.com/grafana/grafana/pkg/services/search" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/tests/testinfra" @@ -32,6 +32,7 @@ func TestDashboardQuota(t *testing.T) { EnableQuota: true, DashboardOrgQuota: &dashboardQuota, }) + grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) // Create user createUser(t, store, models.CreateUserCommand{ @@ -45,7 +46,7 @@ func TestDashboardQuota(t *testing.T) { dashboardDataOne, err := simplejson.NewJson([]byte(`{"title":"just testing"}`)) require.NoError(t, err) buf1 := &bytes.Buffer{} - err = json.NewEncoder(buf1).Encode(dtos.ImportDashboardCommand{ + err = json.NewEncoder(buf1).Encode(dashboardimport.ImportDashboardRequest{ Dashboard: dashboardDataOne, }) require.NoError(t, err) @@ -70,7 +71,7 @@ func TestDashboardQuota(t *testing.T) { dashboardDataOne, err := simplejson.NewJson([]byte(`{"title":"just testing"}`)) require.NoError(t, err) buf1 := &bytes.Buffer{} - err = json.NewEncoder(buf1).Encode(dtos.ImportDashboardCommand{ + err = json.NewEncoder(buf1).Encode(dashboardimport.ImportDashboardRequest{ Dashboard: dashboardDataOne, }) require.NoError(t, err) diff --git a/pkg/tests/api/metrics/api_metrics_test.go b/pkg/tests/api/metrics/api_metrics_test.go index afe2b5a3a0f..02d6cea7383 100644 --- a/pkg/tests/api/metrics/api_metrics_test.go +++ b/pkg/tests/api/metrics/api_metrics_test.go @@ -30,6 +30,7 @@ import ( func TestQueryCloudWatchMetrics(t *testing.T) { grafDir, cfgPath := testinfra.CreateGrafDir(t) + addr, sqlStore := testinfra.StartGrafana(t, grafDir, cfgPath) setUpDatabase(t, sqlStore) diff --git a/pkg/tests/testinfra/testinfra.go b/pkg/tests/testinfra/testinfra.go index 9d255d40269..dec6141fecc 100644 --- a/pkg/tests/testinfra/testinfra.go +++ b/pkg/tests/testinfra/testinfra.go @@ -27,6 +27,11 @@ import ( // StartGrafana starts a Grafana server. // The server address is returned. func StartGrafana(t *testing.T, grafDir, cfgPath string) (string, *sqlstore.SQLStore) { + addr, env := StartGrafanaEnv(t, grafDir, cfgPath) + return addr, env.SQLStore +} + +func StartGrafanaEnv(t *testing.T, grafDir, cfgPath string) (string, *server.TestEnv) { t.Helper() ctx := context.Background() @@ -65,7 +70,7 @@ func StartGrafana(t *testing.T, grafDir, cfgPath string) (string, *sqlstore.SQLS t.Logf("Grafana is listening on %s", addr) - return addr, env.SQLStore + return addr, env } // SetUpDatabase sets up the Grafana database. diff --git a/pkg/tests/web/index_view_test.go b/pkg/tests/web/index_view_test.go index 724f50a26ca..72cf4376b30 100644 --- a/pkg/tests/web/index_view_test.go +++ b/pkg/tests/web/index_view_test.go @@ -18,6 +18,7 @@ func TestIndexView(t *testing.T) { grafDir, cfgPath := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ EnableCSP: true, }) + addr, _ := testinfra.StartGrafana(t, grafDir, cfgPath) // nolint:bodyclose diff --git a/pkg/tsdb/azuremonitor/applicationinsights-datasource.go b/pkg/tsdb/azuremonitor/applicationinsights-datasource.go index 20c1c3a2bb2..26905bfe81e 100644 --- a/pkg/tsdb/azuremonitor/applicationinsights-datasource.go +++ b/pkg/tsdb/azuremonitor/applicationinsights-datasource.go @@ -14,8 +14,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/util/errutil" - "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" "golang.org/x/net/context/ctxhttp" ) @@ -48,7 +49,8 @@ func (e *ApplicationInsightsDatasource) resourceRequest(rw http.ResponseWriter, } func (e *ApplicationInsightsDatasource) executeTimeSeriesQuery(ctx context.Context, - originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string) (*backend.QueryDataResponse, error) { + originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() queries, err := e.buildQueries(originalQueries) @@ -57,7 +59,7 @@ func (e *ApplicationInsightsDatasource) executeTimeSeriesQuery(ctx context.Conte } for _, query := range queries { - queryRes, err := e.executeQuery(ctx, query, dsInfo, client, url) + queryRes, err := e.executeQuery(ctx, query, dsInfo, client, url, tracer) if err != nil { return nil, err } @@ -128,7 +130,7 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []backend.DataQuery return applicationInsightsQueries, nil } -func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query *ApplicationInsightsQuery, dsInfo datasourceInfo, client *http.Client, url string) ( +func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query *ApplicationInsightsQuery, dsInfo datasourceInfo, client *http.Client, url string, tracer tracing.Tracer) ( backend.DataResponse, error) { dataResponse := backend.DataResponse{} @@ -141,23 +143,16 @@ func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query req.URL.Path = path.Join(req.URL.Path, query.ApiURL) req.URL.RawQuery = query.Params.Encode() - span, ctx := opentracing.StartSpanFromContext(ctx, "application insights query") - span.SetTag("target", query.Target) - span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)) - span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)) - span.SetTag("datasource_id", dsInfo.DatasourceID) - span.SetTag("org_id", dsInfo.OrgID) + ctx, span := tracer.Start(ctx, "application insights query") + span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target)) + span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID)) + span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID)) - defer span.Finish() + defer span.End() - err = opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)) - - if err != nil { - azlog.Warn("failed to inject global tracer") - } + tracer.Inject(ctx, req.Header, span) azlog.Debug("ApplicationInsights", "Request URL", req.URL.String()) res, err := ctxhttp.Do(ctx, client, req) diff --git a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go index 84c22371ae9..964ec995c78 100644 --- a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go +++ b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go @@ -16,8 +16,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/util/errutil" - "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" "golang.org/x/net/context/ctxhttp" ) @@ -46,7 +47,8 @@ func (e *AzureLogAnalyticsDatasource) resourceRequest(rw http.ResponseWriter, re // 1. build the AzureMonitor url and querystring for each query // 2. executes each query by calling the Azure Monitor API // 3. parses the responses for each query into data frames -func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string) (*backend.QueryDataResponse, error) { +func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() queries, err := e.buildQueries(originalQueries, dsInfo) @@ -55,7 +57,7 @@ func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context } for _, query := range queries { - result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url) + result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url, tracer) } return result, nil @@ -125,7 +127,8 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []backend.DataQuery, return azureLogAnalyticsQueries, nil } -func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *AzureLogAnalyticsQuery, dsInfo datasourceInfo, client *http.Client, url string) backend.DataResponse { +func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *AzureLogAnalyticsQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) backend.DataResponse { dataResponse := backend.DataResponse{} dataResponseErrorWithExecuted := func(err error) backend.DataResponse { @@ -155,21 +158,16 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A req.URL.Path = path.Join(req.URL.Path, query.URL) req.URL.RawQuery = query.Params.Encode() - span, ctx := opentracing.StartSpanFromContext(ctx, "azure log analytics query") - span.SetTag("target", query.Target) - span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)) - span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)) - span.SetTag("datasource_id", dsInfo.DatasourceID) - span.SetTag("org_id", dsInfo.OrgID) + ctx, span := tracer.Start(ctx, "azure log analytics query") + span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target)) + span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID)) + span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID)) - defer span.Finish() + defer span.End() - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)); err != nil { - return dataResponseErrorWithExecuted(err) - } + tracer.Inject(ctx, req.Header, span) azlog.Debug("AzureLogAnalytics", "Request ApiURL", req.URL.String()) res, err := ctxhttp.Do(ctx, client, req) diff --git a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go index 13d90e81913..26001270be5 100644 --- a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go +++ b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go @@ -11,6 +11,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/stretchr/testify/require" ) @@ -228,7 +229,9 @@ func Test_executeQueryErrorWithDifferentLogAnalyticsCreds(t *testing.T) { Params: url.Values{}, TimeRange: backend.TimeRange{}, } - res := ds.executeQuery(ctx, query, dsInfo, &http.Client{}, dsInfo.Services[azureLogAnalytics].URL) + tracer, err := tracing.InitializeTracerForTest() + require.NoError(t, err) + res := ds.executeQuery(ctx, query, dsInfo, &http.Client{}, dsInfo.Services[azureLogAnalytics].URL, tracer) if res.Error == nil { t.Fatal("expecting an error") } diff --git a/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go index 91b6562bd10..6d12302416a 100644 --- a/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go +++ b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go @@ -15,9 +15,10 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util/errutil" - "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" "golang.org/x/net/context/ctxhttp" ) @@ -48,7 +49,8 @@ func (e *AzureResourceGraphDatasource) resourceRequest(rw http.ResponseWriter, r // 1. builds the AzureMonitor url and querystring for each query // 2. executes each query by calling the Azure Monitor API // 3. parses the responses for each query into data frames -func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string) (*backend.QueryDataResponse, error) { +func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) { result := &backend.QueryDataResponse{ Responses: map[string]backend.DataResponse{}, } @@ -59,7 +61,7 @@ func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Contex } for _, query := range queries { - result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url) + result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url, tracer) } return result, nil @@ -101,7 +103,8 @@ func (e *AzureResourceGraphDatasource) buildQueries(queries []backend.DataQuery, return azureResourceGraphQueries, nil } -func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *AzureResourceGraphQuery, dsInfo datasourceInfo, client *http.Client, dsURL string) backend.DataResponse { +func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *AzureResourceGraphQuery, dsInfo datasourceInfo, client *http.Client, + dsURL string, tracer tracing.Tracer) backend.DataResponse { dataResponse := backend.DataResponse{} params := url.Values{} @@ -148,21 +151,16 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query * req.URL.Path = path.Join(req.URL.Path, argQueryProviderName) req.URL.RawQuery = params.Encode() - span, ctx := opentracing.StartSpanFromContext(ctx, "azure resource graph query") - span.SetTag("interpolated_query", query.InterpolatedQuery) - span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)) - span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)) - span.SetTag("datasource_id", dsInfo.DatasourceID) - span.SetTag("org_id", dsInfo.OrgID) + ctx, span := tracer.Start(ctx, "azure resource graph query") + span.SetAttributes("interpolated_query", query.InterpolatedQuery, attribute.Key("interpolated_query").String(query.InterpolatedQuery)) + span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID)) + span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID)) - defer span.Finish() + defer span.End() - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)); err != nil { - return dataResponseErrorWithExecuted(err) - } + tracer.Inject(ctx, req.Header, span) azlog.Debug("AzureResourceGraph", "Request ApiURL", req.URL.String()) res, err := ctxhttp.Do(ctx, client, req) diff --git a/pkg/tsdb/azuremonitor/azuremonitor-datasource.go b/pkg/tsdb/azuremonitor/azuremonitor-datasource.go index 31f42a6ddda..ff16e0b55e6 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor-datasource.go +++ b/pkg/tsdb/azuremonitor/azuremonitor-datasource.go @@ -14,9 +14,10 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util/errutil" - opentracing "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" "golang.org/x/net/context/ctxhttp" ) @@ -43,7 +44,8 @@ func (e *AzureMonitorDatasource) resourceRequest(rw http.ResponseWriter, req *ht // 1. build the AzureMonitor url and querystring for each query // 2. executes each query by calling the Azure Monitor API // 3. parses the responses for each query into data frames -func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string) (*backend.QueryDataResponse, error) { +func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() queries, err := e.buildQueries(originalQueries, dsInfo) @@ -52,7 +54,7 @@ func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, ori } for _, query := range queries { - result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url) + result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url, tracer) } return result, nil @@ -147,7 +149,8 @@ func (e *AzureMonitorDatasource) buildQueries(queries []backend.DataQuery, dsInf return azureMonitorQueries, nil } -func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureMonitorQuery, dsInfo datasourceInfo, cli *http.Client, url string) backend.DataResponse { +func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureMonitorQuery, dsInfo datasourceInfo, cli *http.Client, + url string, tracer tracing.Tracer) backend.DataResponse { dataResponse := backend.DataResponse{} req, err := e.createRequest(ctx, dsInfo, url) @@ -159,22 +162,15 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureM req.URL.Path = path.Join(req.URL.Path, query.URL) req.URL.RawQuery = query.Params.Encode() - span, ctx := opentracing.StartSpanFromContext(ctx, "azuremonitor query") - span.SetTag("target", query.Target) - span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)) - span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)) - span.SetTag("datasource_id", dsInfo.DatasourceID) - span.SetTag("org_id", dsInfo.OrgID) + ctx, span := tracer.Start(ctx, "azuremonitor query") + span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target)) + span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond))) + span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID)) + span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID)) - defer span.Finish() - - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)); err != nil { - dataResponse.Error = err - return dataResponse - } + defer span.End() + tracer.Inject(ctx, req.Header, span) azlog.Debug("AzureMonitor", "Request ApiURL", req.URL.String()) azlog.Debug("AzureMonitor", "Target", query.Target) diff --git a/pkg/tsdb/azuremonitor/azuremonitor-resource-handler.go b/pkg/tsdb/azuremonitor/azuremonitor-resource-handler.go index 838fa2126f5..92ee30ec8cb 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor-resource-handler.go +++ b/pkg/tsdb/azuremonitor/azuremonitor-resource-handler.go @@ -85,7 +85,7 @@ func writeResponse(rw http.ResponseWriter, code int, msg string) { } } -func (s *Service) resourceHandler(subDataSource string) func(rw http.ResponseWriter, req *http.Request) { +func (s *Service) handleResourceReq(subDataSource string) func(rw http.ResponseWriter, req *http.Request) { return func(rw http.ResponseWriter, req *http.Request) { azlog.Debug("Received resource call", "url", req.URL.String(), "method", req.Method) @@ -115,11 +115,13 @@ func (s *Service) resourceHandler(subDataSource string) func(rw http.ResponseWri } } -// registerRoutes provides route definitions shared with the frontend. +// newResourceMux provides route definitions shared with the frontend. // Check: /public/app/plugins/datasource/grafana-azure-monitor-datasource/utils/common.ts -func (s *Service) registerRoutes(mux *http.ServeMux) { - mux.HandleFunc("/azuremonitor/", s.resourceHandler(azureMonitor)) - mux.HandleFunc("/appinsights/", s.resourceHandler(appInsights)) - mux.HandleFunc("/loganalytics/", s.resourceHandler(azureLogAnalytics)) - mux.HandleFunc("/resourcegraph/", s.resourceHandler(azureResourceGraph)) +func (s *Service) newResourceMux() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("/azuremonitor/", s.handleResourceReq(azureMonitor)) + mux.HandleFunc("/appinsights/", s.handleResourceReq(appInsights)) + mux.HandleFunc("/loganalytics/", s.handleResourceReq(azureLogAnalytics)) + mux.HandleFunc("/resourcegraph/", s.handleResourceReq(azureResourceGraph)) + return mux } diff --git a/pkg/tsdb/azuremonitor/azuremonitor-resource-handler_test.go b/pkg/tsdb/azuremonitor/azuremonitor-resource-handler_test.go index 081d24fcbaf..849bb3c9981 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor-resource-handler_test.go +++ b/pkg/tsdb/azuremonitor/azuremonitor-resource-handler_test.go @@ -91,7 +91,7 @@ func (s *fakeProxy) Do(rw http.ResponseWriter, req *http.Request, cli *http.Clie return nil } -func Test_resourceHandler(t *testing.T) { +func Test_handleResourceReq(t *testing.T) { proxy := &fakeProxy{} s := Service{ im: &fakeInstance{ @@ -102,7 +102,6 @@ func Test_resourceHandler(t *testing.T) { }, }, }, - Cfg: &setting.Cfg{}, executors: map[string]azDatasourceExecutor{ azureMonitor: &AzureMonitorDatasource{ proxy: proxy, @@ -114,7 +113,7 @@ func Test_resourceHandler(t *testing.T) { if err != nil { t.Fatalf("Unexpected error %v", err) } - s.resourceHandler(azureMonitor)(rw, req) + s.handleResourceReq(azureMonitor)(rw, req) expectedURL := "https://management.azure.com/subscriptions/44693801" if proxy.requestedURL != expectedURL { t.Errorf("Unexpected result URL. Got %s, expecting %s", proxy.requestedURL, expectedURL) diff --git a/pkg/tsdb/azuremonitor/azuremonitor.go b/pkg/tsdb/azuremonitor/azuremonitor.go index 9c60374a5bd..b40ecda8695 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor.go +++ b/pkg/tsdb/azuremonitor/azuremonitor.go @@ -12,17 +12,16 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/azuremonitor/azcredentials" ) const ( timeSeries = "time_series" - pluginID = "grafana-azure-monitor-datasource" ) var ( @@ -30,7 +29,7 @@ var ( legendKeyFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) ) -func ProvideService(cfg *setting.Cfg, httpClientProvider *httpclient.Provider, pluginStore plugins.Store) *Service { +func ProvideService(cfg *setting.Cfg, httpClientProvider *httpclient.Provider, tracer tracing.Tracer) *Service { proxy := &httpServiceProxy{} executors := map[string]azDatasourceExecutor{ azureMonitor: &AzureMonitorDatasource{proxy: proxy}, @@ -42,35 +41,36 @@ func ProvideService(cfg *setting.Cfg, httpClientProvider *httpclient.Provider, p im := datasource.NewInstanceManager(NewInstanceSettings(cfg, *httpClientProvider, executors)) s := &Service{ - Cfg: cfg, im: im, executors: executors, + tracer: tracer, } - mux := s.newMux() - resourceMux := http.NewServeMux() - s.registerRoutes(resourceMux) - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: mux, - CallResourceHandler: httpadapter.New(resourceMux), - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - azlog.Error("Failed to register plugin", "error", err) - } + s.queryMux = s.newQueryMux() + s.resourceHandler = httpadapter.New(s.newResourceMux()) return s } +func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + return s.queryMux.QueryData(ctx, req) +} + +func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + return s.resourceHandler.CallResource(ctx, req, sender) +} + type serviceProxy interface { Do(rw http.ResponseWriter, req *http.Request, cli *http.Client) http.ResponseWriter } type Service struct { - Cfg *setting.Cfg im instancemgmt.InstanceManager executors map[string]azDatasourceExecutor + + queryMux *datasource.QueryTypeMux + resourceHandler backend.CallResourceHandler + tracer tracing.Tracer } type azureMonitorSettings struct { @@ -162,7 +162,7 @@ func NewInstanceSettings(cfg *setting.Cfg, clientProvider httpclient.Provider, e } type azDatasourceExecutor interface { - executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string) (*backend.QueryDataResponse, error) + executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) resourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) } @@ -179,7 +179,7 @@ func (s *Service) getDataSourceFromPluginReq(req *backend.QueryDataRequest) (dat return dsInfo, nil } -func (s *Service) newMux() *datasource.QueryTypeMux { +func (s *Service) newQueryMux() *datasource.QueryTypeMux { mux := datasource.NewQueryTypeMux() for dsType := range s.executors { // Make a copy of the string to keep the reference after the iterator @@ -194,7 +194,7 @@ func (s *Service) newMux() *datasource.QueryTypeMux { if !ok { return nil, fmt.Errorf("missing service for %s", dst) } - return executor.executeTimeSeriesQuery(ctx, req.Queries, dsInfo, service.HTTPClient, service.URL) + return executor.executeTimeSeriesQuery(ctx, req.Queries, dsInfo, service.HTTPClient, service.URL, s.tracer) }) } return mux diff --git a/pkg/tsdb/azuremonitor/azuremonitor_test.go b/pkg/tsdb/azuremonitor/azuremonitor_test.go index 7b8e9a8e116..5b556242295 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor_test.go +++ b/pkg/tsdb/azuremonitor/azuremonitor_test.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/azuremonitor/azcredentials" "github.com/stretchr/testify/require" @@ -85,7 +86,8 @@ type fakeExecutor struct { func (f *fakeExecutor) resourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) { } -func (f *fakeExecutor) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string) (*backend.QueryDataResponse, error) { +func (f *fakeExecutor) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) { if client == nil { f.t.Errorf("The HTTP client for %s is missing", f.queryType) } else { @@ -97,10 +99,6 @@ func (f *fakeExecutor) executeTimeSeriesQuery(ctx context.Context, originalQueri } func Test_newMux(t *testing.T) { - cfg := &setting.Cfg{ - Azure: setting.AzureSettings{}, - } - tests := []struct { name string queryType string @@ -124,7 +122,6 @@ func Test_newMux(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &Service{ - Cfg: cfg, im: &fakeInstance{ routes: routes[azureMonitorPublic], services: map[string]datasourceService{ @@ -142,7 +139,7 @@ func Test_newMux(t *testing.T) { }, }, } - mux := s.newMux() + mux := s.newQueryMux() res, err := mux.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{}, Queries: []backend.DataQuery{ diff --git a/pkg/tsdb/azuremonitor/insights-analytics-datasource.go b/pkg/tsdb/azuremonitor/insights-analytics-datasource.go index fa3a165104f..4b4ee49edb6 100644 --- a/pkg/tsdb/azuremonitor/insights-analytics-datasource.go +++ b/pkg/tsdb/azuremonitor/insights-analytics-datasource.go @@ -12,8 +12,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/util/errutil" - "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" "golang.org/x/net/context/ctxhttp" ) @@ -38,7 +39,8 @@ func (e *InsightsAnalyticsDatasource) resourceRequest(rw http.ResponseWriter, re } func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, - originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, url string) (*backend.QueryDataResponse, error) { + originalQueries []backend.DataQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() queries, err := e.buildQueries(originalQueries, dsInfo) @@ -47,7 +49,7 @@ func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context } for _, query := range queries { - result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url) + result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo, client, url, tracer) } return result, nil @@ -86,7 +88,8 @@ func (e *InsightsAnalyticsDatasource) buildQueries(queries []backend.DataQuery, return iaQueries, nil } -func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *InsightsAnalyticsQuery, dsInfo datasourceInfo, client *http.Client, url string) backend.DataResponse { +func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *InsightsAnalyticsQuery, dsInfo datasourceInfo, client *http.Client, + url string, tracer tracing.Tracer) backend.DataResponse { dataResponse := backend.DataResponse{} dataResponseError := func(err error) backend.DataResponse { @@ -101,17 +104,13 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I req.URL.Path = path.Join(req.URL.Path, "query") req.URL.RawQuery = query.Params.Encode() - span, ctx := opentracing.StartSpanFromContext(ctx, "application insights analytics query") - span.SetTag("target", query.Target) - span.SetTag("datasource_id", dsInfo.DatasourceID) - span.SetTag("org_id", dsInfo.OrgID) + ctx, span := tracer.Start(ctx, "application insights analytics query") + span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target)) + span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID)) + span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID)) - defer span.Finish() - - err = opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)) + defer span.End() + tracer.Inject(ctx, req.Header, span) if err != nil { azlog.Warn("failed to inject global tracer") diff --git a/pkg/tsdb/cloudmonitoring/annotation_query.go b/pkg/tsdb/cloudmonitoring/annotation_query.go index 29864f2bbeb..66f2471f865 100644 --- a/pkg/tsdb/cloudmonitoring/annotation_query.go +++ b/pkg/tsdb/cloudmonitoring/annotation_query.go @@ -18,7 +18,7 @@ func (s *Service) executeAnnotationQuery(ctx context.Context, req *backend.Query return resp, err } - queryRes, dr, _, err := queries[0].run(ctx, req, s, dsInfo) + queryRes, dr, _, err := queries[0].run(ctx, req, s, dsInfo, s.tracer) if err != nil { return resp, err } diff --git a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go index 2f730c2fe43..7c8fae24d31 100644 --- a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go +++ b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go @@ -16,6 +16,7 @@ import ( "time" "github.com/grafana/grafana-google-sdk-go/pkg/utils" + "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" @@ -24,10 +25,7 @@ import ( "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" - "github.com/grafana/grafana/pkg/services/datasources" - + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/setting" ) @@ -60,41 +58,31 @@ var ( ) const ( - pluginID string = "stackdriver" - - gceAuthentication string = "gce" - jwtAuthentication string = "jwt" - metricQueryType string = "metrics" - sloQueryType string = "slo" - mqlEditorMode string = "mql" - crossSeriesReducerDefault string = "REDUCE_NONE" - perSeriesAlignerDefault string = "ALIGN_MEAN" + gceAuthentication = "gce" + jwtAuthentication = "jwt" + metricQueryType = "metrics" + sloQueryType = "slo" + mqlEditorMode = "mql" + crossSeriesReducerDefault = "REDUCE_NONE" + perSeriesAlignerDefault = "ALIGN_MEAN" ) -func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, pluginStore plugins.Store, - dsService *datasources.Service) *Service { +func ProvideService(httpClientProvider httpclient.Provider, tracer tracing.Tracer) *Service { s := &Service{ + tracer: tracer, httpClientProvider: httpClientProvider, - cfg: cfg, im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)), - dsService: dsService, } - mux := http.NewServeMux() - s.registerRoutes(mux) - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - CallResourceHandler: httpadapter.New(mux), - CheckHealthHandler: s, - }) + s.resourceHandler = httpadapter.New(s.newResourceMux()) - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - slog.Error("Failed to register plugin", "error", err) - } return s } +func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + return s.resourceHandler.CallResource(ctx, req, sender) +} + func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { dsInfo, err := s.getDSInfo(req.PluginContext) if err != nil { @@ -136,9 +124,10 @@ func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthReque type Service struct { httpClientProvider httpclient.Provider - cfg *setting.Cfg im instancemgmt.InstanceManager - dsService *datasources.Service + tracer tracing.Tracer + + resourceHandler backend.CallResourceHandler } type QueryModel struct { @@ -263,7 +252,7 @@ func (s *Service) executeTimeSeriesQuery(ctx context.Context, req *backend.Query } for _, queryExecutor := range queryExecutors { - queryRes, dr, executedQueryString, err := queryExecutor.run(ctx, req, s, dsInfo) + queryRes, dr, executedQueryString, err := queryExecutor.run(ctx, req, s, dsInfo, s.tracer) if err != nil { return resp, err } diff --git a/pkg/tsdb/cloudmonitoring/resource_handler.go b/pkg/tsdb/cloudmonitoring/resource_handler.go index 32141147a08..038d8c7fc62 100644 --- a/pkg/tsdb/cloudmonitoring/resource_handler.go +++ b/pkg/tsdb/cloudmonitoring/resource_handler.go @@ -25,13 +25,14 @@ const resourceManagerPath = "/v1/projects" type processResponse func(body []byte) ([]json.RawMessage, string, error) -func (s *Service) registerRoutes(mux *http.ServeMux) { +func (s *Service) newResourceMux() *http.ServeMux { + mux := http.NewServeMux() mux.HandleFunc("/gceDefaultProject", getGCEDefaultProject) - - mux.HandleFunc("/metricDescriptors/", s.resourceHandler(cloudMonitor, processMetricDescriptors)) - mux.HandleFunc("/services/", s.resourceHandler(cloudMonitor, processServices)) - mux.HandleFunc("/slo-services/", s.resourceHandler(cloudMonitor, processSLOs)) - mux.HandleFunc("/projects", s.resourceHandler(resourceManager, processProjects)) + mux.HandleFunc("/metricDescriptors/", s.handleResourceReq(cloudMonitor, processMetricDescriptors)) + mux.HandleFunc("/services/", s.handleResourceReq(cloudMonitor, processServices)) + mux.HandleFunc("/slo-services/", s.handleResourceReq(cloudMonitor, processSLOs)) + mux.HandleFunc("/projects", s.handleResourceReq(resourceManager, processProjects)) + return mux } func getGCEDefaultProject(rw http.ResponseWriter, req *http.Request) { @@ -43,7 +44,7 @@ func getGCEDefaultProject(rw http.ResponseWriter, req *http.Request) { writeResponse(rw, http.StatusOK, project) } -func (s *Service) resourceHandler(subDataSource string, responseFn processResponse) func(rw http.ResponseWriter, req *http.Request) { +func (s *Service) handleResourceReq(subDataSource string, responseFn processResponse) func(rw http.ResponseWriter, req *http.Request) { return func(rw http.ResponseWriter, req *http.Request) { client, code, err := s.setRequestVariables(req, subDataSource) if err != nil { diff --git a/pkg/tsdb/cloudmonitoring/time_series_filter.go b/pkg/tsdb/cloudmonitoring/time_series_filter.go index a818e7988e0..818565f858b 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_filter.go +++ b/pkg/tsdb/cloudmonitoring/time_series_filter.go @@ -12,11 +12,12 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/opentracing/opentracing-go" + "github.com/grafana/grafana/pkg/infra/tracing" + "go.opentelemetry.io/otel/attribute" ) func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) run(ctx context.Context, req *backend.QueryDataRequest, - s *Service, dsInfo datasourceInfo) (*backend.DataResponse, cloudMonitoringResponse, string, error) { + s *Service, dsInfo datasourceInfo, tracer tracing.Tracer) (*backend.DataResponse, cloudMonitoringResponse, string, error) { dr := &backend.DataResponse{} projectName := timeSeriesFilter.ProjectName if projectName == "" { @@ -55,22 +56,15 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) run(ctx context.Context } } - span, ctx := opentracing.StartSpanFromContext(ctx, "cloudMonitoring query") - span.SetTag("target", timeSeriesFilter.Target) - span.SetTag("from", req.Queries[0].TimeRange.From) - span.SetTag("until", req.Queries[0].TimeRange.To) - span.SetTag("datasource_id", dsInfo.id) - span.SetTag("org_id", req.PluginContext.OrgID) + ctx, span := tracer.Start(ctx, "cloudMonitoring query") + span.SetAttributes("target", timeSeriesFilter.Target, attribute.Key("target").String(timeSeriesFilter.Target)) + span.SetAttributes("from", req.Queries[0].TimeRange.From, attribute.Key("from").String(req.Queries[0].TimeRange.From.String())) + span.SetAttributes("until", req.Queries[0].TimeRange.To, attribute.Key("until").String(req.Queries[0].TimeRange.To.String())) + span.SetAttributes("datasource_id", dsInfo.id, attribute.Key("datasource_id").Int64(dsInfo.id)) + span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID)) - defer span.Finish() - - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(r.Header)); err != nil { - dr.Error = err - return dr, cloudMonitoringResponse{}, "", nil - } + defer span.End() + tracer.Inject(ctx, r.Header, span) r = r.WithContext(ctx) res, err := dsInfo.services[cloudMonitor].client.Do(r) @@ -91,16 +85,13 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) run(ctx context.Context //nolint: gocyclo func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes *backend.DataResponse, response cloudMonitoringResponse, executedQueryString string) error { - labels := make(map[string]map[string]bool) frames := data.Frames{} - customFrameMeta := map[string]interface{}{} - customFrameMeta["alignmentPeriod"] = timeSeriesFilter.Params.Get("aggregation.alignmentPeriod") - customFrameMeta["perSeriesAligner"] = timeSeriesFilter.Params.Get("aggregation.perSeriesAligner") for _, series := range response.TimeSeries { seriesLabels := data.Labels{} defaultMetricName := series.Metric.Type - labels["resource.type"] = map[string]bool{series.Resource.Type: true} + labels := make(map[string]string) + labels["resource.type"] = series.Resource.Type seriesLabels["resource.type"] = series.Resource.Type frame := data.NewFrameOfFieldTypes("", len(series.Points), data.FieldTypeTime, data.FieldTypeFloat64) @@ -110,10 +101,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes } for key, value := range series.Metric.Labels { - if _, ok := labels["metric.label."+key]; !ok { - labels["metric.label."+key] = map[string]bool{} - } - labels["metric.label."+key][value] = true + labels["metric.label."+key] = value seriesLabels["metric.label."+key] = value if len(timeSeriesFilter.GroupBys) == 0 || containsLabel(timeSeriesFilter.GroupBys, "metric.label."+key) { @@ -122,10 +110,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes } for key, value := range series.Resource.Labels { - if _, ok := labels["resource.label."+key]; !ok { - labels["resource.label."+key] = map[string]bool{} - } - labels["resource.label."+key][value] = true + labels["resource.label."+key] = value seriesLabels["resource.label."+key] = value if containsLabel(timeSeriesFilter.GroupBys, "resource.label."+key) { @@ -136,22 +121,19 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes for labelType, labelTypeValues := range series.MetaData { for labelKey, labelValue := range labelTypeValues { key := toSnakeCase(fmt.Sprintf("metadata.%s.%s", labelType, labelKey)) - if _, ok := labels[key]; !ok { - labels[key] = map[string]bool{} - } switch v := labelValue.(type) { case string: - labels[key][v] = true + labels[key] = v seriesLabels[key] = v case bool: strVal := strconv.FormatBool(v) - labels[key][strVal] = true + labels[key] = strVal seriesLabels[key] = strVal case []interface{}: for _, v := range v { strVal := v.(string) - labels[key][strVal] = true + labels[key] = strVal if len(seriesLabels[key]) > 0 { strVal = fmt.Sprintf("%s, %s", seriesLabels[key], strVal) } @@ -161,6 +143,17 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes } } + customFrameMeta := map[string]interface{}{} + customFrameMeta["alignmentPeriod"] = timeSeriesFilter.Params.Get("aggregation.alignmentPeriod") + customFrameMeta["perSeriesAligner"] = timeSeriesFilter.Params.Get("aggregation.perSeriesAligner") + customFrameMeta["labels"] = labels + customFrameMeta["groupBys"] = timeSeriesFilter.GroupBys + if frame.Meta != nil { + frame.Meta.Custom = customFrameMeta + } else { + frame.SetMeta(&data.FrameMeta{Custom: customFrameMeta}) + } + // reverse the order to be ascending if series.ValueType != "DISTRIBUTION" { timeSeriesFilter.handleNonDistributionSeries(series, defaultMetricName, seriesLabels, frame) @@ -240,23 +233,6 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes frames = addConfigData(frames, dl, response.Unit) } - labelsByKey := make(map[string][]string) - for key, values := range labels { - for value := range values { - labelsByKey[key] = append(labelsByKey[key], value) - } - } - customFrameMeta["labels"] = labelsByKey - customFrameMeta["groupBys"] = timeSeriesFilter.GroupBys - - for _, frame := range frames { - if frame.Meta != nil { - frame.Meta.Custom = customFrameMeta - } else { - frame.SetMeta(&data.FrameMeta{Custom: customFrameMeta}) - } - } - queryRes.Frames = frames return nil diff --git a/pkg/tsdb/cloudmonitoring/time_series_filter_test.go b/pkg/tsdb/cloudmonitoring/time_series_filter_test.go index 6f521d9f87c..909652a3731 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_filter_test.go +++ b/pkg/tsdb/cloudmonitoring/time_series_filter_test.go @@ -370,6 +370,22 @@ func TestTimeSeriesFilter(t *testing.T) { assert.Equal(t, "test-proj - asia-northeast1-c - 6724404429462225363 - 200", frames[0].Fields[1].Name) }) }) + + t.Run("Parse labels", func(t *testing.T) { + data, err := loadTestFile("./test-data/5-series-response-meta-data.json") + require.NoError(t, err) + assert.Equal(t, 3, len(data.TimeSeries)) + res := &backend.DataResponse{} + query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}} + err = query.parseResponse(res, data, "") + require.NoError(t, err) + frames := res.Frames + custom, ok := frames[0].Meta.Custom.(map[string]interface{}) + require.True(t, ok) + labels, ok := custom["labels"].(map[string]string) + require.True(t, ok) + assert.Equal(t, "114250375703598695", labels["resource.label.instance_id"]) + }) } func loadTestFile(path string) (cloudMonitoringResponse, error) { diff --git a/pkg/tsdb/cloudmonitoring/time_series_query.go b/pkg/tsdb/cloudmonitoring/time_series_query.go index b45a93c4e88..f885432e315 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_query.go +++ b/pkg/tsdb/cloudmonitoring/time_series_query.go @@ -11,16 +11,17 @@ import ( "strings" "time" - "github.com/opentracing/opentracing-go" + "go.opentelemetry.io/otel/attribute" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/tsdb/intervalv2" ) func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) run(ctx context.Context, req *backend.QueryDataRequest, - s *Service, dsInfo datasourceInfo) (*backend.DataResponse, cloudMonitoringResponse, string, error) { + s *Service, dsInfo datasourceInfo, tracer tracing.Tracer) (*backend.DataResponse, cloudMonitoringResponse, string, error) { dr := &backend.DataResponse{} projectName := timeSeriesQuery.ProjectName @@ -55,20 +56,13 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) run(ctx context.Context, r return dr, cloudMonitoringResponse{}, "", nil } - span, ctx := opentracing.StartSpanFromContext(ctx, "cloudMonitoring MQL query") - span.SetTag("query", timeSeriesQuery.Query) - span.SetTag("from", req.Queries[0].TimeRange.From) - span.SetTag("until", req.Queries[0].TimeRange.To) + ctx, span := tracer.Start(ctx, "cloudMonitoring MQL query") + span.SetAttributes("query", timeSeriesQuery.Query, attribute.Key("query").String(timeSeriesQuery.Query)) + span.SetAttributes("from", req.Queries[0].TimeRange.From, attribute.Key("from").String(req.Queries[0].TimeRange.From.String())) + span.SetAttributes("until", req.Queries[0].TimeRange.To, attribute.Key("until").String(req.Queries[0].TimeRange.To.String())) - defer span.Finish() - - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(r.Header)); err != nil { - dr.Error = err - return dr, cloudMonitoringResponse{}, "", nil - } + defer span.End() + tracer.Inject(ctx, r.Header, span) r = r.WithContext(ctx) res, err := dsInfo.services[cloudMonitor].client.Do(r) @@ -88,10 +82,8 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) run(ctx context.Context, r func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *backend.DataResponse, response cloudMonitoringResponse, executedQueryString string) error { - labels := make(map[string]map[string]bool) frames := data.Frames{} - customFrameMeta := map[string]interface{}{} for _, series := range response.TimeSeriesData { seriesLabels := make(map[string]string) frame := data.NewFrameOfFieldTypes("", len(series.PointData), data.FieldTypeTime, data.FieldTypeFloat64) @@ -99,25 +91,23 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *ba frame.Meta = &data.FrameMeta{ ExecutedQueryString: executedQueryString, } + labels := make(map[string]string) for n, d := range response.TimeSeriesDescriptor.LabelDescriptors { key := toSnakeCase(d.Key) key = strings.Replace(key, ".", ".label.", 1) - if _, ok := labels[key]; !ok { - labels[key] = map[string]bool{} - } labelValue := series.LabelValues[n] switch d.ValueType { case "BOOL": strVal := strconv.FormatBool(labelValue.BoolValue) - labels[key][strVal] = true + labels[key] = strVal seriesLabels[key] = strVal case "INT64": - labels[key][labelValue.Int64Value] = true + labels[key] = labelValue.Int64Value seriesLabels[key] = labelValue.Int64Value default: - labels[key][labelValue.StringValue] = true + labels[key] = labelValue.StringValue seriesLabels[key] = labelValue.StringValue } } @@ -131,10 +121,7 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *ba continue } - if _, ok := labels["metric.name"]; !ok { - labels["metric.name"] = map[string]bool{} - } - labels["metric.name"][d.Key] = true + labels["metric.name"] = d.Key seriesLabels["metric.name"] = d.Key defaultMetricName := d.Key @@ -245,27 +232,19 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *ba frames = append(frames, buckets[i]) } } - } - if len(response.TimeSeriesData) > 0 { - dl := timeSeriesQuery.buildDeepLink() - frames = addConfigData(frames, dl, response.Unit) - } - labelsByKey := make(map[string][]string) - for key, values := range labels { - for value := range values { - labelsByKey[key] = append(labelsByKey[key], value) - } - } - customFrameMeta["labels"] = labelsByKey - - for _, frame := range frames { + customFrameMeta := map[string]interface{}{} + customFrameMeta["labels"] = labels if frame.Meta != nil { frame.Meta.Custom = customFrameMeta } else { frame.SetMeta(&data.FrameMeta{Custom: customFrameMeta}) } } + if len(response.TimeSeriesData) > 0 { + dl := timeSeriesQuery.buildDeepLink() + frames = addConfigData(frames, dl, response.Unit) + } queryRes.Frames = frames diff --git a/pkg/tsdb/cloudmonitoring/time_series_query_test.go b/pkg/tsdb/cloudmonitoring/time_series_query_test.go index aa5d85006c3..5f1900f9e75 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_query_test.go +++ b/pkg/tsdb/cloudmonitoring/time_series_query_test.go @@ -77,4 +77,28 @@ func TestTimeSeriesQuery(t *testing.T) { assert.Equal(t, "test-proj - asia-northeast1-c - 6724404429462225363 - 200", frames[0].Fields[1].Name) }) }) + + t.Run("Parse labels", func(t *testing.T) { + data, err := loadTestFile("./test-data/7-series-response-mql.json") + require.NoError(t, err) + + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) + res := &backend.DataResponse{} + query := &cloudMonitoringTimeSeriesQuery{ + ProjectName: "test-proj", + Query: "test-query", + timeRange: backend.TimeRange{ + From: fromStart, + To: fromStart.Add(34 * time.Minute), + }, + } + err = query.parseResponse(res, data, "") + require.NoError(t, err) + frames := res.Frames + custom, ok := frames[0].Meta.Custom.(map[string]interface{}) + require.True(t, ok) + labels, ok := custom["labels"].(map[string]string) + require.True(t, ok) + assert.Equal(t, "6724404429462225363", labels["resource.label.instance_id"]) + }) } diff --git a/pkg/tsdb/cloudmonitoring/types.go b/pkg/tsdb/cloudmonitoring/types.go index 77cb29913ac..a2cce2fb8b1 100644 --- a/pkg/tsdb/cloudmonitoring/types.go +++ b/pkg/tsdb/cloudmonitoring/types.go @@ -6,11 +6,12 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/infra/tracing" ) type ( cloudMonitoringQueryExecutor interface { - run(ctx context.Context, req *backend.QueryDataRequest, s *Service, dsInfo datasourceInfo) ( + run(ctx context.Context, req *backend.QueryDataRequest, s *Service, dsInfo datasourceInfo, tracer tracing.Tracer) ( *backend.DataResponse, cloudMonitoringResponse, string, error) parseResponse(dr *backend.DataResponse, data cloudMonitoringResponse, executedQueryString string) error parseToAnnotations(dr *backend.DataResponse, data cloudMonitoringResponse, title, text string) error diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 8ef02c05fcd..66a2d6c4789 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "fmt" + "net/http" "regexp" "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" @@ -24,9 +25,8 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/setting" ) @@ -43,61 +43,54 @@ type datasourceInfo struct { secretKey string datasourceID int64 + + HTTPClient *http.Client } -const cloudWatchTSFormat = "2006-01-02 15:04:05.000" -const defaultRegion = "default" +const ( + cloudWatchTSFormat = "2006-01-02 15:04:05.000" + defaultRegion = "default" -// Constants also defined in datasource/cloudwatch/datasource.ts -const logIdentifierInternal = "__log__grafana_internal__" -const logStreamIdentifierInternal = "__logstream__grafana_internal__" + // Constants also defined in datasource/cloudwatch/datasource.ts + logIdentifierInternal = "__log__grafana_internal__" + logStreamIdentifierInternal = "__logstream__grafana_internal__" -const pluginID = "cloudwatch" + alertMaxAttempts = 8 + alertPollPeriod = 1000 * time.Millisecond +) var plog = log.New("tsdb.cloudwatch") var aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) -func ProvideService(cfg *setting.Cfg, logsService *LogsService, pluginStore plugins.Store) (*CloudWatchService, error) { +func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider) *CloudWatchService { plog.Debug("initing") - executor := newExecutor(logsService, datasource.NewInstanceManager(NewInstanceSettings()), cfg, awsds.NewSessionCache()) - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: executor, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - plog.Error("Failed to register plugin", "error", err) - return nil, err - } + executor := newExecutor(datasource.NewInstanceManager(NewInstanceSettings(httpClientProvider)), cfg, awsds.NewSessionCache()) return &CloudWatchService{ - LogsService: logsService, - Cfg: cfg, - Executor: executor, - }, nil + Cfg: cfg, + Executor: executor, + } } type CloudWatchService struct { - LogsService *LogsService - Cfg *setting.Cfg - Executor *cloudWatchExecutor + Cfg *setting.Cfg + Executor *cloudWatchExecutor } type SessionCache interface { - GetSession(region string, s awsds.AWSDatasourceSettings) (*session.Session, error) + GetSession(c awsds.SessionConfig) (*session.Session, error) } -func newExecutor(logsService *LogsService, im instancemgmt.InstanceManager, cfg *setting.Cfg, sessions SessionCache) *cloudWatchExecutor { +func newExecutor(im instancemgmt.InstanceManager, cfg *setting.Cfg, sessions SessionCache) *cloudWatchExecutor { return &cloudWatchExecutor{ - logsService: logsService, - im: im, - cfg: cfg, - sessions: sessions, + im: im, + cfg: cfg, + sessions: sessions, } } -func NewInstanceSettings() datasource.InstanceFactoryFunc { +func NewInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc { return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { jsonData := struct { Profile string `json:"profile"` @@ -114,6 +107,11 @@ func NewInstanceSettings() datasource.InstanceFactoryFunc { return nil, fmt.Errorf("error reading settings: %w", err) } + httpClient, err := httpClientProvider.New() + if err != nil { + return nil, fmt.Errorf("error creating http client: %w", err) + } + model := datasourceInfo{ profile: jsonData.Profile, region: jsonData.Region, @@ -122,6 +120,7 @@ func NewInstanceSettings() datasource.InstanceFactoryFunc { endpoint: jsonData.Endpoint, namespace: jsonData.Namespace, datasourceID: settings.ID, + HTTPClient: httpClient, } at := awsds.AuthTypeDefault @@ -156,10 +155,9 @@ func NewInstanceSettings() datasource.InstanceFactoryFunc { // cloudWatchExecutor executes CloudWatch requests. type cloudWatchExecutor struct { - logsService *LogsService - im instancemgmt.InstanceManager - cfg *setting.Cfg - sessions SessionCache + im instancemgmt.InstanceManager + cfg *setting.Cfg + sessions SessionCache } func (e *cloudWatchExecutor) newSession(region string, pluginCtx backend.PluginContext) (*session.Session, error) { @@ -172,16 +170,20 @@ func (e *cloudWatchExecutor) newSession(region string, pluginCtx backend.PluginC region = dsInfo.region } - return e.sessions.GetSession(region, awsds.AWSDatasourceSettings{ - Profile: dsInfo.profile, - Region: region, - AuthType: dsInfo.authType, - AssumeRoleARN: dsInfo.assumeRoleARN, - ExternalID: dsInfo.externalID, - Endpoint: dsInfo.endpoint, - DefaultRegion: dsInfo.region, - AccessKey: dsInfo.accessKey, - SecretKey: dsInfo.secretKey, + return e.sessions.GetSession(awsds.SessionConfig{ + HTTPClient: dsInfo.HTTPClient, + Settings: awsds.AWSDatasourceSettings{ + Profile: dsInfo.profile, + Region: region, + AuthType: dsInfo.authType, + AssumeRoleARN: dsInfo.assumeRoleARN, + ExternalID: dsInfo.externalID, + Endpoint: dsInfo.endpoint, + DefaultRegion: dsInfo.region, + AccessKey: dsInfo.accessKey, + SecretKey: dsInfo.secretKey, + }, + UserAgentName: aws.String("Cloudwatch"), }) } @@ -225,9 +227,6 @@ func (e *cloudWatchExecutor) getRGTAClient(region string, pluginCtx backend.Plug func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, queryContext backend.DataQuery, model *simplejson.Json) (*cloudwatchlogs.GetQueryResultsOutput, error) { - const maxAttempts = 8 - const pollPeriod = 1000 * time.Millisecond - startQueryOutput, err := e.executeStartQuery(ctx, logsClient, model, queryContext.TimeRange) if err != nil { return nil, err @@ -238,7 +237,7 @@ func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwat "queryId": *startQueryOutput.QueryId, }) - ticker := time.NewTicker(pollPeriod) + ticker := time.NewTicker(alertPollPeriod) defer ticker.Stop() attemptCount := 1 @@ -250,7 +249,7 @@ func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwat if isTerminated(*res.Status) { return res, err } - if attemptCount >= maxAttempts { + if attemptCount >= alertMaxAttempts { return res, fmt.Errorf("fetching of query results exceeded max number of attempts") } @@ -290,8 +289,6 @@ func (e *cloudWatchExecutor) QueryData(ctx context.Context, req *backend.QueryDa result, err = e.executeAnnotationQuery(ctx, model, q, req.PluginContext) case "logAction": result, err = e.executeLogActions(ctx, req) - case "liveLogAction": - result, err = e.executeLiveLogQuery(ctx, req) case "timeSeriesQuery": fallthrough default: @@ -327,13 +324,6 @@ func (e *cloudWatchExecutor) executeLogAlertQuery(ctx context.Context, req *back return nil, err } - result, err := e.executeStartQuery(ctx, logsClient, model, q.TimeRange) - if err != nil { - return nil, err - } - - model.Set("queryId", *result.QueryId) - getQueryResultsOutput, err := e.alertQuery(ctx, logsClient, q, model) if err != nil { return nil, err @@ -383,24 +373,14 @@ func isTerminated(queryStatus string) bool { // // Stubbable by tests. var NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { - client := cloudwatch.New(sess) - client.Handlers.Send.PushFront(func(r *request.Request) { - r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) - }) - - return client + return cloudwatch.New(sess) } // NewCWLogsClient is a CloudWatch logs client factory. // // Stubbable by tests. var NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { - client := cloudwatchlogs.New(sess) - client.Handlers.Send.PushFront(func(r *request.Request) { - r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) - }) - - return client + return cloudwatchlogs.New(sess) } // EC2 client factory. diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go index def00c2e7e2..928667d61f9 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch_test.go +++ b/pkg/tsdb/cloudwatch/cloudwatch_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/grafana/grafana-aws-sdk/pkg/awsds" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/stretchr/testify/require" ) @@ -50,7 +51,7 @@ func TestNewInstanceSettings(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f := NewInstanceSettings() + f := NewInstanceSettings(httpclient.NewProvider()) model, err := f(tt.settings) tt.Err(t, err) datasourceComparer := cmp.Comparer(func(d1 datasourceInfo, d2 datasourceInfo) bool { diff --git a/pkg/tsdb/cloudwatch/live.go b/pkg/tsdb/cloudwatch/live.go index 8ff9178ddea..639edc9090e 100644 --- a/pkg/tsdb/cloudwatch/live.go +++ b/pkg/tsdb/cloudwatch/live.go @@ -1,295 +1,9 @@ package cloudwatch import ( - "context" - "encoding/json" - "fmt" - "sync" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/aws/aws-sdk-go/service/servicequotas" - "github.com/aws/aws-sdk-go/service/servicequotas/servicequotasiface" - "github.com/google/uuid" - "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util/retryer" - "golang.org/x/sync/errgroup" ) -const defaultConcurrentQueries = 4 - -type LogQueryRunnerSupplier struct { - Publisher models.ChannelPublisher - Service *LogsService -} - -type logQueryRunner struct { - channelName string - publish models.ChannelPublisher - running map[string]bool - runningMu sync.Mutex - service *LogsService -} - -const ( - maxAttempts = 8 - minRetryDelay = 500 * time.Millisecond - maxRetryDelay = 30 * time.Second -) - -// GetHandlerForPath gets the channel handler for a certain path. -func (s *LogQueryRunnerSupplier) GetHandlerForPath(path string) (models.ChannelHandler, error) { - return &logQueryRunner{ - channelName: path, - publish: s.Publisher, - running: make(map[string]bool), - service: s.Service, - }, nil -} - -// OnSubscribe publishes results from the corresponding CloudWatch Logs query to the provided channel -func (r *logQueryRunner) OnSubscribe(ctx context.Context, user *models.SignedInUser, e models.SubscribeEvent) (models.SubscribeReply, backend.SubscribeStreamStatus, error) { - r.runningMu.Lock() - defer r.runningMu.Unlock() - - if _, ok := r.running[e.Channel]; ok { - return models.SubscribeReply{}, backend.SubscribeStreamStatusOK, nil - } - - r.running[e.Channel] = true - go func() { - if err := r.publishResults(user.OrgId, e.Channel); err != nil { - plog.Error(err.Error()) - } - }() - - return models.SubscribeReply{}, backend.SubscribeStreamStatusOK, nil -} - -// OnPublish checks if a message from the websocket can be broadcast on this channel -func (r *logQueryRunner) OnPublish(ctx context.Context, user *models.SignedInUser, e models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) { - return models.PublishReply{}, backend.PublishStreamStatusPermissionDenied, nil -} - -func (r *logQueryRunner) publishResults(orgID int64, channelName string) error { - defer func() { - r.service.DeleteResponseChannel(channelName) - r.runningMu.Lock() - delete(r.running, channelName) - r.runningMu.Unlock() - }() - - responseChannel, err := r.service.GetResponseChannel(channelName) - if err != nil { - return err - } - - for response := range responseChannel { - responseBytes, err := json.Marshal(response) - if err != nil { - return err - } - - if err := r.publish(orgID, channelName, responseBytes); err != nil { - return err - } - } - - return nil -} - -// executeLiveLogQuery executes a CloudWatch Logs query with live updates over WebSocket. -// A WebSocket channel is created, which goroutines send responses over. -func (e *cloudWatchExecutor) executeLiveLogQuery(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { - responseChannelName := uuid.New().String() - responseChannel := make(chan *backend.QueryDataResponse) - if err := e.logsService.AddResponseChannel("plugin/cloudwatch/"+responseChannelName, responseChannel); err != nil { - close(responseChannel) - return nil, err - } - - go e.sendLiveQueriesToChannel(req, responseChannel) - - response := &backend.QueryDataResponse{ - Responses: backend.Responses{ - "A": { - Frames: data.Frames{data.NewFrame("A").SetMeta(&data.FrameMeta{ - Custom: map[string]interface{}{ - "channelName": responseChannelName, - }, - })}, - }, - }, - } - - return response, nil -} - -func (e *cloudWatchExecutor) sendLiveQueriesToChannel(req *backend.QueryDataRequest, responseChannel chan *backend.QueryDataResponse) { - defer close(responseChannel) - - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) - defer cancel() - eg, ectx := errgroup.WithContext(ctx) - - for _, query := range req.Queries { - query := query - eg.Go(func() error { - return e.startLiveQuery(ectx, responseChannel, query, query.TimeRange, req.PluginContext) - }) - } - - if err := eg.Wait(); err != nil { - plog.Error(err.Error()) - } -} - -func (e *cloudWatchExecutor) getQueue(queueKey string, pluginCtx backend.PluginContext) (chan bool, error) { - e.logsService.queueLock.Lock() - defer e.logsService.queueLock.Unlock() - - if queue, ok := e.logsService.queues[queueKey]; ok { - return queue, nil - } - - concurrentQueriesQuota := e.fetchConcurrentQueriesQuota(queueKey, pluginCtx) - - queueChannel := make(chan bool, concurrentQueriesQuota) - e.logsService.queues[queueKey] = queueChannel - - return queueChannel, nil -} - -func (e *cloudWatchExecutor) fetchConcurrentQueriesQuota(region string, pluginCtx backend.PluginContext) int { - sess, err := e.newSession(region, pluginCtx) - if err != nil { - plog.Warn("Could not get service quota client") - return defaultConcurrentQueries - } - - client := newQuotasClient(sess) - - concurrentQueriesQuota, err := client.GetServiceQuota(&servicequotas.GetServiceQuotaInput{ - ServiceCode: aws.String("logs"), - QuotaCode: aws.String("L-32C48FBB"), - }) - if err != nil { - plog.Warn("Could not get service quota") - return defaultConcurrentQueries - } - - if concurrentQueriesQuota != nil && concurrentQueriesQuota.Quota != nil && concurrentQueriesQuota.Quota.Value != nil { - return int(*concurrentQueriesQuota.Quota.Value) - } - - plog.Warn("Could not get service quota") - - defaultConcurrentQueriesQuota, err := client.GetAWSDefaultServiceQuota(&servicequotas.GetAWSDefaultServiceQuotaInput{ - ServiceCode: aws.String("logs"), - QuotaCode: aws.String("L-32C48FBB"), - }) - if err != nil { - plog.Warn("Could not get default service quota") - return defaultConcurrentQueries - } - - if defaultConcurrentQueriesQuota != nil && defaultConcurrentQueriesQuota.Quota != nil && - defaultConcurrentQueriesQuota.Quota.Value != nil { - return int(*defaultConcurrentQueriesQuota.Quota.Value) - } - - plog.Warn("Could not get default service quota") - return defaultConcurrentQueries -} - -func (e *cloudWatchExecutor) startLiveQuery(ctx context.Context, responseChannel chan *backend.QueryDataResponse, query backend.DataQuery, timeRange backend.TimeRange, pluginCtx backend.PluginContext) error { - model, err := simplejson.NewJson(query.JSON) - if err != nil { - return err - } - - dsInfo, err := e.getDSInfo(pluginCtx) - if err != nil { - return err - } - - defaultRegion := dsInfo.region - region := model.Get("region").MustString(defaultRegion) - logsClient, err := e.getCWLogsClient(region, pluginCtx) - if err != nil { - return err - } - - queue, err := e.getQueue(fmt.Sprintf("%s-%d", region, dsInfo.datasourceID), pluginCtx) - if err != nil { - return err - } - - // Wait until there are no more active workers than the concurrent queries quota - queue <- true - defer func() { <-queue }() - - startQueryOutput, err := e.executeStartQuery(ctx, logsClient, model, timeRange) - if err != nil { - responseChannel <- &backend.QueryDataResponse{ - Responses: backend.Responses{ - query.RefID: {Error: err}, - }, - } - return err - } - - queryResultsInput := &cloudwatchlogs.GetQueryResultsInput{ - QueryId: startQueryOutput.QueryId, - } - - recordsMatched := 0.0 - return retryer.Retry(func() (retryer.RetrySignal, error) { - getQueryResultsOutput, err := logsClient.GetQueryResultsWithContext(ctx, queryResultsInput) - if err != nil { - return retryer.FuncError, err - } - - retryNeeded := *getQueryResultsOutput.Statistics.RecordsMatched <= recordsMatched - recordsMatched = *getQueryResultsOutput.Statistics.RecordsMatched - - dataFrame, err := logsResultsToDataframes(getQueryResultsOutput) - if err != nil { - return retryer.FuncError, err - } - - dataFrame.Name = query.RefID - dataFrame.RefID = query.RefID - dataFrames, err := groupResponseFrame(dataFrame, model.Get("statsGroups").MustStringArray()) - if err != nil { - return retryer.FuncError, fmt.Errorf("failed to group dataframe response: %v", err) - } - - responseChannel <- &backend.QueryDataResponse{ - Responses: backend.Responses{ - query.RefID: { - Frames: dataFrames, - }, - }, - } - - if isTerminated(*getQueryResultsOutput.Status) { - return retryer.FuncComplete, nil - } else if retryNeeded { - return retryer.FuncFailure, nil - } - - return retryer.FuncSuccess, nil - }, maxAttempts, minRetryDelay, maxRetryDelay) -} - func groupResponseFrame(frame *data.Frame, statsGroups []string) (data.Frames, error) { var dataFrames data.Frames @@ -337,15 +51,3 @@ func setPreferredVisType(frame *data.Frame, visType data.VisType) { } } } - -// Service quotas client factory. -// -// Stubbable by tests. -var newQuotasClient = func(sess *session.Session) servicequotasiface.ServiceQuotasAPI { - client := servicequotas.New(sess) - client.Handlers.Send.PushFront(func(r *request.Request) { - r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) - }) - - return client -} diff --git a/pkg/tsdb/cloudwatch/log_actions.go b/pkg/tsdb/cloudwatch/log_actions.go index a9099d6cb87..cd553b23b14 100644 --- a/pkg/tsdb/cloudwatch/log_actions.go +++ b/pkg/tsdb/cloudwatch/log_actions.go @@ -18,7 +18,10 @@ import ( "golang.org/x/sync/errgroup" ) -var LimitExceededException = "LimitExceededException" +const ( + LimitExceededException = "LimitExceededException" + defaultLimit = 10 +) type AWSError struct { Code string @@ -126,7 +129,7 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, model *simple func (e *cloudWatchExecutor) handleGetLogEvents(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters *simplejson.Json) (*data.Frame, error) { queryRequest := &cloudwatchlogs.GetLogEventsInput{ - Limit: aws.Int64(parameters.Get("limit").MustInt64(10)), + Limit: aws.Int64(parameters.Get("limit").MustInt64(defaultLimit)), StartFromHead: aws.Bool(parameters.Get("startFromHead").MustBool(false)), } diff --git a/pkg/tsdb/cloudwatch/log_actions_test.go b/pkg/tsdb/cloudwatch/log_actions_test.go index 5691d8f3e00..4d3df038de3 100644 --- a/pkg/tsdb/cloudwatch/log_actions_test.go +++ b/pkg/tsdb/cloudwatch/log_actions_test.go @@ -52,7 +52,7 @@ func TestQuery_DescribeLogGroups(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -108,7 +108,7 @@ func TestQuery_DescribeLogGroups(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -183,7 +183,7 @@ func TestQuery_GetLogGroupFields(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -264,7 +264,7 @@ func TestQuery_StartQuery(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -317,7 +317,7 @@ func TestQuery_StartQuery(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -397,7 +397,7 @@ func TestQuery_StopQuery(t *testing.T) { To: time.Unix(1584700643, 0), } - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -487,7 +487,7 @@ func TestQuery_GetQueryResults(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, diff --git a/pkg/tsdb/cloudwatch/logs.go b/pkg/tsdb/cloudwatch/logs.go deleted file mode 100644 index aa0f06ebd94..00000000000 --- a/pkg/tsdb/cloudwatch/logs.go +++ /dev/null @@ -1,58 +0,0 @@ -package cloudwatch - -import ( - "fmt" - "sync" - - "github.com/grafana/grafana-plugin-sdk-go/backend" -) - -func ProvideLogsService() *LogsService { - return &LogsService{ - responseChannels: make(map[string]chan *backend.QueryDataResponse), - queues: make(map[string](chan bool)), - } -} - -// LogsService provides methods for querying CloudWatch Logs. -type LogsService struct { - channelMu sync.Mutex - responseChannels map[string]chan *backend.QueryDataResponse - queues map[string](chan bool) - queueLock sync.Mutex -} - -func (s *LogsService) AddResponseChannel(name string, channel chan *backend.QueryDataResponse) error { - s.channelMu.Lock() - defer s.channelMu.Unlock() - - if _, ok := s.responseChannels[name]; ok { - return fmt.Errorf("channel with name '%s' already exists", name) - } - - s.responseChannels[name] = channel - return nil -} - -func (s *LogsService) GetResponseChannel(name string) (chan *backend.QueryDataResponse, error) { - s.channelMu.Lock() - defer s.channelMu.Unlock() - - if responseChannel, ok := s.responseChannels[name]; ok { - return responseChannel, nil - } - - return nil, fmt.Errorf("channel with name '%s' not found", name) -} - -func (s *LogsService) DeleteResponseChannel(name string) { - s.channelMu.Lock() - defer s.channelMu.Unlock() - - if _, ok := s.responseChannels[name]; ok { - delete(s.responseChannels, name) - return - } - - plog.Warn("Channel with name '" + name + "' not found") -} diff --git a/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go b/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go index faaca8e4235..cf4b0439129 100644 --- a/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go +++ b/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go @@ -10,7 +10,7 @@ import ( func TestMetricDataQueryBuilder(t *testing.T) { t.Run("buildMetricDataQuery", func(t *testing.T) { t.Run("should use metric stat", func(t *testing.T) { - executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(nil, newTestConfig(), fakeSessionCache{}) query := getBaseQuery() query.MetricEditorMode = MetricEditorModeBuilder query.MetricQueryType = MetricQueryTypeSearch @@ -22,7 +22,7 @@ func TestMetricDataQueryBuilder(t *testing.T) { }) t.Run("should use custom built expression", func(t *testing.T) { - executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(nil, newTestConfig(), fakeSessionCache{}) query := getBaseQuery() query.MetricEditorMode = MetricEditorModeBuilder query.MetricQueryType = MetricQueryTypeSearch @@ -34,7 +34,7 @@ func TestMetricDataQueryBuilder(t *testing.T) { }) t.Run("should use sql expression", func(t *testing.T) { - executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(nil, newTestConfig(), fakeSessionCache{}) query := getBaseQuery() query.MetricEditorMode = MetricEditorModeRaw query.MetricQueryType = MetricQueryTypeQuery @@ -46,7 +46,7 @@ func TestMetricDataQueryBuilder(t *testing.T) { }) t.Run("should use user defined math expression", func(t *testing.T) { - executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(nil, newTestConfig(), fakeSessionCache{}) query := getBaseQuery() query.MetricEditorMode = MetricEditorModeRaw query.MetricQueryType = MetricQueryTypeSearch @@ -58,7 +58,7 @@ func TestMetricDataQueryBuilder(t *testing.T) { }) t.Run("should set period in user defined expression", func(t *testing.T) { - executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(nil, newTestConfig(), fakeSessionCache{}) query := getBaseQuery() query.MetricEditorMode = MetricEditorModeRaw query.MetricQueryType = MetricQueryTypeSearch diff --git a/pkg/tsdb/cloudwatch/metric_find_query_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go index bc4d75d7501..3e14a06dc53 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query_test.go +++ b/pkg/tsdb/cloudwatch/metric_find_query_test.go @@ -53,7 +53,7 @@ func TestQuery_Metrics(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -108,7 +108,7 @@ func TestQuery_Metrics(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -167,7 +167,7 @@ func TestQuery_Regions(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -242,7 +242,7 @@ func TestQuery_InstanceAttributes(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -341,7 +341,7 @@ func TestQuery_EBSVolumeIDs(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -420,7 +420,7 @@ func TestQuery_ResourceARNs(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -471,7 +471,7 @@ func TestQuery_GetAllMetrics(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -526,7 +526,7 @@ func TestQuery_GetDimensionKeys(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -574,7 +574,7 @@ func TestQuery_GetDimensionKeys(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, @@ -686,7 +686,7 @@ func TestQuery_ListMetricsPagination(t *testing.T) { im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, &setting.Cfg{AWSListMetricsPageLimit: 3, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true}, fakeSessionCache{}) + executor := newExecutor(im, &setting.Cfg{AWSListMetricsPageLimit: 3, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true}, fakeSessionCache{}) response, err := executor.listMetrics("default", &cloudwatch.ListMetricsInput{}, backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, }) @@ -701,7 +701,7 @@ func TestQuery_ListMetricsPagination(t *testing.T) { im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, &setting.Cfg{AWSListMetricsPageLimit: 1000, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true}, fakeSessionCache{}) + executor := newExecutor(im, &setting.Cfg{AWSListMetricsPageLimit: 1000, AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true}, fakeSessionCache{}) response, err := executor.listMetrics("default", &cloudwatch.ListMetricsInput{}, backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, }) diff --git a/pkg/tsdb/cloudwatch/test_utils.go b/pkg/tsdb/cloudwatch/test_utils.go index 9633606cbe8..a6727f74ca0 100644 --- a/pkg/tsdb/cloudwatch/test_utils.go +++ b/pkg/tsdb/cloudwatch/test_utils.go @@ -162,7 +162,7 @@ func newTestConfig() *setting.Cfg { type fakeSessionCache struct { } -func (s fakeSessionCache) GetSession(region string, settings awsds.AWSDatasourceSettings) (*session.Session, error) { +func (s fakeSessionCache) GetSession(c awsds.SessionConfig) (*session.Session, error) { return &session.Session{ Config: &aws.Config{}, }, nil diff --git a/pkg/tsdb/cloudwatch/time_series_query_test.go b/pkg/tsdb/cloudwatch/time_series_query_test.go index 58cfede9ce5..8e5c8d1535d 100644 --- a/pkg/tsdb/cloudwatch/time_series_query_test.go +++ b/pkg/tsdb/cloudwatch/time_series_query_test.go @@ -19,7 +19,7 @@ import ( ) func TestTimeSeriesQuery(t *testing.T) { - executor := newExecutor(nil, nil, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(nil, newTestConfig(), fakeSessionCache{}) now := time.Now() origNewCWClient := NewCWClient @@ -54,7 +54,7 @@ func TestTimeSeriesQuery(t *testing.T) { return datasourceInfo{}, nil }) - executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{}) + executor := newExecutor(im, newTestConfig(), fakeSessionCache{}) resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ PluginContext: backend.PluginContext{ DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 611763efc35..ec5d9f68d67 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -136,8 +136,8 @@ func (f *QueryStringFilter) MarshalJSON() ([]byte, error) { type RangeFilter struct { Filter Key string - Gte string - Lte string + Gte int64 + Lte int64 Format string } @@ -264,8 +264,8 @@ type TermsAggregation struct { // ExtendedBounds represents extended bounds type ExtendedBounds struct { - Min string `json:"min"` - Max string `json:"max"` + Min int64 `json:"min"` + Max int64 `json:"max"` } // GeoHashGridAggregation represents a geo hash grid aggregation diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go index a35de074c88..c1625adf4e5 100644 --- a/pkg/tsdb/elasticsearch/client/search_request.go +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -235,7 +235,7 @@ func (b *FilterQueryBuilder) Build() ([]Filter, error) { } // AddDateRangeFilter adds a new time range filter -func (b *FilterQueryBuilder) AddDateRangeFilter(timeField, lte, gte, format string) *FilterQueryBuilder { +func (b *FilterQueryBuilder) AddDateRangeFilter(timeField string, lte, gte int64, format string) *FilterQueryBuilder { b.filters = append(b.filters, &RangeFilter{ Key: timeField, Lte: lte, diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go index f9c29ac25f3..98f99b42f9a 100644 --- a/pkg/tsdb/elasticsearch/client/search_request_test.go +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -50,7 +50,7 @@ func TestSearchRequest(t *testing.T) { b.Size(200) b.SortDesc(timeField, "boolean") filters := b.Query().Bool().Filter() - filters.AddDateRangeFilter(timeField, "$timeTo", "$timeFrom", DateFormatEpochMS) + filters.AddDateRangeFilter(timeField, 10, 5, DateFormatEpochMS) filters.AddQueryStringFilter("test", true) t.Run("When building search request", func(t *testing.T) { @@ -71,8 +71,8 @@ func TestSearchRequest(t *testing.T) { t.Run("Should have range filter", func(t *testing.T) { f, ok := sr.Query.Bool.Filters[0].(*RangeFilter) require.True(t, ok) - require.Equal(t, "$timeFrom", f.Gte) - require.Equal(t, "$timeTo", f.Lte) + require.Equal(t, int64(5), f.Gte) + require.Equal(t, int64(10), f.Lte) require.Equal(t, "epoch_millis", f.Format) }) @@ -95,8 +95,8 @@ func TestSearchRequest(t *testing.T) { require.Equal(t, "boolean", sort.Get("unmapped_type").MustString()) timeRangeFilter := json.GetPath("query", "bool", "filter").GetIndex(0).Get("range").Get(timeField) - require.Equal(t, "$timeFrom", timeRangeFilter.Get("gte").MustString("")) - require.Equal(t, "$timeTo", timeRangeFilter.Get("lte").MustString("")) + require.Equal(t, int64(5), timeRangeFilter.Get("gte").MustInt64()) + require.Equal(t, int64(10), timeRangeFilter.Get("lte").MustInt64()) require.Equal(t, DateFormatEpochMS, timeRangeFilter.Get("format").MustString("")) queryStringFilter := json.GetPath("query", "bool", "filter").GetIndex(1).Get("query_string") diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 558466d57aa..48cb5da2964 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -12,47 +12,24 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" - "github.com/grafana/grafana/pkg/setting" es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" "github.com/grafana/grafana/pkg/tsdb/intervalv2" ) var eslog = log.New("tsdb.elasticsearch") -const pluginID = "elasticsearch" - type Service struct { - HTTPClientProvider httpclient.Provider + httpClientProvider httpclient.Provider intervalCalculator intervalv2.Calculator im instancemgmt.InstanceManager } -func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, pluginStore plugins.Store) (*Service, error) { +func ProvideService(httpClientProvider httpclient.Provider) *Service { eslog.Debug("initializing") - im := datasource.NewInstanceManager(newInstanceSettings()) - s := newService(im, httpClientProvider) - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: newService(im, s.HTTPClientProvider), - }) - - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, - plugins.CoreDataSourcePathResolver(cfg, pluginID)); err != nil { - eslog.Error("Failed to register plugin", "error", err) - return nil, err - } - - return s, nil -} - -// newService creates a new executor func. -func newService(im instancemgmt.InstanceManager, httpClientProvider httpclient.Provider) *Service { return &Service{ - im: im, - HTTPClientProvider: httpClientProvider, + im: datasource.NewInstanceManager(newInstanceSettings()), + httpClientProvider: httpClientProvider, intervalCalculator: intervalv2.NewCalculator(), } } @@ -67,7 +44,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) return &backend.QueryDataResponse{}, err } - client, err := es.NewClient(ctx, s.HTTPClientProvider, dsInfo, req.Queries[0].TimeRange) + client, err := es.NewClient(ctx, s.httpClientProvider, dsInfo, req.Queries[0].TimeRange) if err != nil { return &backend.QueryDataResponse{}, err } diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index 56f2b64f884..18cedfd337b 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -37,8 +37,8 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) { ms := e.client.MultiSearch() - from := fmt.Sprintf("%d", e.dataQueries[0].TimeRange.From.UnixNano()/int64(time.Millisecond)) - to := fmt.Sprintf("%d", e.dataQueries[0].TimeRange.To.UnixNano()/int64(time.Millisecond)) + from := e.dataQueries[0].TimeRange.From.UnixNano() / int64(time.Millisecond) + to := e.dataQueries[0].TimeRange.To.UnixNano() / int64(time.Millisecond) result := backend.QueryDataResponse{ Responses: backend.Responses{}, } @@ -62,7 +62,7 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) { return rp.getTimeSeries() } -func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to string, +func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to int64, result backend.QueryDataResponse) error { minInterval, err := e.client.GetMinInterval(q.Interval) if err != nil { @@ -243,7 +243,7 @@ func (bucketAgg BucketAgg) generateSettingsForDSL() map[string]interface{} { return bucketAgg.Settings.MustMap() } -func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string) es.AggBuilder { +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo int64) es.AggBuilder { aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index 67b6ce47fe0..333bebfb111 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -2,7 +2,6 @@ package elasticsearch import ( "encoding/json" - "fmt" "testing" "time" @@ -17,8 +16,8 @@ import ( func TestExecuteTimeSeriesQuery(t *testing.T) { from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) - fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) - toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + fromMs := from.UnixNano() / int64(time.Millisecond) + toMs := to.UnixNano() / int64(time.Millisecond) t.Run("Test execute time series query", func(t *testing.T) { t.Run("With defaults on es 2", func(t *testing.T) { @@ -32,14 +31,14 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { sr := c.multisearchRequests[0].Requests[0] rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) require.Equal(t, rangeFilter.Key, c.timeField) - require.Equal(t, rangeFilter.Lte, toStr) - require.Equal(t, rangeFilter.Gte, fromStr) + require.Equal(t, rangeFilter.Lte, toMs) + require.Equal(t, rangeFilter.Gte, fromMs) require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS) require.Equal(t, sr.Aggs[0].Key, "2") dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) require.Equal(t, dateHistogramAgg.Field, "@timestamp") - require.Equal(t, dateHistogramAgg.ExtendedBounds.Min, fromStr) - require.Equal(t, dateHistogramAgg.ExtendedBounds.Max, toStr) + require.Equal(t, dateHistogramAgg.ExtendedBounds.Min, fromMs) + require.Equal(t, dateHistogramAgg.ExtendedBounds.Max, toMs) }) t.Run("With defaults on es 5", func(t *testing.T) { @@ -53,8 +52,8 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { sr := c.multisearchRequests[0].Requests[0] require.Equal(t, sr.Query.Bool.Filters[0].(*es.RangeFilter).Key, c.timeField) require.Equal(t, sr.Aggs[0].Key, "2") - require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, fromStr) - require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, toStr) + require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, fromMs) + require.Equal(t, sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, toMs) }) t.Run("With multiple bucket aggs", func(t *testing.T) { @@ -1061,7 +1060,7 @@ func TestSettingsCasting(t *testing.T) { "field": "@timestamp", "id": "2", "settings": { - "interval": "1d" + "interval": "1d" } } ], @@ -1088,7 +1087,7 @@ func TestSettingsCasting(t *testing.T) { "field": "@timestamp", "id": "2", "settings": { - "interval": "1d" + "interval": "1d" } } ], diff --git a/pkg/tsdb/grafanads/grafana.go b/pkg/tsdb/grafanads/grafana.go index d3cd80b77c5..4c28b32e04b 100644 --- a/pkg/tsdb/grafanads/grafana.go +++ b/pkg/tsdb/grafanads/grafana.go @@ -14,8 +14,6 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/testdatasource" ) @@ -32,8 +30,6 @@ const DatasourceID = -1 // Grafana DS command. const DatasourceUID = "grafana" -const pluginID = "grafana" - // Make sure Service implements required interfaces. // This is important to do since otherwise we will only get a // not implemented error response from plugin at runtime. @@ -43,11 +39,11 @@ var ( logger = log.New("tsdb.grafana") ) -func ProvideService(cfg *setting.Cfg, pluginStore plugins.Store) *Service { - return newService(cfg, pluginStore) +func ProvideService(cfg *setting.Cfg) *Service { + return newService(cfg) } -func newService(cfg *setting.Cfg, pluginStore plugins.Store) *Service { +func newService(cfg *setting.Cfg) *Service { s := &Service{ staticRootPath: cfg.StaticRootPath, roots: []string{ @@ -59,14 +55,6 @@ func newService(cfg *setting.Cfg, pluginStore plugins.Store) *Service { }, } - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, coreplugin.New(backend.ServeOpts{ - CheckHealthHandler: s, - QueryDataHandler: s, - }), resolver); err != nil { - logger.Error("Failed to register plugin", "error", err) - return nil - } return s } diff --git a/pkg/tsdb/grafanads/grafana_test.go b/pkg/tsdb/grafanads/grafana_test.go index 07dec0e3325..d16345ce35d 100644 --- a/pkg/tsdb/grafanads/grafana_test.go +++ b/pkg/tsdb/grafanads/grafana_test.go @@ -1,13 +1,10 @@ package grafanads import ( - "context" "encoding/json" "path" "testing" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana-plugin-sdk-go/backend" @@ -21,7 +18,7 @@ func asJSON(v interface{}) json.RawMessage { } func TestReadFolderListing(t *testing.T) { - ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, &fakePluginStore{}) + ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}) dr := ds.doListQuery(backend.DataQuery{ QueryType: "x", JSON: asJSON(listQueryModel{ @@ -33,7 +30,7 @@ func TestReadFolderListing(t *testing.T) { } func TestReadCSVFile(t *testing.T) { - ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, &fakePluginStore{}) + ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}) dr := ds.doReadQuery(backend.DataQuery{ QueryType: "x", JSON: asJSON(readQueryModel{ @@ -43,11 +40,3 @@ func TestReadCSVFile(t *testing.T) { err := experimental.CheckGoldenDataResponse(path.Join("testdata", "jslib.golden.txt"), &dr, true) require.NoError(t, err) } - -type fakePluginStore struct { - plugins.Store -} - -func (ps *fakePluginStore) AddWithFactory(_ context.Context, _ string, _ backendplugin.PluginFactoryFunc, _ plugins.PluginPathResolver) error { - return nil -} diff --git a/pkg/tsdb/graphite/graphite.go b/pkg/tsdb/graphite/graphite.go index 5d75c487f98..bc3b61e9e9b 100644 --- a/pkg/tsdb/graphite/graphite.go +++ b/pkg/tsdb/graphite/graphite.go @@ -14,19 +14,18 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" "golang.org/x/net/context/ctxhttp" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/opentracing/opentracing-go" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/legacydata" ) @@ -34,31 +33,20 @@ import ( type Service struct { logger log.Logger im instancemgmt.InstanceManager + tracer tracing.Tracer } const ( - pluginID = "graphite" TargetFullModelField = "targetFull" TargetModelField = "target" ) -func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, pluginStore plugins.Store) (*Service, error) { - s := &Service{ +func ProvideService(httpClientProvider httpclient.Provider, tracer tracing.Tracer) *Service { + return &Service{ logger: log.New("tsdb.graphite"), im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)), + tracer: tracer, } - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - s.logger.Error("Failed to register plugin", "error", err) - return nil, err - } - - return s, nil } type datasourceInfo struct { @@ -165,21 +153,15 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) return &result, err } - span, ctx := opentracing.StartSpanFromContext(ctx, "graphite query") - span.SetTag("target", target) - span.SetTag("from", from) - span.SetTag("until", until) - span.SetTag("datasource_id", dsInfo.Id) - span.SetTag("org_id", req.PluginContext.OrgID) + ctx, span := s.tracer.Start(ctx, "graphite query") + span.SetAttributes("target", target, attribute.Key("target").String(target)) + span.SetAttributes("from", from, attribute.Key("from").String(from)) + span.SetAttributes("until", until, attribute.Key("until").String(until)) + span.SetAttributes("datasource_id", dsInfo.Id, attribute.Key("datasource_id").Int64(dsInfo.Id)) + span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID)) - defer span.Finish() - - if err := opentracing.GlobalTracer().Inject( - span.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(graphiteReq.Header)); err != nil { - return &result, err - } + defer span.End() + s.tracer.Inject(ctx, graphiteReq.Header, span) res, err := ctxhttp.Do(ctx, dsInfo.HTTPClient, graphiteReq) if err != nil { diff --git a/pkg/tsdb/influxdb/influxdb.go b/pkg/tsdb/influxdb/influxdb.go index fec2e42c304..ebfe3cf20e4 100644 --- a/pkg/tsdb/influxdb/influxdb.go +++ b/pkg/tsdb/influxdb/influxdb.go @@ -15,18 +15,14 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/influxdb/flux" "github.com/grafana/grafana/pkg/tsdb/influxdb/models" ) -const pluginID = "influxdb" - type Service struct { - QueryParser *InfluxdbQueryParser - ResponseParser *ResponseParser + queryParser *InfluxdbQueryParser + responseParser *ResponseParser glog log.Logger im instancemgmt.InstanceManager @@ -34,26 +30,13 @@ type Service struct { var ErrInvalidHttpMode = errors.New("'httpMode' should be either 'GET' or 'POST'") -func ProvideService(cfg *setting.Cfg, httpClient httpclient.Provider, pluginStore plugins.Store) (*Service, error) { - im := datasource.NewInstanceManager(newInstanceSettings(httpClient)) - s := &Service{ - QueryParser: &InfluxdbQueryParser{}, - ResponseParser: &ResponseParser{}, +func ProvideService(httpClient httpclient.Provider) *Service { + return &Service{ + queryParser: &InfluxdbQueryParser{}, + responseParser: &ResponseParser{}, glog: log.New("tsdb.influxdb"), - im: im, + im: datasource.NewInstanceManager(newInstanceSettings(httpClient)), } - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - s.glog.Error("Failed to register plugin", "error", err) - return nil, err - } - - return s, nil } func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc { @@ -146,7 +129,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) return &backend.QueryDataResponse{}, fmt.Errorf("InfluxDB returned error status: %s", res.Status) } - resp := s.ResponseParser.Parse(res.Body, query) + resp := s.responseParser.Parse(res.Body, query) return resp, nil } @@ -162,7 +145,7 @@ func (s *Service) getQuery(dsInfo *models.DatasourceInfo, query *backend.QueryDa q := query.Queries[0] - return s.QueryParser.Parse(q) + return s.queryParser.Parse(q) } func (s *Service) createRequest(ctx context.Context, dsInfo *models.DatasourceInfo, query string) (*http.Request, error) { diff --git a/pkg/tsdb/influxdb/influxdb_test.go b/pkg/tsdb/influxdb/influxdb_test.go index da7424ee00f..f3c7d76f085 100644 --- a/pkg/tsdb/influxdb/influxdb_test.go +++ b/pkg/tsdb/influxdb/influxdb_test.go @@ -20,8 +20,8 @@ func TestExecutor_createRequest(t *testing.T) { } query := "SELECT awesomeness FROM somewhere" s := &Service{ - QueryParser: &InfluxdbQueryParser{}, - ResponseParser: &ResponseParser{}, + queryParser: &InfluxdbQueryParser{}, + responseParser: &ResponseParser{}, glog: log.New("test"), } diff --git a/pkg/tsdb/legacydata/service/service_test.go b/pkg/tsdb/legacydata/service/service_test.go index 24615292f68..1f3c7312e01 100644 --- a/pkg/tsdb/legacydata/service/service_test.go +++ b/pkg/tsdb/legacydata/service/service_test.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/secrets/fakes" @@ -35,7 +36,7 @@ func TestHandleRequest(t *testing.T) { return backend.NewQueryDataResponse(), nil } secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasources.ProvideService(bus.New(), nil, secretsService) + dsService := datasources.ProvideService(bus.New(), nil, secretsService, &acmock.Mock{}) s := ProvideService(client, nil, dsService) ds := &models.DataSource{Id: 12, Type: "unregisteredType", JsonData: simplejson.New()} diff --git a/pkg/tsdb/loki/framing_test.go b/pkg/tsdb/loki/framing_test.go new file mode 100644 index 00000000000..be77657110e --- /dev/null +++ b/pkg/tsdb/loki/framing_test.go @@ -0,0 +1,138 @@ +package loki + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/experimental" + "github.com/grafana/loki/pkg/logcli/client" + "github.com/stretchr/testify/require" +) + +// NOTE: in these tests there several different json-content-types. +// different versions of Loki use different json-content-types. +// this is probably not important when we will parse them, +// but i wanted to test for all of them, to be sure. + +func TestSuccessResponse(t *testing.T) { + tt := []struct { + name string + filepath string + }{ + {name: "parse a simple matrix response", filepath: "matrix_simple"}, + {name: "parse a matrix response with a time-gap in the middle", filepath: "matrix_gap"}, + // you can produce NaN by having a metric query and add ` / 0` to the end + {name: "parse a matrix response with NaN", filepath: "matrix_nan"}, + // you can produce Infinity by using `quantile_over_time(42,` (value larger than 1) + {name: "parse a matrix response with Infinity", filepath: "matrix_inf"}, + } + + for _, test := range tt { + t.Run(test.name, func(t *testing.T) { + responseFileName := filepath.Join("testdata", test.filepath+".json") + goldenFileName := filepath.Join("testdata", test.filepath+".golden.txt") + + bytes, err := os.ReadFile(responseFileName) + require.NoError(t, err) + + frames, err := runQuery(makeMockedClient(200, "application/json", bytes), &lokiQuery{}) + require.NoError(t, err) + + dr := &backend.DataResponse{ + Frames: frames, + Error: err, + } + + err = experimental.CheckGoldenDataResponse(goldenFileName, dr, true) + require.NoError(t, err) + }) + } +} + +func TestErrorResponse(t *testing.T) { + // NOTE: when there is an error-response, it comes with + // HTTP code 400, and the format seems to change between versions: + // 2.3.x: content-type=text/plain, content is plaintext + // 2.4.0: content-type=application/json, content is plaintext !!! + // 2.4.1: same as 2.4.0 + // 2.4.2: same as 2.4.0 (2.4.2 is currently the latest) + // main-branch: content-type=application/json, content is JSON + // we should always be able to to return some kind of error message + // + // also, the returned error message is not what we want to return + // to the user, but this is what is currently returned to the user, + // so the tests check for that. we will have to change this in the future. + + tt := []struct { + name string + body []byte + contentType string + errorMessage string + }{ + { + name: "parse an error response in JSON", + body: []byte(` + { + "status": "error", + "code": 400, + "message": "parse error at line 1, col 8: something is wrong" + }`), + contentType: "application/json; charset=utf-8", + errorMessage: "Run out of attempts while querying the server", + }, + { + name: "parse a non-json error body with json content type (loki 2.4.0,2.4.1,2.4.2)", + body: []byte("parse error at line 1, col 8: something is wrong"), + contentType: "application/json; charset=UTF-8", + errorMessage: "Run out of attempts while querying the server", + }, + { + name: "parse an error response in plain text", + body: []byte("parse error at line 1, col 8: something is wrong"), + contentType: "text/plain; charset=utf-8", + errorMessage: "Run out of attempts while querying the server", + }, + } + + for _, test := range tt { + t.Run(test.name, func(t *testing.T) { + frames, err := runQuery(makeMockedClient(400, test.contentType, test.body), &lokiQuery{}) + + require.Len(t, frames, 0) + require.Error(t, err) + require.EqualError(t, err, test.errorMessage) + }) + } +} + +type MockedRoundTripper struct { + statusCode int + responseBytes []byte + contentType string +} + +func (mockedRT *MockedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + header := http.Header{} + header.Add("Content-Type", mockedRT.contentType) + return &http.Response{ + StatusCode: mockedRT.statusCode, + Header: header, + Body: ioutil.NopCloser(bytes.NewReader(mockedRT.responseBytes)), + }, nil +} + +func makeMockedClient(statusCode int, contentType string, responseBytes []byte) *client.DefaultClient { + client := &client.DefaultClient{ + Address: "http://localhost:9999", + Tripperware: func(t http.RoundTripper) http.RoundTripper { + return &MockedRoundTripper{statusCode: statusCode, responseBytes: responseBytes, contentType: contentType} + }, + } + + return client +} diff --git a/pkg/tsdb/loki/loki.go b/pkg/tsdb/loki/loki.go index 5c5c6a804bc..3cadf7bc66b 100644 --- a/pkg/tsdb/loki/loki.go +++ b/pkg/tsdb/loki/loki.go @@ -16,43 +16,28 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" - "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/loki/pkg/logcli/client" "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" + "go.opentelemetry.io/otel/attribute" - "github.com/opentracing/opentracing-go" "github.com/prometheus/common/config" "github.com/prometheus/common/model" ) -const pluginID = "loki" - type Service struct { - im instancemgmt.InstanceManager - plog log.Logger + im instancemgmt.InstanceManager + plog log.Logger + tracer tracing.Tracer } -func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, pluginStore plugins.Store) (*Service, error) { - im := datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)) - s := &Service{ - im: im, - plog: log.New("tsdb.loki"), +func ProvideService(httpClientProvider httpclient.Provider, tracer tracing.Tracer) *Service { + return &Service{ + im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)), + plog: log.New("tsdb.loki"), + tracer: tracer, } - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - s.plog.Error("Failed to register plugin", "error", err) - return nil, err - } - - return s, nil } var ( @@ -114,7 +99,6 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() - queryRes := backend.DataResponse{} dsInfo, err := s.getDSInfo(req.PluginContext) if err != nil { @@ -140,29 +124,22 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) for _, query := range queries { s.plog.Debug("Sending query", "start", query.Start, "end", query.End, "step", query.Step, "query", query.Expr) - span, _ := opentracing.StartSpanFromContext(ctx, "alerting.loki") - span.SetTag("expr", query.Expr) - span.SetTag("start_unixnano", query.Start.UnixNano()) - span.SetTag("stop_unixnano", query.End.UnixNano()) - defer span.Finish() + _, span := s.tracer.Start(ctx, "alerting.loki") + span.SetAttributes("expr", query.Expr, attribute.Key("expr").String(query.Expr)) + span.SetAttributes("start_unixnano", query.Start, attribute.Key("start_unixnano").Int64(query.Start.UnixNano())) + span.SetAttributes("stop_unixnano", query.End, attribute.Key("stop_unixnano").Int64(query.End.UnixNano())) + defer span.End() - // `limit` only applies to log-producing queries, and we - // currently only support metric queries, so this can be set to any value. - limit := 1 + frames, err := runQuery(client, query) - // we do not use `interval`, so we set it to zero - interval := time.Duration(0) + queryRes := backend.DataResponse{} - value, err := client.QueryRange(query.Expr, limit, query.Start, query.End, logproto.BACKWARD, query.Step, interval, false) if err != nil { - return result, err + queryRes.Error = err + } else { + queryRes.Frames = frames } - frames, err := parseResponse(value, query) - if err != nil { - return result, err - } - queryRes.Frames = frames result.Responses[query.RefID] = queryRes } return result, nil @@ -219,6 +196,23 @@ func parseResponse(value *loghttp.QueryResponse, query *lokiQuery) (data.Frames, return frames, nil } +// we extracted this part of the functionality to make it easy to unit-test it +func runQuery(client *client.DefaultClient, query *lokiQuery) (data.Frames, error) { + // `limit` only applies to log-producing queries, and we + // currently only support metric queries, so this can be set to any value. + limit := 1 + + // we do not use `interval`, so we set it to zero + interval := time.Duration(0) + + value, err := client.QueryRange(query.Expr, limit, query.Start, query.End, logproto.BACKWARD, query.Step, interval, false) + if err != nil { + return data.Frames{}, err + } + + return parseResponse(value, query) +} + func (s *Service) getDSInfo(pluginCtx backend.PluginContext) (*datasourceInfo, error) { i, err := s.im.Get(pluginCtx) if err != nil { diff --git a/pkg/tsdb/loki/loki_bench_test.go b/pkg/tsdb/loki/loki_bench_test.go new file mode 100644 index 00000000000..fec2eefc29e --- /dev/null +++ b/pkg/tsdb/loki/loki_bench_test.go @@ -0,0 +1,58 @@ +package loki + +import ( + "fmt" + "math/rand" + "strings" + "testing" +) + +// when memory-profiling these benchmarks these commands are recommended +// - go test -benchmem -run=^$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -bench ^BenchmarkMatrixJson$ github.com/grafana/grafana/pkg/tsdb/loki +// - go tool pprof -http=localhost:6061 memprofile.out +func BenchmarkMatrixJson(b *testing.B) { + bytes := createJsonTestData(1642000000, 1, 300, 400) + + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, _ = runQuery(makeMockedClient(200, "application/json", bytes), &lokiQuery{}) + } +} + +const nanRate = 0.002 + +// we build the JSON file from strings, +// it was easier to write it this way. +func makeJsonTestMetric(index int) string { + return fmt.Sprintf(`{"server":"main","category":"maintenance","case":"%v"}`, index) +} + +// return a value between -100 and +100, sometimes NaN, in string +func makeJsonTestValue(r *rand.Rand) string { + if r.Float64() < nanRate { + return "NaN" + } else { + return fmt.Sprintf("%f", (r.Float64()*200)-100) + } +} + +// create one time-series +func makeJsonTestSeries(start int64, step int64, timestampCount int, r *rand.Rand, seriesIndex int) string { + var values []string + for i := 0; i < timestampCount; i++ { + value := fmt.Sprintf(`[%d,"%v"]`, start+(int64(i)*step), makeJsonTestValue(r)) + values = append(values, value) + } + return fmt.Sprintf(`{"metric":%v,"values":[%v]}`, makeJsonTestMetric(seriesIndex), strings.Join(values, ",")) +} + +func createJsonTestData(start int64, step int64, timestampCount int, seriesCount int) []byte { + // we use random numbers as values, but they have to be the same numbers + // every time we call this, so we create a random source. + r := rand.New(rand.NewSource(42)) + var allSeries []string + for i := 0; i < seriesCount; i++ { + allSeries = append(allSeries, makeJsonTestSeries(start, step, timestampCount, r, i)) + } + return []byte(fmt.Sprintf(`{"data":{"resultType":"matrix","result":[%v]},"status":"success"}`, strings.Join(allSeries, ","))) +} diff --git a/pkg/tsdb/loki/testdata/matrix_gap.golden.txt b/pkg/tsdb/loki/testdata/matrix_gap.golden.txt new file mode 100644 index 00000000000..56003b4ca2a --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_gap.golden.txt @@ -0,0 +1,21 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] +Name: {} +Dimensions: 2 Fields by 6 Rows ++-------------------------------+-----------------+ +| Name: time | Name: value | +| Labels: | Labels: | +| Type: []time.Time | Type: []float64 | ++-------------------------------+-----------------+ +| 2022-01-24 08:54:13 +0000 UTC | 50 | +| 2022-01-24 08:59:13 +0000 UTC | 57 | +| 2022-01-24 09:04:13 +0000 UTC | 55 | +| 2022-01-24 09:34:13 +0000 UTC | 54 | +| 2022-01-24 09:39:13 +0000 UTC | 55 | +| 2022-01-24 09:44:13 +0000 UTC | 56 | ++-------------------------------+-----------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAFAAAAACAAAAKAAAAAQAAADI/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAOj+//8IAAAADAAAAAIAAAB7fQAABAAAAG5hbWUAAAAAAgAAANQAAAAEAAAARv///xQAAACcAAAAnAAAAAAAAAOcAAAAAwAAAFAAAAAsAAAABAAAADz///8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAAGD///8IAAAADAAAAAIAAAB7fQAABgAAAGxhYmVscwAAgP///wgAAAAkAAAAGgAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7fSJ9AAAGAAAAY29uZmlnAAAAAAAAiv///wAAAgAFAAAAdmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAAB0aW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAHRpbWUAAAAAAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAYAAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAWAAAAAYAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAADAAAAAAAAAAAAAAAAIAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAADS5Z/wKM0WAIpKeTYpzRYAQq9SfCnNFgCSC2sfK80WAEpwRGUrzRYAAtUdqyvNFgAAAAAAAElAAAAAAACATEAAAAAAAIBLQAAAAAAAAEtAAAAAAACAS0AAAAAAAABMQBAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAAEAAEAAADYAQAAAAAAAMAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAMj+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAA6P7//wgAAAAMAAAAAgAAAHt9AAAEAAAAbmFtZQAAAAACAAAA1AAAAAQAAABG////FAAAAJwAAACcAAAAAAAAA5wAAAADAAAAUAAAACwAAAAEAAAAPP///wgAAAAQAAAABQAAAHZhbHVlAAAABAAAAG5hbWUAAAAAYP///wgAAAAMAAAAAgAAAHt9AAAGAAAAbGFiZWxzAACA////CAAAACQAAAAaAAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6Int9In0AAAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAADwAQAAQVJST1cx diff --git a/pkg/tsdb/loki/testdata/matrix_gap.json b/pkg/tsdb/loki/testdata/matrix_gap.json new file mode 100644 index 00000000000..dbb9224b65d --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_gap.json @@ -0,0 +1,19 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [1643014453.815, "50"], + [1643014753.815, "57"], + [1643015053.815, "55"], + [1643016853.815, "54"], + [1643017153.815, "55"], + [1643017453.815, "56"] + ] + } + ] + } +} diff --git a/pkg/tsdb/loki/testdata/matrix_inf.golden.txt b/pkg/tsdb/loki/testdata/matrix_inf.golden.txt new file mode 100644 index 00000000000..4f0963ee6ed --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_inf.golden.txt @@ -0,0 +1,17 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] +Name: {level="info", location="moon", protocol="http"} +Dimensions: 2 Fields by 2 Rows ++-------------------------------+--------------------------------------------------+ +| Name: time | Name: value | +| Labels: | Labels: level=info, location=moon, protocol=http | +| Type: []time.Time | Type: []float64 | ++-------------------------------+--------------------------------------------------+ +| 2022-01-24 10:53:31 +0000 UTC | +Inf | +| 2022-01-24 10:58:31 +0000 UTC | -Inf | ++-------------------------------+--------------------------------------------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////YAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAIAAAAACAAAAKAAAAAQAAAAw/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFD+//8IAAAAPAAAADAAAAB7bGV2ZWw9ImluZm8iLCBsb2NhdGlvbj0ibW9vbiIsIHByb3RvY29sPSJodHRwIn0AAAAABAAAAG5hbWUAAAAAAgAAADwBAAAEAAAA3v7//xQAAAAEAQAABAEAAAAAAAMEAQAAAwAAAIQAAAAsAAAABAAAANT+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAAPj+//8IAAAAQAAAADQAAAB7ImxldmVsIjoiaW5mbyIsImxvY2F0aW9uIjoibW9vbiIsInByb3RvY29sIjoiaHR0cCJ9AAAAAAYAAABsYWJlbHMAAEz///8IAAAAWAAAAE4AAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiaW5mb1wiLCBsb2NhdGlvbj1cIm1vb25cIiwgcHJvdG9jb2w9XCJodHRwXCJ9In0AAAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAAAAAAAA/////7gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAAAgAAAAAAAAABQAAAAAAAADBAAKABgADAAIAAQACgAAABQAAABYAAAAAgAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAgAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAO7xOXMvzRYAplYTuS/NFgAAAAAAAPB/AAAAAAAA8P8QAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAOAAAAAAABAABAAAAcAIAAAAAAADAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAAIAAAAACAAAAKAAAAAQAAAAw/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFD+//8IAAAAPAAAADAAAAB7bGV2ZWw9ImluZm8iLCBsb2NhdGlvbj0ibW9vbiIsIHByb3RvY29sPSJodHRwIn0AAAAABAAAAG5hbWUAAAAAAgAAADwBAAAEAAAA3v7//xQAAAAEAQAABAEAAAAAAAMEAQAAAwAAAIQAAAAsAAAABAAAANT+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAAPj+//8IAAAAQAAAADQAAAB7ImxldmVsIjoiaW5mbyIsImxvY2F0aW9uIjoibW9vbiIsInByb3RvY29sIjoiaHR0cCJ9AAAAAAYAAABsYWJlbHMAAEz///8IAAAAWAAAAE4AAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiaW5mb1wiLCBsb2NhdGlvbj1cIm1vb25cIiwgcHJvdG9jb2w9XCJodHRwXCJ9In0AAAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAACIAgAAQVJST1cx diff --git a/pkg/tsdb/loki/testdata/matrix_inf.json b/pkg/tsdb/loki/testdata/matrix_inf.json new file mode 100644 index 00000000000..271888a00e1 --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_inf.json @@ -0,0 +1,19 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "level": "info", + "location": "moon", + "protocol": "http" + }, + "values": [ + [1643021611.9, "+Inf"], + [1643021911.9, "-Inf"] + ] + } + ] + } +} diff --git a/pkg/tsdb/loki/testdata/matrix_nan.golden.txt b/pkg/tsdb/loki/testdata/matrix_nan.golden.txt new file mode 100644 index 00000000000..56b3d7953f8 --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_nan.golden.txt @@ -0,0 +1,17 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] +Name: {} +Dimensions: 2 Fields by 2 Rows ++-------------------------------+-----------------+ +| Name: time | Name: value | +| Labels: | Labels: | +| Type: []time.Time | Type: []float64 | ++-------------------------------+-----------------+ +| 2022-01-24 08:54:10 +0000 UTC | NaN | +| 2022-01-24 08:59:10 +0000 UTC | NaN | ++-------------------------------+-----------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAFAAAAACAAAAKAAAAAQAAADI/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAOj+//8IAAAADAAAAAIAAAB7fQAABAAAAG5hbWUAAAAAAgAAANQAAAAEAAAARv///xQAAACcAAAAnAAAAAAAAAOcAAAAAwAAAFAAAAAsAAAABAAAADz///8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAAGD///8IAAAADAAAAAIAAAB7fQAABgAAAGxhYmVscwAAgP///wgAAAAkAAAAGgAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7fSJ9AAAGAAAAY29uZmlnAAAAAAAAiv///wAAAgAFAAAAdmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAAB0aW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAHRpbWUAAAAAAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAIAAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAWAAAAAIAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAAAAAAAIAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAB0Fe3vKM0WACx6xjUpzRYBAAAAAAD4fwEAAAAAAPh/EAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAQAAQAAANgBAAAAAAAAwAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABQAAAAAgAAACgAAAAEAAAAyP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADo/v//CAAAAAwAAAACAAAAe30AAAQAAABuYW1lAAAAAAIAAADUAAAABAAAAEb///8UAAAAnAAAAJwAAAAAAAADnAAAAAMAAABQAAAALAAAAAQAAAA8////CAAAABAAAAAFAAAAdmFsdWUAAAAEAAAAbmFtZQAAAABg////CAAAAAwAAAACAAAAe30AAAYAAABsYWJlbHMAAID///8IAAAAJAAAABoAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie30ifQAABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAPABAABBUlJPVzE= diff --git a/pkg/tsdb/loki/testdata/matrix_nan.json b/pkg/tsdb/loki/testdata/matrix_nan.json new file mode 100644 index 00000000000..b5431401c02 --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_nan.json @@ -0,0 +1,15 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [1643014450.417, "NaN"], + [1643014750.417, "NaN"] + ] + } + ] + } +} diff --git a/pkg/tsdb/loki/testdata/matrix_simple.golden.txt b/pkg/tsdb/loki/testdata/matrix_simple.golden.txt new file mode 100644 index 00000000000..68d2d0f5dc5 --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_simple.golden.txt @@ -0,0 +1,37 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] +Name: {level="error", location="moon"} +Dimensions: 2 Fields by 4 Rows ++-------------------------------+------------------------------------+ +| Name: time | Name: value | +| Labels: | Labels: level=error, location=moon | +| Type: []time.Time | Type: []float64 | ++-------------------------------+------------------------------------+ +| 2021-12-10 08:36:06 +0000 UTC | 0.4 | +| 2021-12-10 08:36:46 +0000 UTC | 0.2 | +| 2021-12-10 08:36:56 +0000 UTC | 0.2 | +| 2021-12-10 08:37:46 +0000 UTC | 0.6 | ++-------------------------------+------------------------------------+ + + + +Frame[1] +Name: {level="info", location="mars"} +Dimensions: 2 Fields by 5 Rows ++-------------------------------+-----------------------------------+ +| Name: time | Name: value | +| Labels: | Labels: level=info, location=mars | +| Type: []time.Time | Type: []float64 | ++-------------------------------+-----------------------------------+ +| 2021-12-10 08:36:26 +0000 UTC | 0.6 | +| 2021-12-10 08:36:36 +0000 UTC | 0.8 | +| 2021-12-10 08:36:46 +0000 UTC | 0.8 | +| 2021-12-10 08:36:56 +0000 UTC | 0.8 | +| 2021-12-10 08:37:56 +0000 UTC | 2.6 | ++-------------------------------+-----------------------------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////KAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAHAAAAACAAAAKAAAAAQAAABk/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAIT+//8IAAAALAAAACAAAAB7bGV2ZWw9ImVycm9yIiwgbG9jYXRpb249Im1vb24ifQAAAAAEAAAAbmFtZQAAAAACAAAAGAEAAAQAAAAC////FAAAAOAAAADgAAAAAAAAA+AAAAADAAAAcAAAACwAAAAEAAAA+P7//wgAAAAQAAAABQAAAHZhbHVlAAAABAAAAG5hbWUAAAAAHP///wgAAAAsAAAAIwAAAHsibGV2ZWwiOiJlcnJvciIsImxvY2F0aW9uIjoibW9vbiJ9AAYAAABsYWJlbHMAAFz///8IAAAASAAAADwAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoie2xldmVsPVwiZXJyb3JcIiwgbG9jYXRpb249XCJtb29uXCJ9In0AAAAABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAAB0aW1lAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAQAAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAAQAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAIAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAcw/7VV78WAKzyTt9XvxYAkP6i4Ve/FgAEOkftV78WmpmZmZmZ2T+amZmZmZnJP5qZmZmZmck/MzMzMzMz4z8QAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAOAIAAAAAAADAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABwAAAAAgAAACgAAAAEAAAAZP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAACE/v//CAAAACwAAAAgAAAAe2xldmVsPSJlcnJvciIsIGxvY2F0aW9uPSJtb29uIn0AAAAABAAAAG5hbWUAAAAAAgAAABgBAAAEAAAAAv///xQAAADgAAAA4AAAAAAAAAPgAAAAAwAAAHAAAAAsAAAABAAAAPj+//8IAAAAEAAAAAUAAAB2YWx1ZQAAAAQAAABuYW1lAAAAABz///8IAAAALAAAACMAAAB7ImxldmVsIjoiZXJyb3IiLCJsb2NhdGlvbiI6Im1vb24ifQAGAAAAbGFiZWxzAABc////CAAAAEgAAAA8AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntsZXZlbD1cImVycm9yXCIsIGxvY2F0aW9uPVwibW9vblwifSJ9AAAAAAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAABYAgAAQVJST1cx +FRAME=QVJST1cxAAD/////IAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAGwAAAACAAAAKAAAAAQAAABs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAIz+//8IAAAAKAAAAB8AAAB7bGV2ZWw9ImluZm8iLCBsb2NhdGlvbj0ibWFycyJ9AAQAAABuYW1lAAAAAAIAAAAUAQAABAAAAAb///8UAAAA3AAAANwAAAAAAAAD3AAAAAMAAABwAAAALAAAAAQAAAD8/v//CAAAABAAAAAFAAAAdmFsdWUAAAAEAAAAbmFtZQAAAAAg////CAAAACwAAAAiAAAAeyJsZXZlbCI6ImluZm8iLCJsb2NhdGlvbiI6Im1hcnMifQAABgAAAGxhYmVscwAAYP///wgAAABEAAAAOwAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7bGV2ZWw9XCJpbmZvXCIsIGxvY2F0aW9uPVwibWFyc1wifSJ9AAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAAD/////uAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAAFAAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAFgAAAAFAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAKAAAAAAAAAAoAAAAAAAAAAAAAAACAAAABQAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAA5Nqm2le/FgDI5vrcV78WAKzyTt9XvxYAkP6i4Ve/FgDoRZvvV78WMzMzMzMz4z+amZmZmZnpP5qZmZmZmek/mpmZmZmZ6T/NzMzMzMwEQBAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAADAAEAAAAwAgAAAAAAAMAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAAGwAAAACAAAAKAAAAAQAAABs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAIz+//8IAAAAKAAAAB8AAAB7bGV2ZWw9ImluZm8iLCBsb2NhdGlvbj0ibWFycyJ9AAQAAABuYW1lAAAAAAIAAAAUAQAABAAAAAb///8UAAAA3AAAANwAAAAAAAAD3AAAAAMAAABwAAAALAAAAAQAAAD8/v//CAAAABAAAAAFAAAAdmFsdWUAAAAEAAAAbmFtZQAAAAAg////CAAAACwAAAAiAAAAeyJsZXZlbCI6ImluZm8iLCJsb2NhdGlvbiI6Im1hcnMifQAABgAAAGxhYmVscwAAYP///wgAAABEAAAAOwAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7bGV2ZWw9XCJpbmZvXCIsIGxvY2F0aW9uPVwibWFyc1wifSJ9AAYAAABjb25maWcAAAAAAACK////AAACAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAABQAgAAQVJST1cx diff --git a/pkg/tsdb/loki/testdata/matrix_simple.json b/pkg/tsdb/loki/testdata/matrix_simple.json new file mode 100644 index 00000000000..8b3608d2f0b --- /dev/null +++ b/pkg/tsdb/loki/testdata/matrix_simple.json @@ -0,0 +1,33 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "level": "error", + "location": "moon" + }, + "values": [ + [1639125366.989, "0.4"], + [1639125406.989, "0.2"], + [1639125416.989, "0.2"], + [1639125466.989, "0.6"] + ] + }, + { + "metric": { + "level": "info", + "location": "mars" + }, + "values": [ + [1639125386.989, "0.6"], + [1639125396.989, "0.8"], + [1639125406.989, "0.8"], + [1639125416.989, "0.8"], + [1639125476.989, "2.6"] + ] + } + ] + } +} diff --git a/pkg/tsdb/mssql/mssql.go b/pkg/tsdb/mssql/mssql.go index 792c45a72d5..c130dc2411e 100644 --- a/pkg/tsdb/mssql/mssql.go +++ b/pkg/tsdb/mssql/mssql.go @@ -17,8 +17,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data/sqlutil" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/sqleng" "github.com/grafana/grafana/pkg/util" @@ -26,25 +24,14 @@ import ( var logger = log.New("tsdb.mssql") -const pluginID = "mssql" - type Service struct { im instancemgmt.InstanceManager } -func ProvideService(cfg *setting.Cfg, pluginStore plugins.Store) (*Service, error) { - s := &Service{ +func ProvideService(cfg *setting.Cfg) *Service { + return &Service{ im: datasource.NewInstanceManager(newInstanceSettings(cfg)), } - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - logger.Error("Failed to register plugin", "error", err) - } - return s, nil } func (s *Service) getDataSourceHandler(pluginCtx backend.PluginContext) (*sqleng.DataSourceHandler, error) { @@ -95,7 +82,6 @@ func newInstanceSettings(cfg *setting.Cfg) datasource.InstanceFactoryFunc { if cfg.Env == setting.Dev { logger.Debug("getEngine", "connection", cnnstr) } - config := sqleng.DataPluginConfiguration{ DriverName: "mssql", ConnectionString: cnnstr, @@ -158,8 +144,12 @@ func generateConnectionString(dsInfo sqleng.DataSourceInfo) (string, error) { if addr.Port != "0" { args = append(args, "port", addr.Port) } - logger.Debug("Generating connection string", args...) + + encrypt := dsInfo.JsonData.Encrypt + tlsSkipVerify := dsInfo.JsonData.TlsSkipVerify + hostNameInCertificate := dsInfo.JsonData.Servername + certificate := dsInfo.JsonData.RootCertFile connStr := fmt.Sprintf("server=%s;database=%s;user id=%s;password=%s;", addr.Host, dsInfo.Database, @@ -170,13 +160,15 @@ func generateConnectionString(dsInfo sqleng.DataSourceInfo) (string, error) { if addr.Port != "0" { connStr += fmt.Sprintf("port=%s;", addr.Port) } + if encrypt == "true" { + connStr += fmt.Sprintf("encrypt=%s;TrustServerCertificate=%t;", encrypt, tlsSkipVerify) + if hostNameInCertificate != "" { + connStr += fmt.Sprintf("hostNameInCertificate=%s;", hostNameInCertificate) + } - if dsInfo.JsonData.Encrypt == "" { - dsInfo.JsonData.Encrypt = "false" - } - - if dsInfo.JsonData.Encrypt != "false" { - connStr += fmt.Sprintf("encrypt=%s;", dsInfo.JsonData.Encrypt) + if certificate != "" { + connStr += fmt.Sprintf("certificate=%s;", certificate) + } } return connStr, nil } diff --git a/pkg/tsdb/mysql/mysql.go b/pkg/tsdb/mysql/mysql.go index 19dcf99fbd3..b3ee10a7fe7 100644 --- a/pkg/tsdb/mysql/mysql.go +++ b/pkg/tsdb/mysql/mysql.go @@ -20,14 +20,11 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data/sqlutil" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/sqleng" ) const ( - pluginID = "mysql" dateFormat = "2006-01-02" dateTimeFormat1 = "2006-01-02 15:04:05" dateTimeFormat2 = "2006-01-02T15:04:05Z" @@ -44,19 +41,10 @@ func characterEscape(s string, escapeChar string) string { return strings.ReplaceAll(s, escapeChar, url.QueryEscape(escapeChar)) } -func ProvideService(cfg *setting.Cfg, pluginStore plugins.Store, httpClientProvider httpclient.Provider) (*Service, error) { - s := &Service{ +func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider) *Service { + return &Service{ im: datasource.NewInstanceManager(newInstanceSettings(cfg, httpClientProvider)), } - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - logger.Error("Failed to register plugin", "error", err) - } - return s, nil } func newInstanceSettings(cfg *setting.Cfg, httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc { diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index ef42bf1bdbf..8dc65d0ff5c 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -19,35 +19,20 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/setting" "golang.org/x/net/context/ctxhttp" ) -const pluginID = "opentsdb" - type Service struct { logger log.Logger im instancemgmt.InstanceManager } -func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, pluginStore plugins.Store) (*Service, error) { - im := datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)) - s := &Service{ +func ProvideService(httpClientProvider httpclient.Provider) *Service { + return &Service{ logger: log.New("tsdb.opentsdb"), - im: im, + im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)), } - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - return nil, err - } - - return s, nil } type datasourceInfo struct { diff --git a/pkg/tsdb/postgres/postgres.go b/pkg/tsdb/postgres/postgres.go index 112c9bc9678..e7e73f0c3ea 100644 --- a/pkg/tsdb/postgres/postgres.go +++ b/pkg/tsdb/postgres/postgres.go @@ -14,8 +14,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data/sqlutil" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/sqleng" "github.com/grafana/grafana/pkg/util/errutil" @@ -23,22 +21,12 @@ import ( var logger = log.New("tsdb.postgres") -const pluginID = "postgres" - -func ProvideService(cfg *setting.Cfg, pluginStore plugins.Store) (*Service, error) { +func ProvideService(cfg *setting.Cfg) *Service { s := &Service{ tlsManager: newTLSManager(logger, cfg.DataPath), } s.im = datasource.NewInstanceManager(s.newInstanceSettings(cfg)) - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - logger.Error("Failed to register plugin", "error", err) - } - return s, nil + return s } type Service struct { diff --git a/pkg/tsdb/prometheus/framing_test.go b/pkg/tsdb/prometheus/framing_test.go new file mode 100644 index 00000000000..6db7e7e9c67 --- /dev/null +++ b/pkg/tsdb/prometheus/framing_test.go @@ -0,0 +1,136 @@ +package prometheus + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/experimental" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/prometheus/client_golang/api" + apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" +) + +func TestMatrixResponses(t *testing.T) { + tt := []struct { + name string + filepath string + }{ + {name: "parse a simple matrix response", filepath: "range_simple"}, + {name: "parse a simple matrix response with value missing steps", filepath: "range_missing"}, + {name: "parse a response with Infinity", filepath: "range_infinity"}, + {name: "parse a response with NaN", filepath: "range_nan"}, + } + + for _, test := range tt { + t.Run(test.name, func(t *testing.T) { + queryFileName := filepath.Join("testdata", test.filepath+".query.json") + responseFileName := filepath.Join("testdata", test.filepath+".result.json") + goldenFileName := filepath.Join("testdata", test.filepath+".result.golden.txt") + + query, err := loadStoredPrometheusQuery(queryFileName) + require.NoError(t, err) + + responseBytes, err := os.ReadFile(responseFileName) + require.NoError(t, err) + + result, err := runQuery(responseBytes, query) + require.NoError(t, err) + require.Len(t, result.Responses, 1) + + dr, found := result.Responses["A"] + require.True(t, found) + + require.NoError(t, experimental.CheckGoldenDataResponse(goldenFileName, &dr, true)) + }) + } +} + +type mockedRoundTripper struct { + responseBytes []byte +} + +func (mockedRT *mockedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(mockedRT.responseBytes)), + }, nil +} + +func makeMockedApi(responseBytes []byte) (apiv1.API, error) { + roundTripper := mockedRoundTripper{responseBytes: responseBytes} + + cfg := api.Config{ + Address: "http://localhost:9999", + RoundTripper: &roundTripper, + } + + client, err := api.NewClient(cfg) + if err != nil { + return nil, err + } + + api := apiv1.NewAPI(client) + + return api, nil +} + +// we store the prometheus query data in a json file, here is some minimal code +// to be able to read it back. unfortunately we cannot use the PrometheusQuery +// struct here, because it has `time.time` and `time.duration` fields that +// cannot be unmarshalled from JSON automatically. +type storedPrometheusQuery struct { + RefId string + RangeQuery bool + Start int64 + End int64 + Step int64 + Expr string +} + +func loadStoredPrometheusQuery(fileName string) (PrometheusQuery, error) { + bytes, err := os.ReadFile(fileName) + if err != nil { + return PrometheusQuery{}, err + } + + var query storedPrometheusQuery + + err = json.Unmarshal(bytes, &query) + if err != nil { + return PrometheusQuery{}, err + } + + return PrometheusQuery{ + RefId: query.RefId, + RangeQuery: query.RangeQuery, + Start: time.Unix(query.Start, 0), + End: time.Unix(query.End, 0), + Step: time.Second * time.Duration(query.Step), + Expr: query.Expr, + }, nil +} + +func runQuery(response []byte, query PrometheusQuery) (*backend.QueryDataResponse, error) { + api, err := makeMockedApi(response) + if err != nil { + return nil, err + } + + tracer, err := tracing.InitializeTracerForTest() + if err != nil { + return nil, err + } + + s := Service{tracer: tracer} + return s.runQueries(context.Background(), api, []*PrometheusQuery{&query}, true) +} diff --git a/pkg/tsdb/prometheus/prometeus_bench_test.go b/pkg/tsdb/prometheus/prometeus_bench_test.go new file mode 100644 index 00000000000..5bf01f8ae57 --- /dev/null +++ b/pkg/tsdb/prometheus/prometeus_bench_test.go @@ -0,0 +1,82 @@ +package prometheus + +import ( + "context" + "fmt" + "math/rand" + "strings" + "testing" + "time" + + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/stretchr/testify/require" +) + +// when memory-profiling this benchmark, these commands are recommended: +// - go test -benchmem -run=^$ -benchtime 1x -memprofile memprofile.out -memprofilerate 1 -bench ^BenchmarkJson$ github.com/grafana/grafana/pkg/tsdb/prometheus +// - go tool pprof -http=localhost:6061 memprofile.out +func BenchmarkJson(b *testing.B) { + resp, query := createJsonTestData(1642000000, 1, 300, 400) + + api, err := makeMockedApi(resp) + require.NoError(b, err) + + tracer, err := tracing.InitializeTracerForTest() + require.NoError(b, err) + + s := Service{tracer: tracer} + + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, _ = s.runQueries(context.Background(), api, []*PrometheusQuery{&query}, true) + } +} + +const nanRate = 0.002 + +// we build the JSON file from strings, +// it was easier to write it this way. +func makeJsonTestMetric(index int) string { + return fmt.Sprintf(`{"server":"main","category":"maintenance","case":"%v"}`, index) +} + +// return a value between -100 and +100, sometimes NaN, in string +func makeJsonTestValue(r *rand.Rand) string { + if r.Float64() < nanRate { + return "NaN" + } else { + return fmt.Sprintf("%f", (r.Float64()*200)-100) + } +} + +// create one time-series +func makeJsonTestSeries(start int64, step int64, timestampCount int, r *rand.Rand, seriesIndex int) string { + var values []string + for i := 0; i < timestampCount; i++ { + value := fmt.Sprintf(`[%d,"%v"]`, start+(int64(i)*step), makeJsonTestValue(r)) + values = append(values, value) + } + return fmt.Sprintf(`{"metric":%v,"values":[%v]}`, makeJsonTestMetric(seriesIndex), strings.Join(values, ",")) +} + +func createJsonTestData(start int64, step int64, timestampCount int, seriesCount int) ([]byte, PrometheusQuery) { + // we use random numbers as values, but they have to be the same numbers + // every time we call this, so we create a random source. + r := rand.New(rand.NewSource(42)) + var allSeries []string + for i := 0; i < seriesCount; i++ { + allSeries = append(allSeries, makeJsonTestSeries(start, step, timestampCount, r, i)) + } + bytes := []byte(fmt.Sprintf(`{"data":{"resultType":"matrix","result":[%v]},"status":"success"}`, strings.Join(allSeries, ","))) + + query := PrometheusQuery{ + RefId: "A", + RangeQuery: true, + Start: time.Unix(start, 0), + End: time.Unix(start+((int64(timestampCount)-1)*step), 0), + Step: time.Second * time.Duration(step), + Expr: "test", + } + + return bytes, query +} diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go index 022087fb700..e16597c50fb 100644 --- a/pkg/tsdb/prometheus/prometheus.go +++ b/pkg/tsdb/prometheus/prometheus.go @@ -14,9 +14,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" - "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/tsdb/intervalv2" apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" ) @@ -27,32 +25,19 @@ var ( safeRes = 11000 ) -const pluginID = "prometheus" - type Service struct { intervalCalculator intervalv2.Calculator im instancemgmt.InstanceManager + tracer tracing.Tracer } -func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, pluginStore plugins.Store) (*Service, error) { +func ProvideService(httpClientProvider httpclient.Provider, tracer tracing.Tracer) *Service { plog.Debug("initializing") - im := datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)) - - s := &Service{ + return &Service{ intervalCalculator: intervalv2.NewCalculator(), - im: im, + im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)), + tracer: tracer, } - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - plog.Error("Failed to register plugin", "error", err) - return nil, err - } - - return s, nil } func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc { diff --git a/pkg/tsdb/prometheus/testdata/range_infinity.query.json b/pkg/tsdb/prometheus/testdata/range_infinity.query.json new file mode 100644 index 00000000000..76d612b6824 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_infinity.query.json @@ -0,0 +1,8 @@ +{ + "RefId": "A", + "RangeQuery": true, + "Start": 1641889530, + "End": 1641889532, + "Step": 1, + "Expr": "1 / 0" +} diff --git a/pkg/tsdb/prometheus/testdata/range_infinity.result.golden.txt b/pkg/tsdb/prometheus/testdata/range_infinity.result.golden.txt new file mode 100644 index 00000000000..2aad902f0db --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_infinity.result.golden.txt @@ -0,0 +1,22 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] { + "custom": { + "resultType": "matrix" + } +} +Name: 1 / 0 +Dimensions: 2 Fields by 3 Rows ++-------------------------------+------------------+ +| Name: Time | Name: Value | +| Labels: | Labels: | +| Type: []time.Time | Type: []*float64 | ++-------------------------------+------------------+ +| 2022-01-11 08:25:30 +0000 UTC | +Inf | +| 2022-01-11 08:25:31 +0000 UTC | +Inf | +| 2022-01-11 08:25:32 +0000 UTC | +Inf | ++-------------------------------+------------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////KAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAJgAAAADAAAAUAAAACgAAAAEAAAAbP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAACM/v//CAAAABAAAAAFAAAAMSAvIDAAAAAEAAAAbmFtZQAAAACw/v//CAAAACwAAAAiAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6Im1hdHJpeCJ9fQAABAAAAG1ldGEAAAAAAgAAAOwAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAKAAAACgAAAAAAADAaAAAAADAAAAUAAAACwAAAAEAAAAOP///wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAXP///wgAAAAMAAAAAgAAAHt9AAAGAAAAbGFiZWxzAAB8////CAAAACgAAAAdAAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IjEgLyAwIn0AAAAGAAAAY29uZmlnAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAMAAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAAMAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAABgAAAAAAAAAAAAAAAIAAAADAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAABEFRTUKckWAA6wT9QpyRYA2EqL1CnJFgAAAAAAAPB/AAAAAAAA8H8AAAAAAADwfxAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAADAAEAAAA4AgAAAAAAAMAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAmAAAAAMAAABQAAAAKAAAAAQAAABs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAIz+//8IAAAAEAAAAAUAAAAxIC8gMAAAAAQAAABuYW1lAAAAALD+//8IAAAALAAAACIAAAB7ImN1c3RvbSI6eyJyZXN1bHRUeXBlIjoibWF0cml4In19AAAEAAAAbWV0YQAAAAACAAAA7AAAABgAAAAAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAoAAAAKAAAAAAAAMBoAAAAAMAAABQAAAALAAAAAQAAAA4////CAAAABAAAAAFAAAAVmFsdWUAAAAEAAAAbmFtZQAAAABc////CAAAAAwAAAACAAAAe30AAAYAAABsYWJlbHMAAHz///8IAAAAKAAAAB0AAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoiMSAvIDAifQAAAAYAAABjb25maWcAAAAAAACK////AAACAAUAAABWYWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAABQAgAAQVJST1cx diff --git a/pkg/tsdb/prometheus/testdata/range_infinity.result.json b/pkg/tsdb/prometheus/testdata/range_infinity.result.json new file mode 100644 index 00000000000..7059de48f6f --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_infinity.result.json @@ -0,0 +1,16 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [1641889530, "+Inf"], + [1641889531, "+Inf"], + [1641889532, "+Inf"] + ] + } + ] + } +} diff --git a/pkg/tsdb/prometheus/testdata/range_missing.query.json b/pkg/tsdb/prometheus/testdata/range_missing.query.json new file mode 100644 index 00000000000..3388e0d45b9 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_missing.query.json @@ -0,0 +1,8 @@ +{ + "RefId": "A", + "RangeQuery": true, + "Start": 1641889530, + "End": 1641889538, + "Step": 1, + "Expr": "test1" +} diff --git a/pkg/tsdb/prometheus/testdata/range_missing.result.golden.txt b/pkg/tsdb/prometheus/testdata/range_missing.result.golden.txt new file mode 100644 index 00000000000..e63cc8828b8 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_missing.result.golden.txt @@ -0,0 +1,28 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] { + "custom": { + "resultType": "matrix" + } +} +Name: go_goroutines{job="prometheus"} +Dimensions: 2 Fields by 9 Rows ++-------------------------------+------------------------------------------------+ +| Name: Time | Name: Value | +| Labels: | Labels: __name__=go_goroutines, job=prometheus | +| Type: []time.Time | Type: []*float64 | ++-------------------------------+------------------------------------------------+ +| 2022-01-11 08:25:30 +0000 UTC | null | +| 2022-01-11 08:25:31 +0000 UTC | null | +| 2022-01-11 08:25:32 +0000 UTC | null | +| 2022-01-11 08:25:33 +0000 UTC | 21 | +| 2022-01-11 08:25:34 +0000 UTC | 32 | +| 2022-01-11 08:25:35 +0000 UTC | null | +| 2022-01-11 08:25:36 +0000 UTC | null | +| 2022-01-11 08:25:37 +0000 UTC | 43 | +| 2022-01-11 08:25:38 +0000 UTC | null | ++-------------------------------+------------------------------------------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////iAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAALAAAAADAAAAaAAAACgAAAAEAAAADP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAAAs/v//CAAAACgAAAAfAAAAZ29fZ29yb3V0aW5lc3tqb2I9InByb21ldGhldXMifQAEAAAAbmFtZQAAAABo/v//CAAAACwAAAAiAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6Im1hdHJpeCJ9fQAABAAAAG1ldGEAAAAAAgAAADQBAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAOgAAADoAAAAAAADAegAAAADAAAAfAAAACwAAAAEAAAA8P7//wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAFP///wgAAAA4AAAALwAAAHsiX19uYW1lX18iOiJnb19nb3JvdXRpbmVzIiwiam9iIjoicHJvbWV0aGV1cyJ9AAYAAABsYWJlbHMAAGD///8IAAAARAAAADkAAAB7ImRpc3BsYXlOYW1lRnJvbURTIjoiZ29fZ29yb3V0aW5lc3tqb2I9XCJwcm9tZXRoZXVzXCJ9In0AAAAGAAAAY29uZmlnAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAmAAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAAkAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIAAAAAAAAAEgAAAAAAAAACAAAAAAAAABQAAAAAAAAAEgAAAAAAAAAAAAAAAIAAAAJAAAAAAAAAAAAAAAAAAAACQAAAAAAAAAGAAAAAAAAAABEFRTUKckWAA6wT9QpyRYA2EqL1CnJFgCi5cbUKckWAGyAAtUpyRYANhs+1SnJFgAAtnnVKckWAMpQtdUpyRYAlOvw1SnJFpgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANUAAAAAAAABAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBFQAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAAJgCAAAAAAAAwAAAAAAAAACYAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAACwAAAAAwAAAGgAAAAoAAAABAAAAAz+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAALP7//wgAAAAoAAAAHwAAAGdvX2dvcm91dGluZXN7am9iPSJwcm9tZXRoZXVzIn0ABAAAAG5hbWUAAAAAaP7//wgAAAAsAAAAIgAAAHsiY3VzdG9tIjp7InJlc3VsdFR5cGUiOiJtYXRyaXgifX0AAAQAAABtZXRhAAAAAAIAAAA0AQAAGAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAADoAAAA6AAAAAAAAwHoAAAAAwAAAHwAAAAsAAAABAAAAPD+//8IAAAAEAAAAAUAAABWYWx1ZQAAAAQAAABuYW1lAAAAABT///8IAAAAOAAAAC8AAAB7Il9fbmFtZV9fIjoiZ29fZ29yb3V0aW5lcyIsImpvYiI6InByb21ldGhldXMifQAGAAAAbGFiZWxzAABg////CAAAAEQAAAA5AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6ImdvX2dvcm91dGluZXN7am9iPVwicHJvbWV0aGV1c1wifSJ9AAAABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAFZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAABUaW1lAAAAALACAABBUlJPVzE= diff --git a/pkg/tsdb/prometheus/testdata/range_missing.result.json b/pkg/tsdb/prometheus/testdata/range_missing.result.json new file mode 100644 index 00000000000..9ace3676baf --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_missing.result.json @@ -0,0 +1,16 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { "__name__": "go_goroutines", "job": "prometheus" }, + "values": [ + [1641889533, "21"], + [1641889534, "32"], + [1641889537, "43"] + ] + } + ] + } +} diff --git a/pkg/tsdb/prometheus/testdata/range_nan.query.json b/pkg/tsdb/prometheus/testdata/range_nan.query.json new file mode 100644 index 00000000000..6a5698600d2 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_nan.query.json @@ -0,0 +1,7 @@ +{ + "RefId": "A", + "RangeQuery": true, + "Start": 1641889530, + "End": 1641889532, + "Step": 1 +} diff --git a/pkg/tsdb/prometheus/testdata/range_nan.result.golden.txt b/pkg/tsdb/prometheus/testdata/range_nan.result.golden.txt new file mode 100644 index 00000000000..f4d34e2a377 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_nan.result.golden.txt @@ -0,0 +1,22 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] { + "custom": { + "resultType": "matrix" + } +} +Name: {handler="/api/v1/query_range", job="prometheus"} +Dimensions: 2 Fields by 3 Rows ++-------------------------------+-----------------------------------------------------+ +| Name: Time | Name: Value | +| Labels: | Labels: handler=/api/v1/query_range, job=prometheus | +| Type: []time.Time | Type: []*float64 | ++-------------------------------+-----------------------------------------------------+ +| 2022-01-11 08:25:30 +0000 UTC | null | +| 2022-01-11 08:25:31 +0000 UTC | null | +| 2022-01-11 08:25:32 +0000 UTC | null | ++-------------------------------+-----------------------------------------------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////uAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAMQAAAADAAAAfAAAACgAAAAEAAAA3P3//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAAD8/f//CAAAADwAAAAxAAAAe2hhbmRsZXI9Ii9hcGkvdjEvcXVlcnlfcmFuZ2UiLCBqb2I9InByb21ldGhldXMifQAAAAQAAABuYW1lAAAAAEz+//8IAAAALAAAACIAAAB7ImN1c3RvbSI6eyJyZXN1bHRUeXBlIjoibWF0cml4In19AAAEAAAAbWV0YQAAAAACAAAAUAEAABgAAAAAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAABAEAAAQBAAAAAAMBBAEAAAMAAACEAAAALAAAAAQAAADU/v//CAAAABAAAAAFAAAAVmFsdWUAAAAEAAAAbmFtZQAAAAD4/v//CAAAAEAAAAA0AAAAeyJoYW5kbGVyIjoiL2FwaS92MS9xdWVyeV9yYW5nZSIsImpvYiI6InByb21ldGhldXMifQAAAAAGAAAAbGFiZWxzAABM////CAAAAFgAAABNAAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6IntoYW5kbGVyPVwiL2FwaS92MS9xdWVyeV9yYW5nZVwiLCBqb2I9XCJwcm9tZXRoZXVzXCJ9In0AAAAGAAAAY29uZmlnAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAOAAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAAMAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAABgAAAAAAAAACAAAAAAAAAAgAAAAAAAAABgAAAAAAAAAAAAAAAIAAAADAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAADAAAAAAAAAABEFRTUKckWAA6wT9QpyRYA2EqL1CnJFgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAAMgCAAAAAAAAwAAAAAAAAAA4AAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAADEAAAAAwAAAHwAAAAoAAAABAAAANz9//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAA/P3//wgAAAA8AAAAMQAAAHtoYW5kbGVyPSIvYXBpL3YxL3F1ZXJ5X3JhbmdlIiwgam9iPSJwcm9tZXRoZXVzIn0AAAAEAAAAbmFtZQAAAABM/v//CAAAACwAAAAiAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6Im1hdHJpeCJ9fQAABAAAAG1ldGEAAAAAAgAAAFABAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAAQBAAAEAQAAAAADAQQBAAADAAAAhAAAACwAAAAEAAAA1P7//wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAA+P7//wgAAABAAAAANAAAAHsiaGFuZGxlciI6Ii9hcGkvdjEvcXVlcnlfcmFuZ2UiLCJqb2IiOiJwcm9tZXRoZXVzIn0AAAAABgAAAGxhYmVscwAATP///wgAAABYAAAATQAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJ7aGFuZGxlcj1cIi9hcGkvdjEvcXVlcnlfcmFuZ2VcIiwgam9iPVwicHJvbWV0aGV1c1wifSJ9AAAABgAAAGNvbmZpZwAAAAAAAIr///8AAAIABQAAAFZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAAKTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAABUaW1lAAAAAOACAABBUlJPVzE= diff --git a/pkg/tsdb/prometheus/testdata/range_nan.result.json b/pkg/tsdb/prometheus/testdata/range_nan.result.json new file mode 100644 index 00000000000..06517c9a488 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_nan.result.json @@ -0,0 +1,16 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { "handler": "/api/v1/query_range", "job": "prometheus" }, + "values": [ + [1641889530, "NaN"], + [1641889531, "NaN"], + [1641889532, "NaN"] + ] + } + ] + } +} diff --git a/pkg/tsdb/prometheus/testdata/range_simple.query.json b/pkg/tsdb/prometheus/testdata/range_simple.query.json new file mode 100644 index 00000000000..6a5698600d2 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_simple.query.json @@ -0,0 +1,7 @@ +{ + "RefId": "A", + "RangeQuery": true, + "Start": 1641889530, + "End": 1641889532, + "Step": 1 +} diff --git a/pkg/tsdb/prometheus/testdata/range_simple.result.golden.txt b/pkg/tsdb/prometheus/testdata/range_simple.result.golden.txt new file mode 100644 index 00000000000..7866187f0e6 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_simple.result.golden.txt @@ -0,0 +1,42 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] { + "custom": { + "resultType": "matrix" + } +} +Name: prometheus_http_requests_total{code="200", handler="/api/v1/query_range", job="prometheus"} +Dimensions: 2 Fields by 3 Rows ++-------------------------------+--------------------------------------------------------------------------------------------------------+ +| Name: Time | Name: Value | +| Labels: | Labels: __name__=prometheus_http_requests_total, code=200, handler=/api/v1/query_range, job=prometheus | +| Type: []time.Time | Type: []*float64 | ++-------------------------------+--------------------------------------------------------------------------------------------------------+ +| 2022-01-11 08:25:30 +0000 UTC | 21 | +| 2022-01-11 08:25:31 +0000 UTC | 32 | +| 2022-01-11 08:25:32 +0000 UTC | 43 | ++-------------------------------+--------------------------------------------------------------------------------------------------------+ + + + +Frame[1] { + "custom": { + "resultType": "matrix" + } +} +Name: prometheus_http_requests_total{code="400", handler="/api/v1/query_range", job="prometheus"} +Dimensions: 2 Fields by 3 Rows ++-------------------------------+--------------------------------------------------------------------------------------------------------+ +| Name: Time | Name: Value | +| Labels: | Labels: __name__=prometheus_http_requests_total, code=400, handler=/api/v1/query_range, job=prometheus | +| Type: []time.Time | Type: []*float64 | ++-------------------------------+--------------------------------------------------------------------------------------------------------+ +| 2022-01-11 08:25:30 +0000 UTC | 54 | +| 2022-01-11 08:25:31 +0000 UTC | 65 | +| 2022-01-11 08:25:32 +0000 UTC | 76 | ++-------------------------------+--------------------------------------------------------------------------------------------------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////QAMAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAOwAAAADAAAApAAAACgAAAAEAAAAUP3//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABw/f//CAAAAGQAAABbAAAAcHJvbWV0aGV1c19odHRwX3JlcXVlc3RzX3RvdGFse2NvZGU9IjIwMCIsIGhhbmRsZXI9Ii9hcGkvdjEvcXVlcnlfcmFuZ2UiLCBqb2I9InByb21ldGhldXMifQAEAAAAbmFtZQAAAADo/f//CAAAACwAAAAiAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6Im1hdHJpeCJ9fQAABAAAAG1ldGEAAAAAAgAAALQBAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAGgBAABoAQAAAAADAWgBAAADAAAAvAAAACwAAAAEAAAAcP7//wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAlP7//wgAAAB4AAAAbQAAAHsiX19uYW1lX18iOiJwcm9tZXRoZXVzX2h0dHBfcmVxdWVzdHNfdG90YWwiLCJjb2RlIjoiMjAwIiwiaGFuZGxlciI6Ii9hcGkvdjEvcXVlcnlfcmFuZ2UiLCJqb2IiOiJwcm9tZXRoZXVzIn0AAAAGAAAAbGFiZWxzAAAg////CAAAAIQAAAB5AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6InByb21ldGhldXNfaHR0cF9yZXF1ZXN0c190b3RhbHtjb2RlPVwiMjAwXCIsIGhhbmRsZXI9XCIvYXBpL3YxL3F1ZXJ5X3JhbmdlXCIsIGpvYj1cInByb21ldGhldXNcIn0ifQAAAAYAAABjb25maWcAAAAAAACK////AAACAAUAAABWYWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAD/////uAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAADAAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAFgAAAADAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAYAAAAAAAAAAAAAAACAAAAAwAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAARBUU1CnJFgAOsE/UKckWANhKi9QpyRYAAAAAAAA1QAAAAAAAAEBAAAAAAACARUAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAUAMAAAAAAADAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAADsAAAAAwAAAKQAAAAoAAAABAAAAFD9//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAcP3//wgAAABkAAAAWwAAAHByb21ldGhldXNfaHR0cF9yZXF1ZXN0c190b3RhbHtjb2RlPSIyMDAiLCBoYW5kbGVyPSIvYXBpL3YxL3F1ZXJ5X3JhbmdlIiwgam9iPSJwcm9tZXRoZXVzIn0ABAAAAG5hbWUAAAAA6P3//wgAAAAsAAAAIgAAAHsiY3VzdG9tIjp7InJlc3VsdFR5cGUiOiJtYXRyaXgifX0AAAQAAABtZXRhAAAAAAIAAAC0AQAAGAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABoAQAAaAEAAAAAAwFoAQAAAwAAALwAAAAsAAAABAAAAHD+//8IAAAAEAAAAAUAAABWYWx1ZQAAAAQAAABuYW1lAAAAAJT+//8IAAAAeAAAAG0AAAB7Il9fbmFtZV9fIjoicHJvbWV0aGV1c19odHRwX3JlcXVlc3RzX3RvdGFsIiwiY29kZSI6IjIwMCIsImhhbmRsZXIiOiIvYXBpL3YxL3F1ZXJ5X3JhbmdlIiwiam9iIjoicHJvbWV0aGV1cyJ9AAAABgAAAGxhYmVscwAAIP///wgAAACEAAAAeQAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJwcm9tZXRoZXVzX2h0dHBfcmVxdWVzdHNfdG90YWx7Y29kZT1cIjIwMFwiLCBoYW5kbGVyPVwiL2FwaS92MS9xdWVyeV9yYW5nZVwiLCBqb2I9XCJwcm9tZXRoZXVzXCJ9In0AAAAGAAAAY29uZmlnAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAcAMAAEFSUk9XMQ== +FRAME=QVJST1cxAAD/////QAMAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAOwAAAADAAAApAAAACgAAAAEAAAAUP3//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABw/f//CAAAAGQAAABbAAAAcHJvbWV0aGV1c19odHRwX3JlcXVlc3RzX3RvdGFse2NvZGU9IjQwMCIsIGhhbmRsZXI9Ii9hcGkvdjEvcXVlcnlfcmFuZ2UiLCBqb2I9InByb21ldGhldXMifQAEAAAAbmFtZQAAAADo/f//CAAAACwAAAAiAAAAeyJjdXN0b20iOnsicmVzdWx0VHlwZSI6Im1hdHJpeCJ9fQAABAAAAG1ldGEAAAAAAgAAALQBAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAGgBAABoAQAAAAADAWgBAAADAAAAvAAAACwAAAAEAAAAcP7//wgAAAAQAAAABQAAAFZhbHVlAAAABAAAAG5hbWUAAAAAlP7//wgAAAB4AAAAbQAAAHsiX19uYW1lX18iOiJwcm9tZXRoZXVzX2h0dHBfcmVxdWVzdHNfdG90YWwiLCJjb2RlIjoiNDAwIiwiaGFuZGxlciI6Ii9hcGkvdjEvcXVlcnlfcmFuZ2UiLCJqb2IiOiJwcm9tZXRoZXVzIn0AAAAGAAAAbGFiZWxzAAAg////CAAAAIQAAAB5AAAAeyJkaXNwbGF5TmFtZUZyb21EUyI6InByb21ldGhldXNfaHR0cF9yZXF1ZXN0c190b3RhbHtjb2RlPVwiNDAwXCIsIGhhbmRsZXI9XCIvYXBpL3YxL3F1ZXJ5X3JhbmdlXCIsIGpvYj1cInByb21ldGhldXNcIn0ifQAAAAYAAABjb25maWcAAAAAAACK////AAACAAUAAABWYWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAACkwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAD/////uAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAADAAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAFgAAAADAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAYAAAAAAAAAAAAAAACAAAAAwAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAARBUU1CnJFgAOsE/UKckWANhKi9QpyRYAAAAAAABLQAAAAAAAQFBAAAAAAAAAU0AQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAUAMAAAAAAADAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAADsAAAAAwAAAKQAAAAoAAAABAAAAFD9//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAcP3//wgAAABkAAAAWwAAAHByb21ldGhldXNfaHR0cF9yZXF1ZXN0c190b3RhbHtjb2RlPSI0MDAiLCBoYW5kbGVyPSIvYXBpL3YxL3F1ZXJ5X3JhbmdlIiwgam9iPSJwcm9tZXRoZXVzIn0ABAAAAG5hbWUAAAAA6P3//wgAAAAsAAAAIgAAAHsiY3VzdG9tIjp7InJlc3VsdFR5cGUiOiJtYXRyaXgifX0AAAQAAABtZXRhAAAAAAIAAAC0AQAAGAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABoAQAAaAEAAAAAAwFoAQAAAwAAALwAAAAsAAAABAAAAHD+//8IAAAAEAAAAAUAAABWYWx1ZQAAAAQAAABuYW1lAAAAAJT+//8IAAAAeAAAAG0AAAB7Il9fbmFtZV9fIjoicHJvbWV0aGV1c19odHRwX3JlcXVlc3RzX3RvdGFsIiwiY29kZSI6IjQwMCIsImhhbmRsZXIiOiIvYXBpL3YxL3F1ZXJ5X3JhbmdlIiwiam9iIjoicHJvbWV0aGV1cyJ9AAAABgAAAGxhYmVscwAAIP///wgAAACEAAAAeQAAAHsiZGlzcGxheU5hbWVGcm9tRFMiOiJwcm9tZXRoZXVzX2h0dHBfcmVxdWVzdHNfdG90YWx7Y29kZT1cIjQwMFwiLCBoYW5kbGVyPVwiL2FwaS92MS9xdWVyeV9yYW5nZVwiLCBqb2I9XCJwcm9tZXRoZXVzXCJ9In0AAAAGAAAAY29uZmlnAAAAAAAAiv///wAAAgAFAAAAVmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAcAMAAEFSUk9XMQ== diff --git a/pkg/tsdb/prometheus/testdata/range_simple.result.json b/pkg/tsdb/prometheus/testdata/range_simple.result.json new file mode 100644 index 00000000000..3c510aca9a3 --- /dev/null +++ b/pkg/tsdb/prometheus/testdata/range_simple.result.json @@ -0,0 +1,34 @@ +{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "prometheus_http_requests_total", + "code": "200", + "handler": "/api/v1/query_range", + "job": "prometheus" + }, + "values": [ + [1641889530, "21"], + [1641889531, "32"], + [1641889532, "43"] + ] + }, + { + "metric": { + "__name__": "prometheus_http_requests_total", + "code": "400", + "handler": "/api/v1/query_range", + "job": "prometheus" + }, + "values": [ + [1641889530, "54"], + [1641889531, "65"], + [1641889532, "76"] + ] + } + ] + } +} diff --git a/pkg/tsdb/prometheus/time_series_query.go b/pkg/tsdb/prometheus/time_series_query.go index 83a8adac489..f86638771b8 100644 --- a/pkg/tsdb/prometheus/time_series_query.go +++ b/pkg/tsdb/prometheus/time_series_query.go @@ -13,9 +13,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/tsdb/intervalv2" - "github.com/opentracing/opentracing-go" apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" + "go.opentelemetry.io/otel/attribute" ) //Internal interval and range variables @@ -47,29 +47,19 @@ const ( ExemplarQueryType TimeSeriesQueryType = "exemplar" ) -func (s *Service) executeTimeSeriesQuery(ctx context.Context, req *backend.QueryDataRequest, dsInfo *DatasourceInfo) (*backend.QueryDataResponse, error) { - client, err := dsInfo.getClient(req.Headers) - if err != nil { - return nil, err - } - +func (s *Service) runQueries(ctx context.Context, client apiv1.API, queries []*PrometheusQuery, fillNulls bool) (*backend.QueryDataResponse, error) { result := backend.QueryDataResponse{ Responses: backend.Responses{}, } - queries, err := s.parseTimeSeriesQuery(req, dsInfo) - if err != nil { - return &result, err - } - for _, query := range queries { plog.Debug("Sending query", "start", query.Start, "end", query.End, "step", query.Step, "query", query.Expr) - span, ctx := opentracing.StartSpanFromContext(ctx, "datasource.prometheus") - span.SetTag("expr", query.Expr) - span.SetTag("start_unixnano", query.Start.UnixNano()) - span.SetTag("stop_unixnano", query.End.UnixNano()) - defer span.Finish() + ctx, span := s.tracer.Start(ctx, "datasource.prometheus") + span.SetAttributes("expr", query.Expr, attribute.Key("expr").String(query.Expr)) + span.SetAttributes("start_unixnano", query.Start, attribute.Key("start_unixnano").Int64(query.Start.UnixNano())) + span.SetAttributes("stop_unixnano", query.End, attribute.Key("stop_unixnano").Int64(query.End.UnixNano())) + defer span.End() response := make(map[TimeSeriesQueryType]interface{}) @@ -111,7 +101,7 @@ func (s *Service) executeTimeSeriesQuery(ctx context.Context, req *backend.Query } } - frames, err := parseTimeSeriesResponse(response, query) + frames, err := parseTimeSeriesResponse(response, query, fillNulls) if err != nil { return &result, err } @@ -124,6 +114,28 @@ func (s *Service) executeTimeSeriesQuery(ctx context.Context, req *backend.Query return &result, nil } +func (s *Service) executeTimeSeriesQuery(ctx context.Context, req *backend.QueryDataRequest, dsInfo *DatasourceInfo) (*backend.QueryDataResponse, error) { + client, err := dsInfo.getClient(req.Headers) + if err != nil { + return nil, err + } + + queries, err := s.parseTimeSeriesQuery(req, dsInfo) + if err != nil { + result := backend.QueryDataResponse{ + Responses: backend.Responses{}, + } + return &result, err + } + + fillNulls := true + if req.Headers["FromAlert"] == "true" { + fillNulls = false + } + + return s.runQueries(ctx, client, queries, fillNulls) +} + func formatLegend(metric model.Metric, query *PrometheusQuery) string { var legend string @@ -159,48 +171,14 @@ func (s *Service) parseTimeSeriesQuery(queryContext *backend.QueryDataRequest, d return nil, err } //Final interval value - var interval time.Duration - - //Calculate interval - queryInterval := model.Interval - //If we are using variable or interval/step, we will replace it with calculated interval - if queryInterval == varInterval || queryInterval == varIntervalMs || queryInterval == varRateInterval { - queryInterval = "" - } - //If we are using variable or interval/step with {} syntax, we will replace it with calculated interval - //Repetitive code, we should have functionality to unify these - if queryInterval == varIntervalAlt || queryInterval == varIntervalMsAlt || queryInterval == varRateIntervalAlt { - queryInterval = "" - } - - minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, model.IntervalMS, 15*time.Second) + interval, err := calculatePrometheusInterval(model, dsInfo, query, s.intervalCalculator) if err != nil { return nil, err } - calculatedInterval := s.intervalCalculator.Calculate(query.TimeRange, minInterval, query.MaxDataPoints) - safeInterval := s.intervalCalculator.CalculateSafeInterval(query.TimeRange, int64(safeRes)) - adjustedInterval := safeInterval.Value - - if calculatedInterval.Value > safeInterval.Value { - adjustedInterval = calculatedInterval.Value - } - - if queryInterval == varRateInterval || queryInterval == varRateIntervalAlt { - // Rate interval is final and is not affected by resolution - interval = calculateRateInterval(adjustedInterval, dsInfo.TimeInterval, s.intervalCalculator) - } else { - intervalFactor := model.IntervalFactor - if intervalFactor == 0 { - intervalFactor = 1 - } - interval = time.Duration(int64(adjustedInterval) * intervalFactor) - } - // Interpolate variables in expr timeRange := query.TimeRange.To.Sub(query.TimeRange.From) - expr := interpolateVariables(model.Expr, interval, timeRange, s.intervalCalculator, dsInfo.TimeInterval) - + expr := interpolateVariables(model, interval, timeRange, s.intervalCalculator, dsInfo.TimeInterval) rangeQuery := model.RangeQuery if !model.InstantQuery && !model.RangeQuery { // In older dashboards, we were not setting range query param and !range && !instant was run as range query @@ -229,7 +207,7 @@ func (s *Service) parseTimeSeriesQuery(queryContext *backend.QueryDataRequest, d return qs, nil } -func parseTimeSeriesResponse(value map[TimeSeriesQueryType]interface{}, query *PrometheusQuery) (data.Frames, error) { +func parseTimeSeriesResponse(value map[TimeSeriesQueryType]interface{}, query *PrometheusQuery, fillNulls bool) (data.Frames, error) { var ( frames = data.Frames{} nextFrames = data.Frames{} @@ -241,7 +219,11 @@ func parseTimeSeriesResponse(value map[TimeSeriesQueryType]interface{}, query *P switch v := value.(type) { case model.Matrix: - nextFrames = matrixToDataFrames(v, query, nextFrames) + if fillNulls { + nextFrames = matrixToDataFramesWithNullFill(v, query, nextFrames) + } else { + nextFrames = matrixToDataFrames(v, query, nextFrames) + } case model.Vector: nextFrames = vectorToDataFrames(v, query, nextFrames) case *model.Scalar: @@ -259,6 +241,38 @@ func parseTimeSeriesResponse(value map[TimeSeriesQueryType]interface{}, query *P return frames, nil } +func calculatePrometheusInterval(model *QueryModel, dsInfo *DatasourceInfo, query backend.DataQuery, intervalCalculator intervalv2.Calculator) (time.Duration, error) { + queryInterval := model.Interval + + //If we are using variable for interval/step, we will replace it with calculated interval + if isVariableInterval(queryInterval) { + queryInterval = "" + } + + minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, model.IntervalMS, 15*time.Second) + if err != nil { + return time.Duration(0), err + } + calculatedInterval := intervalCalculator.Calculate(query.TimeRange, minInterval, query.MaxDataPoints) + safeInterval := intervalCalculator.CalculateSafeInterval(query.TimeRange, int64(safeRes)) + + adjustedInterval := safeInterval.Value + if calculatedInterval.Value > safeInterval.Value { + adjustedInterval = calculatedInterval.Value + } + + if model.Interval == varRateInterval || model.Interval == varRateIntervalAlt { + // Rate interval is final and is not affected by resolution + return calculateRateInterval(adjustedInterval, dsInfo.TimeInterval, intervalCalculator), nil + } else { + intervalFactor := model.IntervalFactor + if intervalFactor == 0 { + intervalFactor = 1 + } + return time.Duration(int64(adjustedInterval) * intervalFactor), nil + } +} + func calculateRateInterval(interval time.Duration, scrapeInterval string, intervalCalculator intervalv2.Calculator) time.Duration { scrape := scrapeInterval if scrape == "" { @@ -274,16 +288,24 @@ func calculateRateInterval(interval time.Duration, scrapeInterval string, interv return rateInterval } -func interpolateVariables(expr string, interval time.Duration, timeRange time.Duration, intervalCalculator intervalv2.Calculator, timeInterval string) string { +func interpolateVariables(model *QueryModel, interval time.Duration, timeRange time.Duration, intervalCalculator intervalv2.Calculator, timeInterval string) string { + expr := model.Expr rangeMs := timeRange.Milliseconds() rangeSRounded := int64(math.Round(float64(rangeMs) / 1000.0)) + var rateInterval time.Duration + if model.Interval == varRateInterval || model.Interval == varRateIntervalAlt { + rateInterval = interval + } else { + rateInterval = calculateRateInterval(interval, timeInterval, intervalCalculator) + } + expr = strings.ReplaceAll(expr, varIntervalMs, strconv.FormatInt(int64(interval/time.Millisecond), 10)) expr = strings.ReplaceAll(expr, varInterval, intervalv2.FormatDuration(interval)) expr = strings.ReplaceAll(expr, varRangeMs, strconv.FormatInt(rangeMs, 10)) expr = strings.ReplaceAll(expr, varRangeS, strconv.FormatInt(rangeSRounded, 10)) expr = strings.ReplaceAll(expr, varRange, strconv.FormatInt(rangeSRounded, 10)+"s") - expr = strings.ReplaceAll(expr, varRateInterval, intervalv2.FormatDuration(calculateRateInterval(interval, timeInterval, intervalCalculator))) + expr = strings.ReplaceAll(expr, varRateInterval, rateInterval.String()) // Repetitive code, we should have functionality to unify these expr = strings.ReplaceAll(expr, varIntervalMsAlt, strconv.FormatInt(int64(interval/time.Millisecond), 10)) @@ -291,11 +313,11 @@ func interpolateVariables(expr string, interval time.Duration, timeRange time.Du expr = strings.ReplaceAll(expr, varRangeMsAlt, strconv.FormatInt(rangeMs, 10)) expr = strings.ReplaceAll(expr, varRangeSAlt, strconv.FormatInt(rangeSRounded, 10)) expr = strings.ReplaceAll(expr, varRangeAlt, strconv.FormatInt(rangeSRounded, 10)+"s") - expr = strings.ReplaceAll(expr, varRateIntervalAlt, intervalv2.FormatDuration(calculateRateInterval(interval, timeInterval, intervalCalculator))) + expr = strings.ReplaceAll(expr, varRateIntervalAlt, rateInterval.String()) return expr } -func matrixToDataFrames(matrix model.Matrix, query *PrometheusQuery, frames data.Frames) data.Frames { +func matrixToDataFramesWithNullFill(matrix model.Matrix, query *PrometheusQuery, frames data.Frames) data.Frames { for _, v := range matrix { tags := make(map[string]string, len(v.Metric)) for k, v := range v.Metric { @@ -328,6 +350,11 @@ func matrixToDataFrames(matrix model.Matrix, query *PrometheusQuery, frames data idx++ } + for t := baseTimestamp; t <= endTimestamp; t += query.Step.Milliseconds() { + timeField.Set(idx, time.Unix(0, t*1000000).UTC()) + idx++ + } + name := formatLegend(v.Metric, query) timeField.Name = data.TimeSeriesTimeFieldName valueField.Name = data.TimeSeriesValueFieldName @@ -340,6 +367,35 @@ func matrixToDataFrames(matrix model.Matrix, query *PrometheusQuery, frames data return frames } +func matrixToDataFrames(matrix model.Matrix, query *PrometheusQuery, frames data.Frames) data.Frames { + for _, v := range matrix { + tags := make(map[string]string, len(v.Metric)) + for k, v := range v.Metric { + tags[string(k)] = string(v) + } + timeField := data.NewFieldFromFieldType(data.FieldTypeTime, len(v.Values)) + valueField := data.NewFieldFromFieldType(data.FieldTypeNullableFloat64, len(v.Values)) + + for i, k := range v.Values { + timeField.Set(i, time.Unix(k.Timestamp.Unix(), 0).UTC()) + value := float64(k.Value) + + if !math.IsNaN(value) { + valueField.Set(i, &value) + } + } + + name := formatLegend(v.Metric, query) + timeField.Name = data.TimeSeriesTimeFieldName + valueField.Name = data.TimeSeriesValueFieldName + valueField.Config = &data.FieldConfig{DisplayNameFromDS: name} + valueField.Labels = tags + frames = append(frames, newDataFrame(name, "matrix", timeField, valueField)) + } + + return frames +} + func scalarToDataFrames(scalar *model.Scalar, query *PrometheusQuery, frames data.Frames) data.Frames { timeVector := []time.Time{time.Unix(scalar.Timestamp.Unix(), 0).UTC()} values := []float64{float64(scalar.Value)} @@ -525,3 +581,14 @@ func newDataFrame(name string, typ string, fields ...*data.Field) *data.Frame { func alignTimeRange(t time.Time, step time.Duration, offset int64) time.Time { return time.Unix(int64(math.Floor((float64(t.Unix()+offset)/step.Seconds()))*step.Seconds()-float64(offset)), 0) } + +func isVariableInterval(interval string) bool { + if interval == varInterval || interval == varIntervalMs || interval == varRateInterval { + return true + } + //Repetitive code, we should have functionality to unify these + if interval == varIntervalAlt || interval == varIntervalMsAlt || interval == varRateIntervalAlt { + return true + } + return false +} diff --git a/pkg/tsdb/prometheus/time_series_query_test.go b/pkg/tsdb/prometheus/time_series_query_test.go index b4b363acf2d..2f5ec14f160 100644 --- a/pkg/tsdb/prometheus/time_series_query_test.go +++ b/pkg/tsdb/prometheus/time_series_query_test.go @@ -424,13 +424,35 @@ func TestPrometheus_timeSeriesQuery_parseTimeSeriesQuery(t *testing.T) { "expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})", "format": "time_series", "intervalFactor": 1, + "interval": "5m", "refId": "A" }`, timeRange) dsInfo := &DatasourceInfo{} models, err := service.parseTimeSeriesQuery(query, dsInfo) require.NoError(t, err) - require.Equal(t, "rate(ALERTS{job=\"test\" [1m]})", models[0].Expr) + require.Equal(t, "rate(ALERTS{job=\"test\" [5m15s]})", models[0].Expr) + }) + + t.Run("parsing query model with $__rate_interval variable in expr and interval", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(5 * time.Minute), + } + + query := queryContext(`{ + "expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})", + "format": "time_series", + "intervalFactor": 1, + "interval": "$__rate_interval", + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseTimeSeriesQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, "rate(ALERTS{job=\"test\" [1m0s]})", models[0].Expr) + require.Equal(t, 1*time.Minute, models[0].Step) }) t.Run("parsing query model of range query", func(t *testing.T) { @@ -534,7 +556,7 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) { query := &PrometheusQuery{ LegendFormat: "legend {{app}}", } - res, err := parseTimeSeriesResponse(value, query) + res, err := parseTimeSeriesResponse(value, query, true) require.NoError(t, err) // Test fields @@ -572,7 +594,7 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) { End: time.Unix(5, 0).UTC(), UtcOffsetSec: 0, } - res, err := parseTimeSeriesResponse(value, query) + res, err := parseTimeSeriesResponse(value, query, true) require.NoError(t, err) require.Len(t, res, 1) @@ -609,7 +631,7 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) { End: time.Unix(4, 0).UTC(), UtcOffsetSec: 0, } - res, err := parseTimeSeriesResponse(value, query) + res, err := parseTimeSeriesResponse(value, query, true) require.NoError(t, err) require.Len(t, res, 1) @@ -621,6 +643,39 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) { require.Nil(t, res[0].Fields[1].At(2)) }) + t.Run("matrix response with from alerting missed data points should be parsed correctly", func(t *testing.T) { + values := []p.SamplePair{ + {Value: 1, Timestamp: 1000}, + {Value: 4, Timestamp: 4000}, + } + value := make(map[TimeSeriesQueryType]interface{}) + value[RangeQueryType] = p.Matrix{ + &p.SampleStream{ + Metric: p.Metric{"app": "Application", "tag2": "tag2"}, + Values: values, + }, + } + query := &PrometheusQuery{ + LegendFormat: "", + Step: 1 * time.Second, + Start: time.Unix(1, 0).UTC(), + End: time.Unix(4, 0).UTC(), + UtcOffsetSec: 0, + } + res, err := parseTimeSeriesResponse(value, query, false) + + require.NoError(t, err) + require.Len(t, res, 1) + require.Equal(t, res[0].Name, "{app=\"Application\", tag2=\"tag2\"}") + require.Len(t, res[0].Fields, 2) + require.Len(t, res[0].Fields[0].Labels, 0) + require.Equal(t, res[0].Fields[0].Name, "Time") + require.Len(t, res[0].Fields[1].Labels, 2) + require.Equal(t, res[0].Fields[1].Labels.String(), "app=Application, tag2=tag2") + require.Equal(t, res[0].Fields[1].Name, "Value") + require.Equal(t, res[0].Fields[1].Config.DisplayNameFromDS, "{app=\"Application\", tag2=\"tag2\"}") + }) + t.Run("matrix response with NaN value should be changed to null", func(t *testing.T) { value := make(map[TimeSeriesQueryType]interface{}) value[RangeQueryType] = p.Matrix{ @@ -638,7 +693,7 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) { End: time.Unix(4, 0).UTC(), UtcOffsetSec: 0, } - res, err := parseTimeSeriesResponse(value, query) + res, err := parseTimeSeriesResponse(value, query, true) require.NoError(t, err) var nilPointer *float64 @@ -658,7 +713,7 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) { query := &PrometheusQuery{ LegendFormat: "legend {{app}}", } - res, err := parseTimeSeriesResponse(value, query) + res, err := parseTimeSeriesResponse(value, query, true) require.NoError(t, err) require.Len(t, res, 1) @@ -685,7 +740,7 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) { } query := &PrometheusQuery{} - res, err := parseTimeSeriesResponse(value, query) + res, err := parseTimeSeriesResponse(value, query, true) require.NoError(t, err) require.Len(t, res, 1) diff --git a/pkg/tsdb/sqleng/sql_engine.go b/pkg/tsdb/sqleng/sql_engine.go index fbfb933c8e1..51c286201ad 100644 --- a/pkg/tsdb/sqleng/sql_engine.go +++ b/pkg/tsdb/sqleng/sql_engine.go @@ -56,11 +56,13 @@ type JsonData struct { Timescaledb bool `json:"timescaledb"` Mode string `json:"sslmode"` ConfigurationMethod string `json:"tlsConfigurationMethod"` + TlsSkipVerify bool `json:"tlsSkipVerify"` RootCertFile string `json:"sslRootCertFile"` CertFile string `json:"sslCertFile"` CertKeyFile string `json:"sslKeyFile"` Timezone string `json:"timezone"` Encrypt string `json:"encrypt"` + Servername string `json:"servername"` TimeInterval string `json:"timeInterval"` } diff --git a/pkg/tsdb/tempo/tempo.go b/pkg/tsdb/tempo/tempo.go index 49ce72dcacd..866f68276e4 100644 --- a/pkg/tsdb/tempo/tempo.go +++ b/pkg/tsdb/tempo/tempo.go @@ -13,38 +13,19 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" - "github.com/grafana/grafana/pkg/setting" "go.opentelemetry.io/collector/model/otlp" ) -const pluginID = "tempo" - type Service struct { im instancemgmt.InstanceManager tlog log.Logger } -func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, pluginStore plugins.Store) (*Service, error) { - im := datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)) - - s := &Service{ +func ProvideService(httpClientProvider httpclient.Provider) *Service { + return &Service{ tlog: log.New("tsdb.tempo"), - im: im, + im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider)), } - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s, - }) - - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - if err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver); err != nil { - s.tlog.Error("Failed to register plugin", "error", err) - return nil, err - } - - return s, nil } type datasourceInfo struct { diff --git a/pkg/tsdb/testdatasource/resource_handler.go b/pkg/tsdb/testdatasource/resource_handler.go index 2eb5ff1029d..76bbed47e42 100644 --- a/pkg/tsdb/testdatasource/resource_handler.go +++ b/pkg/tsdb/testdatasource/resource_handler.go @@ -15,13 +15,15 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter" ) -func (s *Service) RegisterRoutes(mux *http.ServeMux) { +func (s *Service) registerRoutes() *http.ServeMux { + mux := http.NewServeMux() mux.HandleFunc("/", s.testGetHandler) mux.HandleFunc("/scenarios", s.getScenariosHandler) mux.HandleFunc("/stream", s.testStreamHandler) mux.Handle("/test", createJSONHandler(s.logger)) mux.Handle("/test/json", createJSONHandler(s.logger)) mux.HandleFunc("/boom", s.testPanicHandler) + return mux } func (s *Service) testGetHandler(rw http.ResponseWriter, req *http.Request) { diff --git a/pkg/tsdb/testdatasource/scenarios.go b/pkg/tsdb/testdatasource/scenarios.go index c73377f9382..b93cfe0ae9b 100644 --- a/pkg/tsdb/testdatasource/scenarios.go +++ b/pkg/tsdb/testdatasource/scenarios.go @@ -195,7 +195,7 @@ Timestamps will line up evenly on timeStepSeconds (For example, 60 seconds means s.registerScenario(&Scenario{ ID: string(rawFrameQuery), - Name: "Raw Frame", + Name: "Raw Frames", }) s.registerScenario(&Scenario{ diff --git a/pkg/tsdb/testdatasource/stream_handler.go b/pkg/tsdb/testdatasource/stream_handler.go index 0b49ea2d24c..c71b9e3b03c 100644 --- a/pkg/tsdb/testdatasource/stream_handler.go +++ b/pkg/tsdb/testdatasource/stream_handler.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/services/featuremgmt" ) var random20HzStreamRegex = regexp.MustCompile(`random-20Hz-stream(-\d+)?`) @@ -37,7 +38,7 @@ func (s *Service) SubscribeStream(_ context.Context, req *backend.SubscribeStrea } } - if s.cfg.FeatureToggles["live-pipeline"] { + if s.features.IsEnabled(featuremgmt.FlagLivePipeline) { // While developing Live pipeline avoid sending initial data. initialData = nil } @@ -126,7 +127,7 @@ func (s *Service) runTestStream(ctx context.Context, path string, conf testStrea } mode := data.IncludeDataOnly - if s.cfg.FeatureToggles["live-pipeline"] { + if s.features.IsEnabled(featuremgmt.FlagLivePipeline) { mode = data.IncludeAll } diff --git a/pkg/tsdb/testdatasource/testdata.go b/pkg/tsdb/testdatasource/testdata.go index 6242d4f8a70..b99b735d046 100644 --- a/pkg/tsdb/testdatasource/testdata.go +++ b/pkg/tsdb/testdatasource/testdata.go @@ -2,23 +2,21 @@ package testdatasource import ( "context" - "net/http" "time" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" ) -const pluginID = "testdata" - -func ProvideService(cfg *setting.Cfg, pluginStore plugins.Store) (*Service, error) { +func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles) *Service { s := &Service{ + features: features, queryMux: datasource.NewQueryTypeMux(), scenarios: map[string]*Scenario{}, frame: data.NewFrame("testdata", @@ -37,29 +35,26 @@ func ProvideService(cfg *setting.Cfg, pluginStore plugins.Store) (*Service, erro } s.registerScenarios() + s.resourceHandler = httpadapter.New(s.registerRoutes()) - rMux := http.NewServeMux() - s.RegisterRoutes(rMux) - - factory := coreplugin.New(backend.ServeOpts{ - QueryDataHandler: s.queryMux, - CallResourceHandler: httpadapter.New(rMux), - StreamHandler: s, - }) - resolver := plugins.CoreDataSourcePathResolver(cfg, pluginID) - err := pluginStore.AddWithFactory(context.Background(), pluginID, factory, resolver) - if err != nil { - return nil, err - } - - return s, nil + return s } type Service struct { - cfg *setting.Cfg - logger log.Logger - scenarios map[string]*Scenario - frame *data.Frame - labelFrame *data.Frame - queryMux *datasource.QueryTypeMux + cfg *setting.Cfg + logger log.Logger + scenarios map[string]*Scenario + frame *data.Frame + labelFrame *data.Frame + queryMux *datasource.QueryTypeMux + resourceHandler backend.CallResourceHandler + features featuremgmt.FeatureToggles +} + +func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + return s.queryMux.QueryData(ctx, req) +} + +func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + return s.resourceHandler.CallResource(ctx, req, sender) } diff --git a/pkg/web/binding.go b/pkg/web/binding.go index 6bd47cc3994..94862b30621 100644 --- a/pkg/web/binding.go +++ b/pkg/web/binding.go @@ -26,14 +26,27 @@ type Validator interface { } func validate(obj interface{}) error { + // First check if obj is nil, because we cannot validate those. + if obj == nil { + return nil + } + + // Second, check if obj has a nil interface value. + // This is to prevent panics when obj is an instance of uninitialised struct pointer / interface. + t := reflect.TypeOf(obj) + v := reflect.ValueOf(obj) + + if v.Kind() == reflect.Ptr && v.IsNil() { + return nil + } + // If type has a Validate() method - use that if validator, ok := obj.(Validator); ok { return validator.Validate() } + // Otherwise, use reflection to match `binding:"Required"` struct field tags. // Resolve all pointers and interfaces, until we get a concrete type. - t := reflect.TypeOf(obj) - v := reflect.ValueOf(obj) for v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr { t = t.Elem() v = v.Elem() diff --git a/pkg/web/binding_test.go b/pkg/web/binding_test.go index c0ccbd13f50..38d983bd0e2 100644 --- a/pkg/web/binding_test.go +++ b/pkg/web/binding_test.go @@ -50,8 +50,22 @@ func (sv StructWithValidation) Validate() error { return nil } +type StructWithPointerValidation struct { + A int +} + +func (sv *StructWithPointerValidation) Validate() error { + if sv.A < 10 { + return errors.New("too small") + } + return nil +} + func TestValidationSuccess(t *testing.T) { + var nilInterface *StructWithPointerValidation + for _, x := range []interface{}{ + nil, 42, "foo", struct{ A int }{}, @@ -65,6 +79,8 @@ func TestValidationSuccess(t *testing.T) { StructWithStruct{StructWithInt{3}}, StructWithStructPointer{&StructWithInt{3}}, StructWithValidation{42}, + &StructWithPointerValidation{42}, + nilInterface, } { if err := validate(x); err != nil { t.Error("Validation failed:", x, err) @@ -93,6 +109,7 @@ func TestValidationFailure(t *testing.T) { StructWithStructPointer{}, StructWithStructPointer{&StructWithInt{}}, StructWithValidation{2}, + &StructWithPointerValidation{2}, } { if err := validate(x); err == nil { t.Error("Validation should fail:", i, x) diff --git a/pkg/web/context.go b/pkg/web/context.go index 9cbc85de99b..b9b05145ead 100644 --- a/pkg/web/context.go +++ b/pkg/web/context.go @@ -179,13 +179,6 @@ func (ctx *Context) QueryInt64(name string) int64 { return n } -// ParamsInt64 returns params result in int64 type. -// e.g. ctx.ParamsInt64(":uid") -func (ctx *Context) ParamsInt64(name string) int64 { - n, _ := strconv.ParseInt(Params(ctx.Req)[name], 10, 64) - return n -} - // GetCookie returns given cookie value from request header. func (ctx *Context) GetCookie(name string) string { cookie, err := ctx.Req.Cookie(name) diff --git a/pkg/web/webtest/webtest.go b/pkg/web/webtest/webtest.go new file mode 100644 index 00000000000..d7c480c9b28 --- /dev/null +++ b/pkg/web/webtest/webtest.go @@ -0,0 +1,128 @@ +package webtest + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/web" +) + +var requests = map[string]*models.ReqContext{} + +type Server struct { + t testing.TB + Mux *web.Mux + RouteRegister routing.RouteRegister + TestServer *httptest.Server +} + +// NewServer starts and returns a new server. +func NewServer(t testing.TB, routeRegister routing.RouteRegister) *Server { + t.Helper() + + m := web.New() + initCtx := &models.ReqContext{} + m.Use(func(c *web.Context) { + initCtx.Context = c + initCtx.Logger = log.New("api-test") + c.Map(initCtx) + }) + + m.Use(requestContextMiddleware()) + + routeRegister.Register(m.Router) + testServer := httptest.NewServer(m) + t.Cleanup(testServer.Close) + + return &Server{ + t: t, + RouteRegister: routeRegister, + Mux: m, + TestServer: testServer, + } +} + +// NewRequest creates a new request setup for test. +func (s *Server) NewRequest(method string, target string, body io.Reader) *http.Request { + s.t.Helper() + + if !strings.HasPrefix(target, "/") { + target = "/" + target + } + + target = s.TestServer.URL + target + req := httptest.NewRequest(method, target, body) + reqID := generateRequestIdentifier() + req = requestWithRequestIdentifier(req, reqID) + req.RequestURI = "" + return req +} + +// Send sends an HTTP request to the test server and returns an HTTP response +func (s *Server) Send(req *http.Request) (*http.Response, error) { + return http.DefaultClient.Do(req) +} + +func generateRequestIdentifier() string { + return uuid.NewString() +} + +func requestWithRequestIdentifier(req *http.Request, id string) *http.Request { + req.Header.Set("X-GRAFANA-WEB-TEST-ID", id) + return req +} + +func requestIdentifierFromRequest(req *http.Request) string { + return req.Header.Get("X-GRAFANA-WEB-TEST-ID") +} + +func RequestWithWebContext(req *http.Request, c *models.ReqContext) *http.Request { + reqID := requestIdentifierFromRequest(req) + requests[reqID] = c + return req +} + +func RequestWithSignedInUser(req *http.Request, user *models.SignedInUser) *http.Request { + return RequestWithWebContext(req, &models.ReqContext{ + SignedInUser: &models.SignedInUser{}, + IsSignedIn: true, + }) +} + +func requestContextFromRequest(req *http.Request) *models.ReqContext { + reqID := requestIdentifierFromRequest(req) + val, exists := requests[reqID] + if !exists { + return nil + } + + return val +} + +func requestContextMiddleware() web.Handler { + return func(res http.ResponseWriter, req *http.Request, c *models.ReqContext) { + ctx := requestContextFromRequest(req) + if ctx == nil { + c.Next() + return + } + + c.SignedInUser = ctx.SignedInUser + c.UserToken = ctx.UserToken + c.IsSignedIn = ctx.IsSignedIn + c.IsRenderCall = ctx.IsRenderCall + c.AllowAnonymous = ctx.AllowAnonymous + c.SkipCache = ctx.SkipCache + c.RequestNonce = ctx.RequestNonce + c.PerfmonTimer = ctx.PerfmonTimer + c.LookupTokenErr = ctx.LookupTokenErr + c.Map(c) + } +} diff --git a/pkg/web/webtest/webtest_test.go b/pkg/web/webtest/webtest_test.go new file mode 100644 index 00000000000..4a5bee8a71b --- /dev/null +++ b/pkg/web/webtest/webtest_test.go @@ -0,0 +1,66 @@ +package webtest + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "testing" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/models" + "github.com/stretchr/testify/require" +) + +func TestServerClient(t *testing.T) { + routeRegister := routing.NewRouteRegister() + routeRegister.Get("/test", routing.Wrap(func(c *models.ReqContext) response.Response { + return response.JSON(http.StatusOK, c.SignedInUser) + })) + + s := NewServer(t, routeRegister) + + t.Run("Making a request with user 1 should return user 1 as signed in user", func(t *testing.T) { + req := s.NewRequest(http.MethodGet, "/test", nil) + req = RequestWithWebContext(req, &models.ReqContext{ + SignedInUser: &models.SignedInUser{ + UserId: 1, + }, + }) + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, http.StatusOK, resp.StatusCode) + bytes, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + var user *models.SignedInUser + err = json.Unmarshal(bytes, &user) + require.NoError(t, err) + require.NotNil(t, user) + require.Equal(t, int64(1), user.UserId) + }) + + t.Run("Making a request with user 2 should return user 2 as signed in user", func(t *testing.T) { + req := s.NewRequest(http.MethodGet, "/test", nil) + req = RequestWithWebContext(req, &models.ReqContext{ + SignedInUser: &models.SignedInUser{ + UserId: 2, + }, + }) + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, http.StatusOK, resp.StatusCode) + bytes, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + var user *models.SignedInUser + err = json.Unmarshal(bytes, &user) + require.NoError(t, err) + require.NotNil(t, user) + require.Equal(t, int64(2), user.UserId) + }) +} diff --git a/public/app/AppWrapper.tsx b/public/app/AppWrapper.tsx index d3234f35a37..4bb5b60ce7f 100644 --- a/public/app/AppWrapper.tsx +++ b/public/app/AppWrapper.tsx @@ -37,6 +37,7 @@ export function addBodyRenderHook(fn: ComponentType) { export function addPageBanner(fn: ComponentType) { pageBanners.push(fn); } + export class AppWrapper extends React.Component { container = React.createRef(); diff --git a/public/app/angular/angular_wrappers.ts b/public/app/angular/angular_wrappers.ts index 4747c83e977..e70b00537cc 100644 --- a/public/app/angular/angular_wrappers.ts +++ b/public/app/angular/angular_wrappers.ts @@ -134,6 +134,7 @@ export function registerAngularDirectives() { 'isConfigured', 'inputWidth', 'labelWidth', + 'aria-label', ['onReset', { watchDepth: 'reference', wrapApply: true }], ['onChange', { watchDepth: 'reference', wrapApply: true }], ]); diff --git a/public/app/angular/components/code_editor/code_editor.ts b/public/app/angular/components/code_editor/code_editor.ts index c6e144249d6..64723b24b64 100644 --- a/public/app/angular/components/code_editor/code_editor.ts +++ b/public/app/angular/components/code_editor/code_editor.ts @@ -99,6 +99,9 @@ async function link(scope: any, elem: any, attrs: any) { const textarea = elem.find('textarea'); textarea.addClass('gf-form-input'); + // All aria-label to be set for accessibility + textarea.attr('aria-label', attrs.textareaLabel); + if (scope.codeEditorFocus) { setTimeout(() => { textarea.focus(); diff --git a/public/app/angular/components/query_part.ts b/public/app/angular/components/query_part.ts index 26f68057272..79fc58ce3bb 100644 --- a/public/app/angular/components/query_part.ts +++ b/public/app/angular/components/query_part.ts @@ -106,14 +106,14 @@ export function functionRenderer(part: any, innerExpr: string) { return str + parameters.join(', ') + ')'; } -export function suffixRenderer(part: QueryPartDef, innerExpr: string) { +export function suffixRenderer(part: QueryPart, innerExpr: string) { return innerExpr + ' ' + part.params[0]; } -export function identityRenderer(part: QueryPartDef, innerExpr: string) { +export function identityRenderer(part: QueryPart, innerExpr: string) { return part.params[0]; } -export function quotedIdentityRenderer(part: QueryPartDef, innerExpr: string) { +export function quotedIdentityRenderer(part: QueryPart, innerExpr: string) { return '"' + part.params[0] + '"'; } diff --git a/public/app/angular/services/ng_react.ts b/public/app/angular/services/ng_react.ts index 3cf65f48708..3d87d7f0ab4 100644 --- a/public/app/angular/services/ng_react.ts +++ b/public/app/angular/services/ng_react.ts @@ -168,9 +168,9 @@ function getPropExpression(prop: any) { * @param attrs All attributes of the component. * @param propName Name of the prop that react component expects. */ -function findAttribute(attrs: string, propName: string): string { - const index = Object.keys(attrs).find((attr) => { - return attr.toLowerCase() === propName.toLowerCase() || attr.toLowerCase() === kebabCase(propName); +function findAttribute(attrs: object, propName: string): string { + const index = Object.keys(attrs).find((attr: any) => { + return attr.toLowerCase() === propName.toLowerCase() || kebabCase(attr) === kebabCase(propName); }); // @ts-ignore return attrs[index]; diff --git a/public/app/app.ts b/public/app/app.ts index dde68451093..7f5bcb17836 100644 --- a/public/app/app.ts +++ b/public/app/app.ts @@ -38,8 +38,8 @@ import { Echo } from './core/services/echo/Echo'; import { reportPerformance } from './core/services/echo/EchoSrv'; import { PerformanceBackend } from './core/services/echo/backends/PerformanceBackend'; import 'app/features/all'; -import { getScrollbarWidth, getStandardFieldConfigs } from '@grafana/ui'; -import { getDefaultVariableAdapters, variableAdapters } from './features/variables/adapters'; +import { getScrollbarWidth } from '@grafana/ui'; +import { variableAdapters } from './features/variables/adapters'; import { initDevFeatures } from './dev'; import { getStandardTransformers } from 'app/core/utils/standardTransformers'; import { SentryEchoBackend } from './core/services/echo/backends/sentry/SentryBackend'; @@ -56,7 +56,7 @@ import { contextSrv } from './core/services/context_srv'; import { GAEchoBackend } from './core/services/echo/backends/analytics/GABackend'; import { ApplicationInsightsBackend } from './core/services/echo/backends/analytics/ApplicationInsightsBackend'; import { RudderstackBackend } from './core/services/echo/backends/analytics/RudderstackBackend'; -import { getAllOptionEditors } from './core/components/editors/registry'; +import { getAllOptionEditors, getAllStandardFieldConfigs } from './core/components/editors/registry'; import { backendSrv } from './core/services/backend_srv'; import { setPanelRenderer } from '@grafana/runtime/src/components/PanelRenderer'; import { PanelDataErrorView } from './features/panel/components/PanelDataErrorView'; @@ -65,6 +65,14 @@ import { DatasourceSrv } from './features/plugins/datasource_srv'; import { AngularApp } from './angular'; import { ModalManager } from './core/services/ModalManager'; import { initWindowRuntime } from './features/runtime/init'; +import { createQueryVariableAdapter } from './features/variables/query/adapter'; +import { createCustomVariableAdapter } from './features/variables/custom/adapter'; +import { createTextBoxVariableAdapter } from './features/variables/textbox/adapter'; +import { createConstantVariableAdapter } from './features/variables/constant/adapter'; +import { createDataSourceVariableAdapter } from './features/variables/datasource/adapter'; +import { createIntervalVariableAdapter } from './features/variables/interval/adapter'; +import { createAdHocVariableAdapter } from './features/variables/adhoc/adapter'; +import { createSystemVariableAdapter } from './features/variables/system/adapter'; // add move to lodash for backward compatabilty with plugins // @ts-ignore @@ -104,9 +112,18 @@ export class GrafanaApp { initExtensions(); standardEditorsRegistry.setInit(getAllOptionEditors); - standardFieldConfigEditorRegistry.setInit(getStandardFieldConfigs); + standardFieldConfigEditorRegistry.setInit(getAllStandardFieldConfigs); standardTransformersRegistry.setInit(getStandardTransformers); - variableAdapters.setInit(getDefaultVariableAdapters); + variableAdapters.setInit(() => [ + createQueryVariableAdapter(), + createCustomVariableAdapter(), + createTextBoxVariableAdapter(), + createConstantVariableAdapter(), + createDataSourceVariableAdapter(), + createIntervalVariableAdapter(), + createAdHocVariableAdapter(), + createSystemVariableAdapter(), + ]); monacoLanguageRegistry.setInit(getDefaultMonacoLanguages); setQueryRunnerFactory(() => new QueryRunner()); diff --git a/public/app/core/components/AccessControl/AddPermission.tsx b/public/app/core/components/AccessControl/AddPermission.tsx index 17d7495af65..b2da564b1fe 100644 --- a/public/app/core/components/AccessControl/AddPermission.tsx +++ b/public/app/core/components/AccessControl/AddPermission.tsx @@ -7,6 +7,7 @@ import { CloseButton } from 'app/core/components/CloseButton/CloseButton'; import { Assignments, PermissionTarget, SetPermission } from './types'; export interface Props { + title?: string; permissions: string[]; assignments: Assignments; canListUsers: boolean; @@ -14,7 +15,14 @@ export interface Props { onAdd: (state: SetPermission) => void; } -export const AddPermission = ({ permissions, assignments, canListUsers, onAdd, onCancel }: Props) => { +export const AddPermission = ({ + title = 'Add Permission For', + permissions, + assignments, + canListUsers, + onAdd, + onCancel, +}: Props) => { const [target, setPermissionTarget] = useState(PermissionTarget.User); const [teamId, setTeamId] = useState(0); const [userId, setUserId] = useState(0); @@ -49,7 +57,7 @@ export const AddPermission = ({ permissions, assignments, canListUsers, onAdd, o return (
-
Add Permission For
+
{title}
p === permission)} diff --git a/public/app/core/components/AccessControl/PermissionList.tsx b/public/app/core/components/AccessControl/PermissionList.tsx index defeb5289c0..9b1fb961235 100644 --- a/public/app/core/components/AccessControl/PermissionList.tsx +++ b/public/app/core/components/AccessControl/PermissionList.tsx @@ -6,27 +6,35 @@ interface Props { title: string; items: ResourcePermission[]; permissionLevels: string[]; - canRemove: boolean; + canSet: boolean; onRemove: (item: ResourcePermission) => void; onChange: (resourcePermission: ResourcePermission, permission: string) => void; } -export const PermissionList = ({ title, items, permissionLevels, canRemove, onRemove, onChange }: Props) => { +export const PermissionList = ({ title, items, permissionLevels, canSet, onRemove, onChange }: Props) => { if (items.length === 0) { return null; } return (
-
{title}
+ + + + + + {items.map((item, index) => ( diff --git a/public/app/core/components/AccessControl/PermissionListItem.tsx b/public/app/core/components/AccessControl/PermissionListItem.tsx index bff7d462cbb..ca88d08a5d8 100644 --- a/public/app/core/components/AccessControl/PermissionListItem.tsx +++ b/public/app/core/components/AccessControl/PermissionListItem.tsx @@ -5,22 +5,21 @@ import { Button, Icon, Select, Tooltip } from '@grafana/ui'; interface Props { item: ResourcePermission; permissionLevels: string[]; - canRemove: boolean; + canSet: boolean; onRemove: (item: ResourcePermission) => void; onChange: (item: ResourcePermission, permission: string) => void; } -export const PermissionListItem = ({ item, permissionLevels, canRemove, onRemove, onChange }: Props) => ( +export const PermissionListItem = ({ item, permissionLevels, canSet, onRemove, onChange }: Props) => ( - - +
+ {title}Permission + +
{getAvatar(item)} {getDescription(item)} - Can
{session.isActive ? 'Now' : session.seenAt}{session.createdAt}{i18n.date(session.createdAt, { dateStyle: 'long' })} {session.clientIp} {`${session.browser} on ${session.os} ${session.osVersion}`} @@ -113,3 +114,5 @@ export class UserSessions extends PureComponent { ); } } + +export const UserSessions = withI18n()(BaseUserSessions); diff --git a/public/app/features/admin/state/actions.ts b/public/app/features/admin/state/actions.ts index b3b60b8d2e2..2297783226d 100644 --- a/public/app/features/admin/state/actions.ts +++ b/public/app/features/admin/state/actions.ts @@ -1,5 +1,5 @@ import config from 'app/core/config'; -import { dateTimeFormat, dateTimeFormatTimeAgo } from '@grafana/data'; +import { dateTimeFormatTimeAgo } from '@grafana/data'; import { featureEnabled, getBackendSrv, locationService } from '@grafana/runtime'; import { ThunkResult, LdapUser, UserSession, UserDTO, AccessControlAction, UserFilter } from 'app/types'; @@ -25,7 +25,7 @@ import { } from './reducers'; import { debounce } from 'lodash'; import { contextSrv } from 'app/core/core'; -import { addAccessControlQueryParam } from 'app/core/utils/accessControl'; +import { accessControlQueryParam } from 'app/core/utils/accessControl'; // UserAdminPage export function loadAdminUserPage(userId: number): ThunkResult { @@ -54,7 +54,7 @@ export function loadAdminUserPage(userId: number): ThunkResult { export function loadUserProfile(userId: number): ThunkResult { return async (dispatch) => { - const user = await getBackendSrv().get(addAccessControlQueryParam(`/api/users/${userId}`)); + const user = await getBackendSrv().get(`/api/users/${userId}`, accessControlQueryParam()); dispatch(userProfileLoadedAction(user)); }; } @@ -144,12 +144,13 @@ export function loadUserSessions(userId: number): ThunkResult { const tokens = await getBackendSrv().get(`/api/admin/users/${userId}/auth-tokens`); tokens.reverse(); + const sessions = tokens.map((session: UserSession) => { return { id: session.id, isActive: session.isActive, seenAt: dateTimeFormatTimeAgo(session.seenAt), - createdAt: dateTimeFormat(session.createdAt, { format: 'MMMM DD, YYYY' }), + createdAt: session.createdAt, clientIp: session.clientIp, browser: session.browser, browserVersion: session.browserVersion, @@ -158,6 +159,7 @@ export function loadUserSessions(userId: number): ThunkResult { device: session.device, }; }); + dispatch(userSessionsLoadedAction(sessions)); }; } diff --git a/public/app/features/alerting/AlertRuleItem.tsx b/public/app/features/alerting/AlertRuleItem.tsx index 36d72ac579a..53468797393 100644 --- a/public/app/features/alerting/AlertRuleItem.tsx +++ b/public/app/features/alerting/AlertRuleItem.tsx @@ -25,7 +25,7 @@ const AlertRuleItem = ({ rule, search, onTogglePause }: Props) => { ); return ( - + {renderText(rule.name)} diff --git a/public/app/features/alerting/getAlertingValidationMessage.test.ts b/public/app/features/alerting/getAlertingValidationMessage.test.ts index 6deb2188b10..f633db400e9 100644 --- a/public/app/features/alerting/getAlertingValidationMessage.test.ts +++ b/public/app/features/alerting/getAlertingValidationMessage.test.ts @@ -35,6 +35,7 @@ describe('getAlertingValidationMessage', () => { return []; }, getInstanceSettings: (() => {}) as any, + reload: () => jest.fn(), }; const targets: ElasticsearchQuery[] = [ { refId: 'A', query: '@hostname:$hostname' }, @@ -77,6 +78,7 @@ describe('getAlertingValidationMessage', () => { getList(): DataSourceInstanceSettings[] { return []; }, + reload: () => jest.fn(), }; const targets: any[] = [ { refId: 'A', query: 'some query', datasource: 'alertingDatasource' }, @@ -108,6 +110,7 @@ describe('getAlertingValidationMessage', () => { getList(): DataSourceInstanceSettings[] { return []; }, + reload: () => jest.fn(), }; const targets: ElasticsearchQuery[] = [ { refId: 'A', query: '@hostname:$hostname' }, @@ -142,6 +145,7 @@ describe('getAlertingValidationMessage', () => { getList(): DataSourceInstanceSettings[] { return []; }, + reload: () => jest.fn(), }; const targets: ElasticsearchQuery[] = [ { refId: 'A', query: '@hostname:hostname' }, @@ -175,6 +179,7 @@ describe('getAlertingValidationMessage', () => { getList(): DataSourceInstanceSettings[] { return []; }, + reload: () => jest.fn(), }; const targets: ElasticsearchQuery[] = [ { refId: 'A', query: '@hostname:hostname' }, diff --git a/public/app/features/alerting/unified/Silences.test.tsx b/public/app/features/alerting/unified/Silences.test.tsx index 3b468dd0ed7..6e361744962 100644 --- a/public/app/features/alerting/unified/Silences.test.tsx +++ b/public/app/features/alerting/unified/Silences.test.tsx @@ -60,7 +60,6 @@ const ui = { matcherName: byPlaceholderText('label'), matcherValue: byPlaceholderText('value'), comment: byPlaceholderText('Details about the silence'), - createdBy: byPlaceholderText('User'), matcherOperatorSelect: byLabelText('operator'), matcherOperator: (operator: MatcherOperator) => byText(operator, { exact: true }), addMatcherButton: byRole('button', { name: 'Add matcher' }), @@ -174,9 +173,10 @@ describe('Silence edit', () => { it( 'prefills the matchers field with matchers params', async () => { - renderSilences( - `${baseUrlPath}?matchers=${encodeURIComponent('foo=bar,bar=~ba.+,hello!=world,cluster!~us-central.*')}` - ); + const matchersParams = ['foo=bar', 'bar=~ba.+', 'hello!=world', 'cluster!~us-central.*']; + const matchersQueryString = matchersParams.map((matcher) => `matcher=${encodeURIComponent(matcher)}`).join('&'); + + renderSilences(`${baseUrlPath}?${matchersQueryString}`); await waitFor(() => expect(ui.editor.durationField.query()).not.toBeNull()); const matchers = ui.editor.matchersField.queryAll(); @@ -209,6 +209,7 @@ describe('Silence edit', () => { const start = new Date(); const end = new Date(start.getTime() + 24 * 60 * 60 * 1000); + const now = dateTime().format('YYYY-MM-DD HH:mm'); const startDateString = dateTime(start).format('YYYY-MM-DD'); const endDateString = dateTime(end).format('YYYY-MM-DD'); @@ -246,17 +247,13 @@ describe('Silence edit', () => { userEvent.tab(); userEvent.type(ui.editor.matcherValue.getAll()[3], 'dev|staging'); - userEvent.type(ui.editor.comment.get(), 'Test'); - userEvent.type(ui.editor.createdBy.get(), 'Homer Simpson'); - userEvent.click(ui.editor.submit.get()); await waitFor(() => expect(mocks.api.createOrUpdateSilence).toHaveBeenCalledWith( 'grafana', expect.objectContaining({ - comment: 'Test', - createdBy: 'Homer Simpson', + comment: `created ${now}`, matchers: [ { isEqual: true, isRegex: false, name: 'foo', value: 'bar' }, { isEqual: false, isRegex: false, name: 'bar', value: 'buzz' }, diff --git a/public/app/features/alerting/unified/api/ruler.ts b/public/app/features/alerting/unified/api/ruler.ts index a849fbb4e49..8def1d17da4 100644 --- a/public/app/features/alerting/unified/api/ruler.ts +++ b/public/app/features/alerting/unified/api/ruler.ts @@ -108,9 +108,6 @@ async function rulerGetRequest(url: string, empty: T, params?: Record(url: string, empty: T, params?: Record) { + return ( + error.status === 500 && + error.data.message?.includes('unexpected content type from upstream. expected YAML, got text/html') + ); +} + function isCortexErrorResponse(error: FetchResponse) { - return error.data.error?.includes('group does not exist') || error.data.error?.includes('no rule groups found'); + return error.data.message?.includes('group does not exist') || error.data.message?.includes('no rule groups found'); } export async function deleteNamespace(dataSourceName: string, namespace: string): Promise { diff --git a/public/app/features/alerting/unified/components/EmptyAreaWithCTA.tsx b/public/app/features/alerting/unified/components/EmptyAreaWithCTA.tsx index 85d289bf7ae..c507512ab07 100644 --- a/public/app/features/alerting/unified/components/EmptyAreaWithCTA.tsx +++ b/public/app/features/alerting/unified/components/EmptyAreaWithCTA.tsx @@ -1,12 +1,13 @@ import React, { ButtonHTMLAttributes, FC } from 'react'; import { css } from '@emotion/css'; import { GrafanaTheme } from '@grafana/data'; -import { Button, ButtonVariant, IconName, useStyles } from '@grafana/ui'; +import { Button, ButtonVariant, IconName, LinkButton, useStyles } from '@grafana/ui'; import { EmptyArea } from './EmptyArea'; export interface EmptyAreaWithCTAProps { buttonLabel: string; - onButtonClick: ButtonHTMLAttributes['onClick']; + href?: string; + onButtonClick?: ButtonHTMLAttributes['onClick']; text: string; buttonIcon?: IconName; @@ -21,23 +22,30 @@ export const EmptyAreaWithCTA: FC = ({ buttonVariant = 'primary', onButtonClick, text, + href, }) => { const styles = useStyles(getStyles); + const commonProps = { + className: styles.button, + icon: buttonIcon, + size: buttonSize, + variant: buttonVariant, + }; + return ( <>

{text}

- + {href ? ( + + {buttonLabel} + + ) : ( + + )} ); diff --git a/public/app/features/alerting/unified/components/alert-groups/AlertDetails.tsx b/public/app/features/alerting/unified/components/alert-groups/AlertDetails.tsx index 8b16461f7d0..77e83600c5b 100644 --- a/public/app/features/alerting/unified/components/alert-groups/AlertDetails.tsx +++ b/public/app/features/alerting/unified/components/alert-groups/AlertDetails.tsx @@ -2,11 +2,9 @@ import { css } from '@emotion/css'; import { GrafanaTheme2 } from '@grafana/data'; import { LinkButton, useStyles2 } from '@grafana/ui'; import { AlertmanagerAlert, AlertState } from 'app/plugins/datasource/alertmanager/types'; - import React, { FC } from 'react'; -import { makeAMLink } from '../../utils/misc'; +import { makeAMLink, makeLabelBasedSilenceLink } from '../../utils/misc'; import { AnnotationDetailsField } from '../AnnotationDetailsField'; -import { getMatcherQueryParams } from '../../utils/matchers'; interface AmNotificationsAlertDetailsProps { alertManagerSourceName: string; @@ -33,9 +31,7 @@ export const AlertDetails: FC = ({ alert, aler )} {alert.status.state === AlertState.Active && ( { setQueryParams({ queryString: value ? value : null })} /> void; } -export const MatcherFilter = ({ className, onFilterChange, queryString }: Props) => { +export const MatcherFilter = ({ className, onFilterChange, defaultQueryString, queryString }: Props) => { const styles = useStyles2(getStyles); const handleSearchChange = (e: FormEvent) => { const target = e.target as HTMLInputElement; @@ -33,7 +34,8 @@ export const MatcherFilter = ({ className, onFilterChange, queryString }: Props) = {}): FormAmRoute => { + return { ...defaultAmRoute, ...override }; +}; + +const buildMatcher = (name: string, value: string, operator: MatcherOperator): MatcherFieldValue => { + return { name, value, operator }; +}; + +describe('getFilteredRoutes', () => { + it('Shoult return all entries when filters are empty', () => { + // Arrange + const routes: FormAmRoute[] = [buildAmRoute({ id: '1' }), buildAmRoute({ id: '2' }), buildAmRoute({ id: '3' })]; + + // Act + const filteredRoutes = getFilteredRoutes(routes, undefined, undefined); + + // Assert + expect(filteredRoutes).toHaveLength(3); + expect(filteredRoutes).toContain(routes[0]); + expect(filteredRoutes).toContain(routes[1]); + expect(filteredRoutes).toContain(routes[2]); + }); + + it('Should only return entries matching provided label query', () => { + // Arrange + const routes: FormAmRoute[] = [ + buildAmRoute({ id: '1' }), + buildAmRoute({ id: '2', object_matchers: [buildMatcher('severity', 'critical', MatcherOperator.equal)] }), + buildAmRoute({ id: '3' }), + ]; + + // Act + const filteredRoutes = getFilteredRoutes(routes, 'severity=critical', undefined); + + // Assert + expect(filteredRoutes).toHaveLength(1); + expect(filteredRoutes).toContain(routes[1]); + }); + + it('Should only return entries matching provided contact query', () => { + // Arrange + const routes: FormAmRoute[] = [ + buildAmRoute({ id: '1' }), + buildAmRoute({ id: '2', receiver: 'TestContactPoint' }), + buildAmRoute({ id: '3' }), + ]; + + // Act + const filteredRoutes = getFilteredRoutes(routes, undefined, 'contact'); + + // Assert + expect(filteredRoutes).toHaveLength(1); + expect(filteredRoutes).toContain(routes[1]); + }); + + it('Should only return entries matching provided label and contact query', () => { + // Arrange + const routes: FormAmRoute[] = [ + buildAmRoute({ id: '1' }), + buildAmRoute({ + id: '2', + receiver: 'TestContactPoint', + object_matchers: [buildMatcher('severity', 'critical', MatcherOperator.equal)], + }), + buildAmRoute({ id: '3' }), + ]; + + // Act + const filteredRoutes = getFilteredRoutes(routes, 'severity=critical', 'contact'); + + // Assert + expect(filteredRoutes).toHaveLength(1); + expect(filteredRoutes).toContain(routes[1]); + }); + + it('Should return entries matching regex matcher configuration without regex evaluation', () => { + // Arrange + const routes: FormAmRoute[] = [ + buildAmRoute({ id: '1' }), + buildAmRoute({ id: '2', object_matchers: [buildMatcher('severity', 'critical', MatcherOperator.equal)] }), + buildAmRoute({ id: '3', object_matchers: [buildMatcher('severity', 'crit', MatcherOperator.regex)] }), + ]; + + // Act + const filteredRoutes = getFilteredRoutes(routes, 'severity=~crit', undefined); + + // Assert + expect(filteredRoutes).toHaveLength(1); + expect(filteredRoutes).toContain(routes[2]); + }); +}); diff --git a/public/app/features/alerting/unified/components/amroutes/AmRoutesTable.tsx b/public/app/features/alerting/unified/components/amroutes/AmRoutesTable.tsx index c3543129a35..0d4bf235f04 100644 --- a/public/app/features/alerting/unified/components/amroutes/AmRoutesTable.tsx +++ b/public/app/features/alerting/unified/components/amroutes/AmRoutesTable.tsx @@ -6,7 +6,9 @@ import { DynamicTable, DynamicTableColumnProps, DynamicTableItemProps } from '.. import { AmRoutesExpandedForm } from './AmRoutesExpandedForm'; import { AmRoutesExpandedRead } from './AmRoutesExpandedRead'; import { Matchers } from '../silences/Matchers'; -import { matcherFieldToMatcher } from '../../utils/alertmanager'; +import { matcherFieldToMatcher, parseMatchers } from '../../utils/alertmanager'; +import { intersectionWith, isEqual } from 'lodash'; +import { EmptyArea } from '../EmptyArea'; export interface AmRoutesTableProps { isAddMode: boolean; @@ -14,26 +16,47 @@ export interface AmRoutesTableProps { onCancelAdd: () => void; receivers: AmRouteReceiver[]; routes: FormAmRoute[]; + filters?: { queryString?: string; contactPoint?: string }; readOnly?: boolean; } type RouteTableColumnProps = DynamicTableColumnProps; type RouteTableItemProps = DynamicTableItemProps; +export const getFilteredRoutes = (routes: FormAmRoute[], labelMatcherQuery?: string, contactPointQuery?: string) => { + const matchers = parseMatchers(labelMatcherQuery ?? ''); + + let filteredRoutes = routes; + + if (matchers.length) { + filteredRoutes = routes.filter((route) => { + const routeMatchers = route.object_matchers.map(matcherFieldToMatcher); + return intersectionWith(routeMatchers, matchers, isEqual).length > 0; + }); + } + + if (contactPointQuery && contactPointQuery.length > 0) { + filteredRoutes = filteredRoutes.filter((route) => + route.receiver.toLowerCase().includes(contactPointQuery.toLowerCase()) + ); + } + + return filteredRoutes; +}; + export const AmRoutesTable: FC = ({ isAddMode, onCancelAdd, onChange, receivers, routes, + filters, readOnly = false, }) => { const [editMode, setEditMode] = useState(false); - const [expandedId, setExpandedId] = useState(); const expandItem = useCallback((item: RouteTableItemProps) => setExpandedId(item.id), []); - const collapseItem = useCallback(() => setExpandedId(undefined), []); const cols: RouteTableColumnProps[] = [ @@ -56,6 +79,12 @@ export const AmRoutesTable: FC = ({ renderCell: (item) => item.data.receiver || '-', size: 5, }, + { + id: 'muteTimings', + label: 'Mute timings', + renderCell: (item) => item.data.muteTimeIntervals.join(', ') || '-', + size: 5, + }, ...(readOnly ? [] : [ @@ -105,20 +134,37 @@ export const AmRoutesTable: FC = ({ ]), ]; - const items = useMemo(() => prepareItems(routes), [routes]); + const filteredRoutes = useMemo(() => getFilteredRoutes(routes, filters?.queryString, filters?.contactPoint), [ + routes, + filters, + ]); + + const dynamicTableRoutes = useMemo(() => prepareItems(isAddMode ? routes : filteredRoutes), [ + isAddMode, + routes, + filteredRoutes, + ]); // expand the last item when adding useEffect(() => { - if (isAddMode && items.length) { - setExpandedId(items[items.length - 1].id); + if (isAddMode && dynamicTableRoutes.length) { + setExpandedId(dynamicTableRoutes[dynamicTableRoutes.length - 1].id); } - }, [isAddMode, items]); + }, [isAddMode, dynamicTableRoutes]); + + if (routes.length > 0 && filteredRoutes.length === 0) { + return ( + +

No policies found

+ + ); + } return ( 'am-routes-row'} onCollapse={collapseItem} onExpand={expandItem} diff --git a/public/app/features/alerting/unified/components/amroutes/AmSpecificRouting.tsx b/public/app/features/alerting/unified/components/amroutes/AmSpecificRouting.tsx index d4d36a8a197..c2acff52421 100644 --- a/public/app/features/alerting/unified/components/amroutes/AmSpecificRouting.tsx +++ b/public/app/features/alerting/unified/components/amroutes/AmSpecificRouting.tsx @@ -1,12 +1,16 @@ -import React, { FC, useState } from 'react'; import { css } from '@emotion/css'; import { GrafanaTheme2 } from '@grafana/data'; -import { Button, useStyles2 } from '@grafana/ui'; +import { Button, Icon, Input, Label, useStyles2 } from '@grafana/ui'; +import React, { FC, useState } from 'react'; +import { useDebounce } from 'react-use'; +import { useURLSearchParams } from '../../hooks/useURLSearchParams'; import { AmRouteReceiver, FormAmRoute } from '../../types/amroutes'; import { emptyArrayFieldMatcher, emptyRoute } from '../../utils/amroutes'; +import { getNotificationPoliciesFilters } from '../../utils/misc'; +import { MatcherFilter } from '../alert-groups/MatcherFilter'; import { EmptyArea } from '../EmptyArea'; -import { AmRoutesTable } from './AmRoutesTable'; import { EmptyAreaWithCTA } from '../EmptyAreaWithCTA'; +import { AmRoutesTable } from './AmRoutesTable'; export interface AmSpecificRoutingProps { onChange: (routes: FormAmRoute) => void; @@ -16,6 +20,11 @@ export interface AmSpecificRoutingProps { readOnly?: boolean; } +interface Filters { + queryString?: string; + contactPoint?: string; +} + export const AmSpecificRouting: FC = ({ onChange, onRootRouteEdit, @@ -23,15 +32,34 @@ export const AmSpecificRouting: FC = ({ routes, readOnly = false, }) => { - const [actualRoutes, setActualRoutes] = useState(routes.routes); + const [actualRoutes, setActualRoutes] = useState([...routes.routes]); const [isAddMode, setIsAddMode] = useState(false); + const [searchParams, setSearchParams] = useURLSearchParams(); + const { queryString, contactPoint } = getNotificationPoliciesFilters(searchParams); + + const [filters, setFilters] = useState({ queryString, contactPoint }); + + useDebounce( + () => { + setSearchParams({ queryString: filters.queryString, contactPoint: filters.contactPoint }); + }, + 400, + [filters] + ); + const styles = useStyles2(getStyles); + const clearFilters = () => { + setFilters({ queryString: undefined, contactPoint: undefined }); + setSearchParams({ queryString: undefined, contactPoint: undefined }); + }; + const addNewRoute = () => { + clearFilters(); setIsAddMode(true); - setActualRoutes((actualRoutes) => [ - ...actualRoutes, + setActualRoutes(() => [ + ...routes.routes, { ...emptyRoute, matchers: [emptyArrayFieldMatcher], @@ -39,6 +67,21 @@ export const AmSpecificRouting: FC = ({ ]); }; + const onCancelAdd = () => { + setIsAddMode(false); + setActualRoutes([...routes.routes]); + }; + + const onTableRouteChange = (newRoutes: FormAmRoute[]): void => { + onChange({ + ...routes, + routes: newRoutes, + }); + + if (isAddMode) { + setIsAddMode(false); + } + }; return (
Specific routing
@@ -58,35 +101,52 @@ export const AmSpecificRouting: FC = ({ ) ) : actualRoutes.length > 0 ? ( <> - {!isAddMode && !readOnly && ( - - )} +
+ {!isAddMode && ( +
+ + setFilters((currentFilters) => ({ ...currentFilters, queryString: filter })) + } + queryString={filters.queryString ?? ''} + className={styles.filterInput} + /> +
+ + + setFilters((currentFilters) => ({ ...currentFilters, contactPoint: currentTarget.value })) + } + value={filters.contactPoint ?? ''} + placeholder="Search by contact point" + data-testid="search-query-input" + prefix={} + /> +
+ {(queryString || contactPoint) && ( + + )} +
+ )} + + {!isAddMode && !readOnly && ( +
+ +
+ )} +
{ - setIsAddMode(false); - setActualRoutes((actualRoutes) => { - const newRoutes = [...actualRoutes]; - newRoutes.pop(); - - return newRoutes; - }); - }} - onChange={(newRoutes) => { - onChange({ - ...routes, - routes: newRoutes, - }); - - if (isAddMode) { - setIsAddMode(false); - } - }} + onCancelAdd={onCancelAdd} + onChange={onTableRouteChange} receivers={receivers} routes={actualRoutes} + filters={{ queryString, contactPoint }} /> ) : readOnly ? ( @@ -108,12 +168,32 @@ export const AmSpecificRouting: FC = ({ const getStyles = (theme: GrafanaTheme2) => { return { container: css` + display: flex; + flex-flow: column wrap; + `, + searchContainer: css` + display: flex; + flex-flow: row nowrap; + padding-bottom: ${theme.spacing(2)}; + border-bottom: 1px solid ${theme.colors.border.strong}; + `, + clearFilterBtn: css` + align-self: flex-end; + margin-left: ${theme.spacing(1)}; + `, + filterInput: css` + width: 340px; + & + & { + margin-left: ${theme.spacing(1)}; + } + `, + addMatcherBtnRow: css` display: flex; flex-flow: column nowrap; + padding: ${theme.spacing(2)} 0; `, addMatcherBtn: css` align-self: flex-end; - margin-bottom: ${theme.spacing(3.5)}; `, }; }; diff --git a/public/app/features/alerting/unified/components/amroutes/MuteTimingTimeRange.tsx b/public/app/features/alerting/unified/components/amroutes/MuteTimingTimeRange.tsx index dfb1b91e64f..46e3fc2747f 100644 --- a/public/app/features/alerting/unified/components/amroutes/MuteTimingTimeRange.tsx +++ b/public/app/features/alerting/unified/components/amroutes/MuteTimingTimeRange.tsx @@ -22,7 +22,7 @@ export const MuteTimingTimeRange: FC = ({ intervalIndex }) => { return true; } const [hour, minutes] = timeString.split(':').map((x) => parseInt(x, 10)); - const isHourValid = hour > 0 && hour < 25; + const isHourValid = hour >= 0 && hour < 25; const isMinuteValid = minutes > -1 && minutes < 60; const isTimeValid = hour === 24 ? minutes === 0 : isHourValid && isMinuteValid; diff --git a/public/app/features/alerting/unified/components/amroutes/MuteTimingsTable.tsx b/public/app/features/alerting/unified/components/amroutes/MuteTimingsTable.tsx index 2c32e5138ce..7ee43ea5b43 100644 --- a/public/app/features/alerting/unified/components/amroutes/MuteTimingsTable.tsx +++ b/public/app/features/alerting/unified/components/amroutes/MuteTimingsTable.tsx @@ -16,6 +16,7 @@ import { getMonthsString, getYearsString, } from '../../utils/alertmanager'; +import { EmptyAreaWithCTA } from '../EmptyAreaWithCTA'; interface Props { alertManagerSourceName: string; @@ -46,7 +47,7 @@ export const MuteTimingsTable: FC = ({ alertManagerSourceName, muteTiming const columns = useColumns(alertManagerSourceName, hideActions, setMuteTimingName); return ( -
+
{!hideActions &&
Mute timings
} {!hideActions && (

@@ -54,7 +55,29 @@ export const MuteTimingsTable: FC = ({ alertManagerSourceName, muteTiming particular notification policies for specific times of the day.

)} - {items.length > 0 ? :

No mute timings configured

} + {!hideActions && items.length > 0 && ( + + New mute timing + + )} + {items.length > 0 ? ( + + ) : !hideActions ? ( + + ) : ( +

No mute timings configured

+ )} {!hideActions && ( = ({ alertManagerSourceName, muteTiming onDismiss={() => setMuteTimingName('')} /> )} - {!hideActions && ( - - Add mute timing - - )}
); }; @@ -139,7 +153,12 @@ function renderTimeIntervals(timeIntervals: TimeInterval[]) { } const getStyles = (theme: GrafanaTheme2) => ({ + container: css` + display: flex; + flex-flow: column nowrap; + `, addMuteButton: css` - margin-top: ${theme.spacing(1)}; + margin-bottom: ${theme.spacing(2)}; + align-self: flex-end; `, }); diff --git a/public/app/features/alerting/unified/components/receivers/form/CloudReceiverForm.tsx b/public/app/features/alerting/unified/components/receivers/form/CloudReceiverForm.tsx index c78aa7f026b..15c8b97ec1d 100644 --- a/public/app/features/alerting/unified/components/receivers/form/CloudReceiverForm.tsx +++ b/public/app/features/alerting/unified/components/receivers/form/CloudReceiverForm.tsx @@ -64,7 +64,7 @@ export const CloudReceiverForm: FC = ({ existing, alertManagerSourceName, <> {!isVanillaAM && ( - Note that empty string values will be replaced with global defaults were appropriate. + Note that empty string values will be replaced with global defaults where appropriate. )} diff --git a/public/app/features/alerting/unified/components/rule-editor/QueryEditor.tsx b/public/app/features/alerting/unified/components/rule-editor/QueryEditor.tsx index e94779bf16b..3eb91a9d545 100644 --- a/public/app/features/alerting/unified/components/rule-editor/QueryEditor.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/QueryEditor.tsx @@ -9,7 +9,7 @@ import { RelativeTimeRange, } from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; -import { Button, HorizontalGroup, Icon, stylesFactory, Tooltip } from '@grafana/ui'; +import { Button, HorizontalGroup, stylesFactory } from '@grafana/ui'; import { config } from '@grafana/runtime'; import { QueryRows } from './QueryRows'; import { @@ -121,18 +121,15 @@ export class QueryEditor extends PureComponent { Query {config.expressionsEnabled && ( - - - + )} ); diff --git a/public/app/features/alerting/unified/components/rule-editor/QueryRows.tsx b/public/app/features/alerting/unified/components/rule-editor/QueryRows.tsx index 1bd3a6598f1..d6d7a653dc4 100644 --- a/public/app/features/alerting/unified/components/rule-editor/QueryRows.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/QueryRows.tsx @@ -1,4 +1,4 @@ -import React, { PureComponent } from 'react'; +import React, { PureComponent, useState } from 'react'; import { DragDropContext, Droppable, DropResult } from 'react-beautiful-dnd'; import { DataQuery, @@ -10,10 +10,14 @@ import { ThresholdsMode, } from '@grafana/data'; import { config, getDataSourceSrv } from '@grafana/runtime'; -import { QueryWrapper } from './QueryWrapper'; -import { AlertQuery } from 'app/types/unified-alerting-dto'; +import { EmptyQueryWrapper, QueryWrapper } from './QueryWrapper'; +import { AlertDataQuery, AlertQuery } from 'app/types/unified-alerting-dto'; import { isExpressionQuery } from 'app/features/expressions/guards'; import { queriesWithUpdatedReferences } from './util'; +import { Button, Card, Icon } from '@grafana/ui'; +import { QueryOperationRow } from 'app/core/components/QueryOperationRow/QueryOperationRow'; +import { getDatasourceSrv } from 'app/features/plugins/datasource_srv'; +import { omit } from 'lodash'; interface Props { // The query configuration @@ -100,30 +104,14 @@ export class QueryRows extends PureComponent { onChangeDataSource = (settings: DataSourceInstanceSettings, index: number) => { const { queries, onQueriesChange } = this.props; - onQueriesChange( - queries.map((item, itemIndex) => { - if (itemIndex !== index) { - return item; - } + const updatedQueries = queries.map((item, itemIndex) => { + if (itemIndex !== index) { + return item; + } - const previous = getDataSourceSrv().getInstanceSettings(item.datasourceUid); - - if (previous?.type === settings.uid) { - return { - ...item, - datasourceUid: settings.uid, - }; - } - - const { refId, hide } = item.model; - - return { - ...item, - datasourceUid: settings.uid, - model: { refId, hide }, - }; - }) - ); + return copyModel(item, settings.uid); + }); + onQueriesChange(updatedQueries); }; onChangeQuery = (query: DataQuery, index: number) => { @@ -245,8 +233,24 @@ export class QueryRows extends PureComponent { const dsSettings = this.getDataSourceSettings(query); if (!dsSettings) { - return null; + return ( + { + const defaultDataSource = getDatasourceSrv().getInstanceSettings(null); + if (defaultDataSource) { + this.onChangeDataSource(defaultDataSource, index); + } + }} + onRemoveQuery={() => { + this.onRemoveQuery(query); + }} + /> + ); } + return ( { ); } } + +function copyModel(item: AlertQuery, uid: string): Omit { + return { + ...item, + model: omit(item.model, 'datasource'), + datasourceUid: uid, + }; +} + +interface DatasourceNotFoundProps { + index: number; + model: AlertDataQuery; + onUpdateDatasource: () => void; + onRemoveQuery: () => void; +} + +const DatasourceNotFound = ({ index, onUpdateDatasource, onRemoveQuery, model }: DatasourceNotFoundProps) => { + const refId = model.refId; + + const [showDetails, setShowDetails] = useState(false); + + const toggleDetails = () => { + setShowDetails((show) => !show); + }; + + const handleUpdateDatasource = () => { + onUpdateDatasource(); + }; + + return ( + + + + + + + + + + + + + + + {showDetails && ( +
+ 
+              {JSON.stringify(model, null, 2)}
+
+
+ )} + + + ); +}; diff --git a/public/app/features/alerting/unified/components/rule-editor/QueryStep.tsx b/public/app/features/alerting/unified/components/rule-editor/QueryStep.tsx index a77c592c4d6..e55f4d5d549 100644 --- a/public/app/features/alerting/unified/components/rule-editor/QueryStep.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/QueryStep.tsx @@ -14,6 +14,7 @@ export const QueryStep: FC = () => { } = useFormContext(); const type = watch('type'); const dataSourceName = watch('dataSourceName'); + return ( = ({ ); }; +export const EmptyQueryWrapper: FC<{}> = ({ children }) => { + const styles = useStyles2(getStyles); + return
{children}
; +}; + const getStyles = (theme: GrafanaTheme2) => ({ wrapper: css` label: AlertingQueryWrapper; margin-bottom: ${theme.spacing(1)}; border: 1px solid ${theme.colors.border.medium}; border-radius: ${theme.shape.borderRadius(1)}; - padding-bottom: ${theme.spacing(1)}; `, }); diff --git a/public/app/features/alerting/unified/components/rules/RuleDetailsActionButtons.tsx b/public/app/features/alerting/unified/components/rules/RuleDetailsActionButtons.tsx index ea1c1ce3247..1d7aea69627 100644 --- a/public/app/features/alerting/unified/components/rules/RuleDetailsActionButtons.tsx +++ b/public/app/features/alerting/unified/components/rules/RuleDetailsActionButtons.tsx @@ -10,7 +10,7 @@ import { appEvents } from 'app/core/core'; import { useIsRuleEditable } from '../../hooks/useIsRuleEditable'; import { Annotation } from '../../utils/constants'; import { getRulesSourceName, isCloudRulesSource, isGrafanaRulesSource } from '../../utils/datasource'; -import { createExploreLink, createViewLink, makeSilenceLink } from '../../utils/misc'; +import { createExploreLink, createViewLink, makeRuleBasedSilenceLink } from '../../utils/misc'; import * as ruleId from '../../utils/rule-id'; import { deleteRuleAction } from '../../state/actions'; import { CombinedRule, RulesSource } from 'app/types/unified-alerting'; @@ -140,7 +140,7 @@ export const RuleDetailsActionButtons: FC = ({ rule, rulesSource }) => { key="silence" icon="bell-slash" target="__blank" - href={makeSilenceLink(alertmanagerSourceName, rule)} + href={makeRuleBasedSilenceLink(alertmanagerSourceName, rule)} > Silence diff --git a/public/app/features/alerting/unified/components/rules/RuleDetailsMatchingInstances.tsx b/public/app/features/alerting/unified/components/rules/RuleDetailsMatchingInstances.tsx index db039e769b7..80efbc627eb 100644 --- a/public/app/features/alerting/unified/components/rules/RuleDetailsMatchingInstances.tsx +++ b/public/app/features/alerting/unified/components/rules/RuleDetailsMatchingInstances.tsx @@ -48,7 +48,7 @@ export function RuleDetailsMatchingInstances(props: Props): JSX.Element | null { setQueryString(value)} /> ; +type MatchedRulesTableColumnProps = DynamicTableColumnProps<{ matchedRule: CombinedRule }>; + +export const MatchedSilencedRules = () => { + const [matchedAlertRules, setMatchedAlertRules] = useState([]); + const formApi = useFormContext(); + const dispatch = useDispatch(); + const { watch } = formApi; + const matchers: MatcherFieldValue[] = watch('matchers'); + const styles = useStyles2(getStyles); + const columns = useColumns(); + + useEffect(() => { + dispatch(fetchAllPromAndRulerRulesAction()); + }, [dispatch]); + + const combinedNamespaces = useCombinedRuleNamespaces(); + useDebounce( + () => { + const matchedRules = combinedNamespaces.flatMap((namespace) => { + return namespace.groups.flatMap((group) => { + return findAlertRulesWithMatchers(group.rules, matchers); + }); + }); + setMatchedAlertRules(matchedRules); + }, + 500, + [combinedNamespaces, matchers] + ); + + return ( +
+

+ Affected alerts + {matchedAlertRules.length > 0 ? ( + + ) : null} +

+
+ {matchers.every((matcher) => !matcher.value && !matcher.name) ? ( + Add a valid matcher to see affected alerts + ) : ( + <> + + {matchedAlertRules.length > 5 && ( +
and {matchedAlertRules.length - 5} more
+ )} + + )} +
+
+ ); +}; + +function useColumns(): MatchedRulesTableColumnProps[] { + return [ + { + id: 'state', + label: 'State', + renderCell: function renderStateTag({ data: { matchedRule } }) { + return ; + }, + size: '160px', + }, + { + id: 'name', + label: 'Name', + renderCell: function renderName({ data: { matchedRule } }) { + return matchedRule.name; + }, + size: '250px', + }, + { + id: 'summary', + label: 'Summary', + renderCell: function renderSummary({ data: { matchedRule } }) { + return matchedRule.annotations[Annotation.summary] ?? ''; + }, + size: '400px', + }, + ]; +} + +const getStyles = (theme: GrafanaTheme2) => ({ + table: css` + max-width: ${theme.breakpoints.values.lg}px; + `, + moreMatches: css` + margin-top: ${theme.spacing(1)}; + `, + title: css` + display: flex; + align-items: center; + `, + badge: css` + margin-left: ${theme.spacing(1)}; + `, +}); diff --git a/public/app/features/alerting/unified/components/silences/MatchersField.tsx b/public/app/features/alerting/unified/components/silences/MatchersField.tsx index 96ee773c510..7464af9ef2f 100644 --- a/public/app/features/alerting/unified/components/silences/MatchersField.tsx +++ b/public/app/features/alerting/unified/components/silences/MatchersField.tsx @@ -128,7 +128,7 @@ const getStyles = (theme: GrafanaTheme2) => { min-width: 140px; `, matchers: css` - max-width: 585px; + max-width: ${theme.breakpoints.values.sm}px; margin: ${theme.spacing(1)} 0; padding-top: ${theme.spacing(0.5)}; `, diff --git a/public/app/features/alerting/unified/components/silences/SilencesEditor.tsx b/public/app/features/alerting/unified/components/silences/SilencesEditor.tsx index a2bb9c45aa5..dd2bcdadf64 100644 --- a/public/app/features/alerting/unified/components/silences/SilencesEditor.tsx +++ b/public/app/features/alerting/unified/components/silences/SilencesEditor.tsx @@ -8,13 +8,13 @@ import { addDurationToDate, dateTime, isValidDate, - UrlQueryMap, GrafanaTheme2, } from '@grafana/data'; import { useDebounce } from 'react-use'; import { config } from '@grafana/runtime'; import { pickBy } from 'lodash'; import MatchersField from './MatchersField'; +import { MatchedSilencedRules } from './MatchedSilencedRules'; import { useForm, FormProvider } from 'react-hook-form'; import { SilenceFormFields } from '../../types/silence-form'; import { useDispatch } from 'react-redux'; @@ -24,35 +24,34 @@ import { css, cx } from '@emotion/css'; import { useUnifiedAlertingSelector } from '../../hooks/useUnifiedAlertingSelector'; import { makeAMLink } from '../../utils/misc'; import { useCleanup } from 'app/core/hooks/useCleanup'; -import { useQueryParams } from 'app/core/hooks/useQueryParams'; import { parseQueryParamMatchers } from '../../utils/matchers'; import { matcherToMatcherField, matcherFieldToMatcher } from '../../utils/alertmanager'; +import { useURLSearchParams } from '../../hooks/useURLSearchParams'; interface Props { silence?: Silence; alertManagerSourceName: string; } -const defaultsFromQuery = (queryParams: UrlQueryMap): Partial => { +const defaultsFromQuery = (searchParams: URLSearchParams): Partial => { const defaults: Partial = {}; - const { matchers, comment } = queryParams; + const comment = searchParams.get('comment'); + const matchers = searchParams.getAll('matcher'); - if (typeof matchers === 'string') { - const formMatchers = parseQueryParamMatchers(matchers); - if (formMatchers.length) { - defaults.matchers = formMatchers.map(matcherToMatcherField); - } + const formMatchers = parseQueryParamMatchers(matchers); + if (formMatchers.length) { + defaults.matchers = formMatchers.map(matcherToMatcherField); } - if (typeof comment === 'string') { + if (comment) { defaults.comment = comment; } return defaults; }; -const getDefaultFormValues = (queryParams: UrlQueryMap, silence?: Silence): SilenceFormFields => { +const getDefaultFormValues = (searchParams: URLSearchParams, silence?: Silence): SilenceFormFields => { const now = new Date(); if (silence) { const isExpired = Date.parse(silence.endsAt) < Date.now(); @@ -81,7 +80,7 @@ const getDefaultFormValues = (queryParams: UrlQueryMap, silence?: Silence): Sile id: '', startsAt: now.toISOString(), endsAt: endsAt.toISOString(), - comment: '', + comment: `created ${dateTime().format('YYYY-MM-DD HH:mm')}`, createdBy: config.bootData.user.name, duration: '2h', isRegex: false, @@ -89,14 +88,15 @@ const getDefaultFormValues = (queryParams: UrlQueryMap, silence?: Silence): Sile matcherName: '', matcherValue: '', timeZone: DefaultTimeZone, - ...defaultsFromQuery(queryParams), + ...defaultsFromQuery(searchParams), }; } }; export const SilencesEditor: FC = ({ silence, alertManagerSourceName }) => { - const [queryParams] = useQueryParams(); - const defaultValues = useMemo(() => getDefaultFormValues(queryParams, silence), [silence, queryParams]); + const [urlSearchParams] = useURLSearchParams(); + + const defaultValues = useMemo(() => getDefaultFormValues(urlSearchParams, silence), [silence, urlSearchParams]); const formAPI = useForm({ defaultValues }); const dispatch = useDispatch(); const styles = useStyles2(getStyles); @@ -165,7 +165,7 @@ export const SilencesEditor: FC = ({ silence, alertManagerSourceName }) =
-
+
= ({ silence, alertManagerSourceName }) = >