diff --git a/.bra.toml b/.bra.toml index 36421a3c5a6..aa7a1680adc 100644 --- a/.bra.toml +++ b/.bra.toml @@ -1,17 +1,18 @@ [run] init_cmds = [ - ["go", "build", "-o", "./bin/grafana-server", "./pkg/cmd/grafana-server"], + ["go", "run", "build.go", "-dev", "build-server"], ["./bin/grafana-server", "cfg:app_mode=development"] ] watch_all = true +follow_symlinks = true watch_dirs = [ "$WORKDIR/pkg", "$WORKDIR/public/views", "$WORKDIR/conf", ] -watch_exts = [".go", ".ini", ".toml"] +watch_exts = [".go", ".ini", ".toml", ".template.html"] build_delay = 1500 cmds = [ - ["go", "build", "-o", "./bin/grafana-server", "./pkg/cmd/grafana-server"], + ["go", "run", "build.go", "-dev", "build-server"], ["./bin/grafana-server", "cfg:app_mode=development"] ] diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..424744324ae --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,543 @@ +aliases: + # Workflow filters + - &filter-only-release + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - &filter-not-release-or-master + tags: + ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + branches: + ignore: master + - &filter-only-master + branches: + only: master + +version: 2 + +jobs: + mysql-integration-test: + docker: + - image: circleci/golang:1.11 + - image: circleci/mysql:5.6-ram + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: grafana_tests + MYSQL_USER: grafana + MYSQL_PASSWORD: password + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: sudo apt update + - run: sudo apt install -y mysql-client + - run: dockerize -wait tcp://127.0.0.1:3306 -timeout 120s + - run: cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h 127.0.0.1 -P 3306 -u root -prootpass + - run: + name: mysql integration tests + command: 'GRAFANA_TEST_DB=mysql go test ./pkg/services/sqlstore/... ./pkg/tsdb/mysql/... ' + + postgres-integration-test: + docker: + - image: circleci/golang:1.11 + - image: circleci/postgres:9.3-ram + environment: + POSTGRES_USER: grafanatest + POSTGRES_PASSWORD: grafanatest + POSTGRES_DB: grafanatest + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: sudo apt update + - run: sudo apt install -y postgresql-client + - run: dockerize -wait tcp://127.0.0.1:5432 -timeout 120s + - run: 'PGPASSWORD=grafanatest psql -p 5432 -h 127.0.0.1 -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql' + - run: + name: postgres integration tests + command: 'GRAFANA_TEST_DB=postgres go test ./pkg/services/sqlstore/... ./pkg/tsdb/postgres/...' + + codespell: + docker: + - image: circleci/python + steps: + - checkout + - run: + name: install codespell + command: 'sudo pip install codespell' + - run: + # Important: all words have to be in lowercase, and separated by "\n". + name: exclude known exceptions + command: 'echo -e "unknwon" > words_to_ignore.txt' + - run: + name: check documentation spelling errors + command: 'codespell -I ./words_to_ignore.txt docs/' + + gometalinter: + docker: + - image: circleci/golang:1.11 + environment: + # we need CGO because of go-sqlite3 + CGO_ENABLED: 1 + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: 'go get -u github.com/alecthomas/gometalinter' + - run: 'go get -u github.com/tsenart/deadcode' + - run: 'go get -u github.com/jgautheron/goconst/cmd/goconst' + - run: 'go get -u github.com/gordonklaus/ineffassign' + - run: 'go get -u honnef.co/go/tools/cmd/megacheck' + - run: 'go get -u github.com/opennota/check/cmd/structcheck' + - run: 'go get -u github.com/mdempsky/unconvert' + - run: 'go get -u github.com/opennota/check/cmd/varcheck' + - run: + name: run linters + command: 'gometalinter --enable-gc --vendor --deadline 10m --disable-all --enable=deadcode --enable=goconst --enable=gofmt --enable=ineffassign --enable=megacheck --enable=structcheck --enable=unconvert --enable=varcheck ./...' + - run: + name: run go vet + command: 'go vet ./pkg/...' + + test-frontend: + docker: + - image: circleci/node:8 + steps: + - checkout + - restore_cache: + key: dependency-cache-{{ checksum "yarn.lock" }} + - run: + name: yarn install + command: 'yarn install --pure-lockfile --no-progress' + no_output_timeout: 15m + - save_cache: + key: dependency-cache-{{ checksum "yarn.lock" }} + paths: + - node_modules + - run: + name: frontend tests + command: './scripts/circle-test-frontend.sh' + + test-backend: + docker: + - image: circleci/golang:1.11 + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: + name: build backend and run go tests + command: './scripts/circle-test-backend.sh' + + build-all: + docker: + - image: grafana/build-container:1.2.1 + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' + - restore_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + - run: + name: download phantomjs binaries + command: './scripts/build/download-phantomjs.sh' + - save_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + paths: + - /tmp/phantomjs + - run: + name: build and package grafana + command: './scripts/build/build-all.sh' + - run: + name: sign packages + command: './scripts/build/sign_packages.sh' + - run: + name: verify signed packages + command: | + mkdir -p ~/.rpmdb/pubkeys + curl -s https://grafanarel.s3.amazonaws.com/RPM-GPG-KEY-grafana > ~/.rpmdb/pubkeys/grafana.key + ./scripts/build/verify_signed_packages.sh dist/*.rpm + - run: + name: sha-sum packages + command: 'go run build.go sha-dist' + - run: + name: Build Grafana.com master publisher + command: 'go build -o scripts/publish scripts/build/publish.go' + - run: + name: Build Grafana.com release publisher + command: 'cd scripts/build/release_publisher && go build -o release_publisher .' + - persist_to_workspace: + root: . + paths: + - dist/grafana* + - scripts/*.sh + - scripts/publish + - scripts/build/release_publisher/release_publisher + - scripts/build/publish.sh + + build: + docker: + - image: grafana/build-container:1.2.1 + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' + - run: + name: build and package grafana + command: './scripts/build/build.sh' + - run: + name: sign packages + command: './scripts/build/sign_packages.sh' + - run: + name: sha-sum packages + command: 'go run build.go sha-dist' + - persist_to_workspace: + root: . + paths: + - dist/grafana* + + grafana-docker-master: + docker: + - image: docker:stable-git + steps: + - checkout + - attach_workspace: + at: . + - setup_remote_docker + - run: docker info + - run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker + - run: cd packaging/docker && ./build-deploy.sh "master-${CIRCLE_SHA1}" + - run: rm packaging/docker/grafana-latest.linux-x64.tar.gz + - run: cp enterprise-dist/grafana-enterprise-*.linux-amd64.tar.gz packaging/docker/grafana-latest.linux-x64.tar.gz + - run: cd packaging/docker && ./build-enterprise.sh "master" + + + grafana-docker-pr: + docker: + - image: docker:stable-git + steps: + - checkout + - attach_workspace: + at: . + - setup_remote_docker + - run: docker info + - run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker + - run: cd packaging/docker && ./build.sh "${CIRCLE_SHA1}" + + grafana-docker-release: + docker: + - image: docker:stable-git + steps: + - checkout + - attach_workspace: + at: . + - setup_remote_docker + - run: docker info + - run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker + - run: cd packaging/docker && ./build-deploy.sh "${CIRCLE_TAG}" + - run: rm packaging/docker/grafana-latest.linux-x64.tar.gz + - run: cp enterprise-dist/grafana-enterprise-*.linux-amd64.tar.gz packaging/docker/grafana-latest.linux-x64.tar.gz + - run: cd packaging/docker && ./build-enterprise.sh "${CIRCLE_TAG}" + + build-enterprise: + docker: + - image: grafana/build-container:1.2.1 + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' + - run: + name: checkout enterprise + command: './scripts/build/prepare-enterprise.sh' + - run: + name: test enterprise + command: 'go test ./pkg/extensions/...' + - run: + name: build and package enterprise + command: './scripts/build/build.sh -enterprise' + - run: + name: sign packages + command: './scripts/build/sign_packages.sh' + - run: + name: sha-sum packages + command: 'go run build.go sha-dist' + - run: + name: move enterprise packages into their own folder + command: 'mv dist enterprise-dist' + - persist_to_workspace: + root: . + paths: + - enterprise-dist/grafana-enterprise* + + build-all-enterprise: + docker: + - image: grafana/build-container:1.2.1 + working_directory: /go/src/github.com/grafana/grafana + steps: + - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' + - run: + name: checkout enterprise + command: './scripts/build/prepare-enterprise.sh' + - restore_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + - run: + name: download phantomjs binaries + command: './scripts/build/download-phantomjs.sh' + - save_cache: + key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }} + paths: + - /tmp/phantomjs + - run: + name: test enterprise + command: 'go test ./pkg/extensions/...' + - run: + name: build and package grafana + command: './scripts/build/build-all.sh -enterprise' + - run: + name: sign packages + command: './scripts/build/sign_packages.sh' + - run: + name: verify signed packages + command: | + mkdir -p ~/.rpmdb/pubkeys + curl -s https://grafanarel.s3.amazonaws.com/RPM-GPG-KEY-grafana > ~/.rpmdb/pubkeys/grafana.key + ./scripts/build/verify_signed_packages.sh dist/*.rpm + - run: + name: sha-sum packages + command: 'go run build.go sha-dist' + - run: + name: move enterprise packages into their own folder + command: 'mv dist enterprise-dist' + - persist_to_workspace: + root: . + paths: + - enterprise-dist/grafana-enterprise* + + deploy-enterprise-master: + docker: + - image: grafana/grafana-ci-deploy:1.0.0 + steps: + - attach_workspace: + at: . + - run: + name: gcp credentials + command: 'echo ${GCP_GRAFANA_UPLOAD_KEY} > /tmp/gcpkey.json' + - run: + name: sign in to gcp + command: '/opt/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json' + - run: + name: deploy to s3 + command: 'aws s3 sync ./enterprise-dist s3://$ENTERPRISE_BUCKET_NAME/master' + - run: + name: deploy to gcp + command: '/opt/google-cloud-sdk/bin/gsutil cp ./enterprise-dist/* gs://$GCP_BUCKET_NAME/enterprise/master' + - run: + name: Deploy to grafana.com + command: 'cd enterprise-dist && ../scripts/build/release_publisher/release_publisher -apikey ${GRAFANA_COM_API_KEY} -enterprise -from-local' + + + deploy-enterprise-release: + docker: + - image: grafana/grafana-ci-deploy:1.0.0 + steps: + - attach_workspace: + at: . + - run: + name: gcp credentials + command: 'echo ${GCP_GRAFANA_UPLOAD_KEY} > /tmp/gcpkey.json' + - run: + name: sign in to gcp + command: '/opt/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json' + - run: + name: deploy to s3 + command: 'aws s3 sync ./enterprise-dist s3://$ENTERPRISE_BUCKET_NAME/release' + - run: + name: deploy to gcp + command: '/opt/google-cloud-sdk/bin/gsutil cp ./enterprise-dist/* gs://$GCP_BUCKET_NAME/enterprise/release' + + deploy-master: + docker: + - image: grafana/grafana-ci-deploy:1.0.0 + steps: + - attach_workspace: + at: . + - run: + name: deploy to s3 + command: | + # Also + cp dist/grafana-latest.linux-x64.tar.gz dist/grafana-master-$(echo "${CIRCLE_SHA1}" | cut -b1-7).linux-x64.tar.gz + aws s3 sync ./dist s3://$BUCKET_NAME/master + - run: + name: Trigger Windows build + command: './scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} master' + - run: + name: gcp credentials + command: 'echo ${GCP_GRAFANA_UPLOAD_KEY} > /tmp/gcpkey.json' + - run: + name: sign in to gcp + command: '/opt/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json' + - run: + name: deploy to gcp + command: '/opt/google-cloud-sdk/bin/gsutil cp ./dist/* gs://$GCP_BUCKET_NAME/oss/master' + - run: + name: Publish to Grafana.com + command: | + rm dist/grafana-master-$(echo "${CIRCLE_SHA1}" | cut -b1-7).linux-x64.tar.gz + ./scripts/publish -apiKey ${GRAFANA_COM_API_KEY} + + deploy-release: + docker: + - image: grafana/grafana-ci-deploy:1.0.0 + steps: + - attach_workspace: + at: . + - run: + name: deploy to s3 + command: 'aws s3 sync ./dist s3://$BUCKET_NAME/release' + - run: + name: gcp credentials + command: 'echo ${GCP_GRAFANA_UPLOAD_KEY} > /tmp/gcpkey.json' + - run: + name: sign in to gcp + command: '/opt/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=/tmp/gcpkey.json' + - run: + name: deploy to gcp + command: '/opt/google-cloud-sdk/bin/gsutil cp ./dist/* gs://$GCP_BUCKET_NAME/oss/release' + - run: + name: Deploy to Grafana.com + command: './scripts/build/publish.sh' + +workflows: + version: 2 + build-master: + jobs: + - build-all: + filters: *filter-only-master + - build-all-enterprise: + filters: *filter-only-master + - codespell: + filters: *filter-only-master + - gometalinter: + filters: *filter-only-master + - test-frontend: + filters: *filter-only-master + - test-backend: + filters: *filter-only-master + - mysql-integration-test: + filters: *filter-only-master + - postgres-integration-test: + filters: *filter-only-master + - deploy-master: + requires: + - build-all + - test-backend + - test-frontend + - codespell + - gometalinter + - mysql-integration-test + - postgres-integration-test + filters: *filter-only-master + - grafana-docker-master: + requires: + - build-all + - build-all-enterprise + - test-backend + - test-frontend + - codespell + - gometalinter + - mysql-integration-test + - postgres-integration-test + filters: *filter-only-master + - deploy-enterprise-master: + requires: + - build-all + - test-backend + - test-frontend + - codespell + - gometalinter + - mysql-integration-test + - postgres-integration-test + - build-all-enterprise + filters: *filter-only-master + + release: + jobs: + - build-all: + filters: *filter-only-release + - build-all-enterprise: + filters: *filter-only-release + - codespell: + filters: *filter-only-release + - gometalinter: + filters: *filter-only-release + - test-frontend: + filters: *filter-only-release + - test-backend: + filters: *filter-only-release + - mysql-integration-test: + filters: *filter-only-release + - postgres-integration-test: + filters: *filter-only-release + - deploy-release: + requires: + - build-all + - test-backend + - test-frontend + - codespell + - gometalinter + - mysql-integration-test + - postgres-integration-test + filters: *filter-only-release + - deploy-enterprise-release: + requires: + - build-all + - build-all-enterprise + - test-backend + - test-frontend + - codespell + - gometalinter + - mysql-integration-test + - postgres-integration-test + filters: *filter-only-release + - grafana-docker-release: + requires: + - build-all + - test-backend + - test-frontend + - codespell + - gometalinter + - mysql-integration-test + - postgres-integration-test + filters: *filter-only-release + + build-branches-and-prs: + jobs: + - build: + filters: *filter-not-release-or-master + - codespell: + filters: *filter-not-release-or-master + - gometalinter: + filters: *filter-not-release-or-master + - test-frontend: + filters: *filter-not-release-or-master + - test-backend: + filters: *filter-not-release-or-master + - mysql-integration-test: + filters: *filter-not-release-or-master + - postgres-integration-test: + filters: *filter-not-release-or-master + - grafana-docker-pr: + requires: + - build + - test-backend + - test-frontend + - codespell + - gometalinter + - mysql-integration-test + - postgres-integration-test + filters: *filter-not-release-or-master diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..c535fa427b5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.awcache +.dockerignore +.git +.gitignore +.github +.vscode +bin +data* +dist +docker +Dockerfile +docs +dump.rdb +node_modules +/local +/tmp +*.yml +*.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index fe0a1d6c548..00000000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1,22 +0,0 @@ -Follow the setup guide in README.md - -### Rebuild frontend assets on source change -``` -grunt && grunt watch -``` - -### Rerun tests on source change -``` -grunt karma:dev -``` - -### Run tests for backend assets before commit -``` -test -z "$(gofmt -s -l . | grep -v -E 'vendor/(github.com|golang.org|gopkg.in)' | tee /dev/stderr)" -``` - -### Run tests for frontend assets before commit -``` -npm test -go test -v ./pkg/... -``` diff --git a/.gitignore b/.gitignore index 72f6684ef20..05ae4907e89 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,14 @@ node_modules npm-debug.log +yarn-error.log coverage/ .aws-config.json awsconfig +/.awcache /dist /public/build /public/views/index.html +/public/views/error.html /emails/dist /public_gen /public/vendor/npm @@ -31,25 +34,33 @@ public/css/*.min.css *.tmp .DS_Store .vscode/ +.vs/ /data/* /bin/* conf/custom.ini fig.yml -docker-compose.yml -docker-compose.yaml +devenv/docker-compose.yml +devenv/docker-compose.yaml /conf/provisioning/**/custom.yaml +/conf/provisioning/**/dev.yaml +/conf/ldap_dev.toml profile.cov /grafana +/local .notouch +/Makefile.local /pkg/cmd/grafana-cli/grafana-cli /pkg/cmd/grafana-server/grafana-server /pkg/cmd/grafana-server/debug +/pkg/extensions +/public/app/extensions debug.test /examples/*/dist /packaging/**/*.rpm /packaging/**/*.deb +/packaging/**/*.tar.gz # Ignore OSX indexing .DS_Store @@ -59,5 +70,9 @@ debug.test /vendor/**/*.yml /vendor/**/*_test.go /vendor/**/.editorconfig -/vendor/**/appengine* -*.orig \ No newline at end of file +*.orig + +/devenv/bulk-dashboards/*.json +/devenv/bulk_alerting_dashboards/*.json + +/scripts/build/release_publisher/release_publisher diff --git a/.jscs.json b/.jscs.json deleted file mode 100644 index 8fdad332de5..00000000000 --- a/.jscs.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "disallowImplicitTypeConversion": ["string"], - "disallowKeywords": ["with"], - "disallowMultipleLineBreaks": true, - "disallowMixedSpacesAndTabs": true, - "disallowTrailingWhitespace": true, - "requireSpacesInFunctionExpression": { - "beforeOpeningCurlyBrace": true - }, - "disallowSpacesInsideArrayBrackets": true, - "disallowSpacesInsideParentheses": true, - "validateIndentation": 2 -} diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index 1d8fad63173..00000000000 --- a/.jshintrc +++ /dev/null @@ -1,37 +0,0 @@ -{ - "browser": true, - "esversion": 6, - "bitwise":false, - "curly": true, - "eqnull": true, - "strict": false, - "devel": true, - "eqeqeq": true, - "forin": false, - "immed": true, - "supernew": true, - "expr": true, - "indent": 2, - "latedef": false, - "newcap": true, - "noarg": true, - "noempty": true, - "undef": true, - "boss": true, - "trailing": true, - "laxbreak": true, - "laxcomma": true, - "sub": true, - "unused": true, - "maxdepth": 6, - "maxlen": 140, - - "globals": { - "System": true, - "Promise": true, - "define": true, - "require": true, - "Chromath": false, - "setImmediate": true - } -} diff --git a/CHANGELOG.md b/CHANGELOG.md index 304b1ba6d0b..ea6b5b9732f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,24 +1,431 @@ -# 5.1.0 (unreleased) +# 5.4.0 (unreleased) + +### New Features + +* **Alerting**: Option to disable OK alert notifications [#12330](https://github.com/grafana/grafana/issues/12330) & [#6696](https://github.com/grafana/grafana/issues/6696), thx [@davewat](https://github.com/davewat) +* **Postgres/MySQL/MSSQL**: Adds support for configuration of max open/idle connections and connection max lifetime. Also, panels with multiple SQL queries will now be executed concurrently [#11711](https://github.com/grafana/grafana/issues/11711), thx [@connection-reset](https://github.com/connection-reset) +* **MySQL**: Graphical query builder [#13762](https://github.com/grafana/grafana/issues/13762), thx [svenklemm](https://github.com/svenklemm) +* **MySQL**: Support connecting thru Unix socket for MySQL datasource [#12342](https://github.com/grafana/grafana/issues/12342), thx [@Yukinoshita-Yukino](https://github.com/Yukinoshita-Yukino) +* **MSSQL**: Add encrypt setting to allow configuration of how data sent between client and server are encrypted [#13629](https://github.com/grafana/grafana/issues/13629), thx [@ramiro](https://github.com/ramiro) +* **Stackdriver**: Not possible to authenticate using GCE metadata server [#13669](https://github.com/grafana/grafana/issues/13669) +* **Teams**: Team preferences (theme, home dashboard, timezone) support [#12550](https://github.com/grafana/grafana/issues/12550) +* **Graph**: Time regions support enabling highlight of weekdays and/or certain timespans [#5930](https://github.com/grafana/grafana/issues/5930) + +### Minor + +* **Cloudwatch**: Show all available CloudWatch regions [#12308](https://github.com/grafana/grafana/issues/12308), thx [@mtanda](https://github.com/mtanda) +* **Cloudwatch**: AWS/Connect metrics and dimensions [#13970](https://github.com/grafana/grafana/pull/13970), thx [@zcoffy](https://github.com/zcoffy) +* **Postgres**: Add delta window function to postgres query builder [#13925](https://github.com/grafana/grafana/issues/13925), thx [svenklemm](https://github.com/svenklemm) +* **Elasticsearch**: Fix switching to/from es raw document metric query [#6367](https://github.com/grafana/grafana/issues/6367) +* **Elasticsearch**: Fix deprecation warning about terms aggregation order key in Elasticsearch 6.x [#11977](https://github.com/grafana/grafana/issues/11977) +* **Table**: Fix CSS alpha background-color applied twice in table cell with link [#13606](https://github.com/grafana/grafana/issues/13606), thx [@grisme](https://github.com/grisme) +* **Units**: New clock time format, to format ms or second values as for example `01h:59m`, [#13635](https://github.com/grafana/grafana/issues/13635), thx [@franciscocpg](https://github.com/franciscocpg) +* **Alerting**: Increaste default duration for queries [#13945](https://github.com/grafana/grafana/pull/13945) +* **Alerting**: More options for the Slack Alert notifier [#13993](https://github.com/grafana/grafana/issues/13993), thx [@andreykaipov](https://github.com/andreykaipov) +* **Alerting**: Can't receive DingDing alert when alert is triggered [#13723](https://github.com/grafana/grafana/issues/13723), thx [@Yukinoshita-Yukino](https://github.com/Yukinoshita-Yukino) +* **Internal metrics**: Renamed `grafana_info` to `grafana_build_info` and added branch, goversion and revision [#13876](https://github.com/grafana/grafana/pull/13876) +* **Datasource Proxy**: Keep trailing slash for datasource proxy requests [#13326](https://github.com/grafana/grafana/pull/13326), thx [@ryantxu](https://github.com/ryantxu) + +### Breaking changes + +* Postgres/MySQL/MSSQL datasources now per default uses `max open connections` = `unlimited` (earlier 10), `max idle connections` = `2` (earlier 10) and `connection max lifetime` = `4` hours (earlier unlimited) + +# 5.3.5 (unreleased) + +* **Security**: Upgrade macaron session package to fix security issue. [#14043](https://github.com/grafana/grafana/pull/14043) + +# 5.3.4 (2018-11-13) + +* **Alerting**: Delete alerts when parent folder was deleted [#13322](https://github.com/grafana/grafana/issues/13322) +* **MySQL**: Fix `$__timeFilter()` should respect local time zone [#13769](https://github.com/grafana/grafana/issues/13769) +* **Dashboard**: Fix datasource selection in panel by enter key [#13932](https://github.com/grafana/grafana/issues/13932) +* **Graph**: Fix table legend height when positioned below graph and using Internet Explorer 11 [#13903](https://github.com/grafana/grafana/issues/13903) +* **Dataproxy**: Drop origin and referer http headers [#13328](https://github.com/grafana/grafana/issues/13328) [#13949](https://github.com/grafana/grafana/issues/13949), thx [@roidelapluie](https://github.com/roidelapluie) + +# 5.3.3 (2018-11-13) + +### File Exfiltration vulnerability Security fix + +See [security announcement](https://community.grafana.com/t/grafana-5-3-3-and-4-6-5-security-update/11961) for details. + +# 5.3.2 (2018-10-24) + +* **InfluxDB/Graphite/Postgres**: Prevent cross site scripting (XSS) in query editor [#13667](https://github.com/grafana/grafana/issues/13667), thx [@svenklemm](https://github.com/svenklemm) +* **Postgres**: Fix template variables error [#13692](https://github.com/grafana/grafana/issues/13692), thx [@svenklemm](https://github.com/svenklemm) +* **Cloudwatch**: Fix service panic because of race conditions [#13674](https://github.com/grafana/grafana/issues/13674), thx [@mtanda](https://github.com/mtanda) +* **Cloudwatch**: Fix check for invalid percentile statistics [#13633](https://github.com/grafana/grafana/issues/13633), thx [@apalaniuk](https://github.com/apalaniuk) +* **Stackdriver/Cloudwatch**: Allow user to change unit in graph panel if cloudwatch/stackdriver datasource response doesn't include unit [#13718](https://github.com/grafana/grafana/issues/13718), thx [@mtanda](https://github.com/mtanda) +* **Stackdriver**: stackdriver user-metrics duplicated response when multiple resource types [#13691](https://github.com/grafana/grafana/issues/13691) +* **Variables**: Fix text box template variable doesn't work properly without a default value [#13666](https://github.com/grafana/grafana/issues/13666) +* **Variables**: Fix variable dependency check when using `${var}` format [#13600](https://github.com/grafana/grafana/issues/13600) +* **Dashboard**: Fix kiosk=1 url parameter should put dashboard in kiosk mode [#13764](https://github.com/grafana/grafana/pull/13764) +* **LDAP**: Fix super admins can also be admins of orgs [#13710](https://github.com/grafana/grafana/issues/13710), thx [@adrien-f](https://github.com/adrien-f) +* **Provisioning**: Fix deleting provisioned dashboard folder should cleanup provisioning meta data [#13280](https://github.com/grafana/grafana/issues/13280) + +### Minor + +* **Docker**: adds curl back into the docker image for utility. [#13794](https://github.com/grafana/grafana/pull/13794) + +# 5.3.1 (2018-10-16) + +* **Render**: Fix PhantomJS render of graph panel when legend displayed as table to the right [#13616](https://github.com/grafana/grafana/issues/13616) +* **Stackdriver**: Filter option disappears after removing initial filter [#13607](https://github.com/grafana/grafana/issues/13607) +* **Elasticsearch**: Fix no limit size in terms aggregation for alerting queries [#13172](https://github.com/grafana/grafana/issues/13172), thx [@Yukinoshita-Yukino](https://github.com/Yukinoshita-Yukino) +* **InfluxDB**: Fix for annotation issue that caused text to be shown twice [#13553](https://github.com/grafana/grafana/issues/13553) +* **Variables**: Fix nesting variables leads to exception and missing refresh [#13628](https://github.com/grafana/grafana/issues/13628) +* **Variables**: Prometheus: Single letter labels are not supported [#13641](https://github.com/grafana/grafana/issues/13641), thx [@olshansky](https://github.com/olshansky) +* **Graph**: Fix graph time formatting for Last 24h ranges [#13650](https://github.com/grafana/grafana/issues/13650) +* **Playlist**: Fix cannot add dashboards with long names to playlist [#13464](https://github.com/grafana/grafana/issues/13464), thx [@neufeldtech](https://github.com/neufeldtech) +* **HTTP API**: Fix /api/org/users so that query and limit querystrings works + +# 5.3.0 (2018-10-10) + +* **Stackdriver**: Filter wildcards and regex matching are not yet supported [#13495](https://github.com/grafana/grafana/issues/13495) +* **Stackdriver**: Support the distribution metric type for heatmaps [#13559](https://github.com/grafana/grafana/issues/13559) +* **Cloudwatch**: Automatically set graph yaxis unit [#13575](https://github.com/grafana/grafana/issues/13575), thx [@mtanda](https://github.com/mtanda) + +# 5.3.0-beta3 (2018-10-03) + +* **Stackdriver**: Fix for missing ngInject [#13511](https://github.com/grafana/grafana/pull/13511) +* **Permissions**: Fix for broken permissions selector [#13507](https://github.com/grafana/grafana/issues/13507) +* **Alerting**: Alert reminders deduping not working as expected when running multiple Grafana instances [#13492](https://github.com/grafana/grafana/issues/13492) + +# 5.3.0-beta2 (2018-10-01) + +### New Features + +* **Annotations**: Enable template variables in tagged annotations queries [#9735](https://github.com/grafana/grafana/issues/9735) +* **Stackdriver**: Support for Google Stackdriver Datasource [#13289](https://github.com/grafana/grafana/pull/13289) + +### Minor + +* **Provisioning**: Dashboard Provisioning now support symlinks that changes target [#12534](https://github.com/grafana/grafana/issues/12534), thx [@auhlig](https://github.com/auhlig) +* **OAuth**: Allow oauth email attribute name to be configurable [#12986](https://github.com/grafana/grafana/issues/12986), thx [@bobmshannon](https://github.com/bobmshannon) +* **Tags**: Default sort order for GetDashboardTags [#11681](https://github.com/grafana/grafana/pull/11681), thx [@Jonnymcc](https://github.com/Jonnymcc) +* **Prometheus**: Label completion queries respect dashboard time range [#12251](https://github.com/grafana/grafana/pull/12251), thx [@mtanda](https://github.com/mtanda) +* **Prometheus**: Allow to display annotations based on Prometheus series value [#10159](https://github.com/grafana/grafana/issues/10159), thx [@mtanda](https://github.com/mtanda) +* **Prometheus**: Adhoc-filtering for Prometheus dashboards [#13212](https://github.com/grafana/grafana/issues/13212) +* **Singlestat**: Fix gauge display accuracy for percents [#13270](https://github.com/grafana/grafana/issues/13270), thx [@tianon](https://github.com/tianon) +* **Dashboard**: Prevent auto refresh from starting when loading dashboard with absolute time range [#12030](https://github.com/grafana/grafana/issues/12030) +* **Templating**: New templating variable type `Text box` that allows free text input [#3173](https://github.com/grafana/grafana/issues/3173) +* **Alerting**: Link to view full size image in Microsoft Teams alert notifier [#13121](https://github.com/grafana/grafana/issues/13121), thx [@holiiveira](https://github.com/holiiveira) +* **Alerting**: Fixes a bug where all alerts would send reminders after upgrade & restart [#13402](https://github.com/grafana/grafana/pull/13402) +* **Alerting**: Concurrent render limit for graphs used in notifications [#13401](https://github.com/grafana/grafana/pull/13401) +* **Postgres/MySQL/MSSQL**: Add support for replacing $__interval and $__interval_ms in alert queries [#11555](https://github.com/grafana/grafana/issues/11555), thx [@svenklemm](https://github.com/svenklemm) + +# 5.3.0-beta1 (2018-09-06) + +### New Major Features + +* **Alerting**: Notification reminders [#7330](https://github.com/grafana/grafana/issues/7330), thx [@jbaublitz](https://github.com/jbaublitz) +* **Dashboard**: TV & Kiosk mode changes, new cycle view mode button in dashboard toolbar [#13025](https://github.com/grafana/grafana/pull/13025) +* **OAuth**: Gitlab OAuth with support for filter by groups [#5623](https://github.com/grafana/grafana/issues/5623), thx [@BenoitKnecht](https://github.com/BenoitKnecht) +* **Postgres**: Graphical query builder [#10095](https://github.com/grafana/grafana/issues/10095), thx [svenklemm](https://github.com/svenklemm) + +### New Features + +* **LDAP**: Define Grafana Admin permission in ldap group mappings [#2469](https://github.com/grafana/grafana/issues/2496), PR [#12622](https://github.com/grafana/grafana/issues/12622) +* **LDAP**: Client certificates support [#12805](https://github.com/grafana/grafana/issues/12805), thx [@nyxi](https://github.com/nyxi) +* **Profile**: List teams that the user is member of in current/active organization [#12476](https://github.com/grafana/grafana/issues/12476) +* **Configuration**: Allow auto-assigning users to specific organization (other than Main. Org) [#1823](https://github.com/grafana/grafana/issues/1823) [#12801](https://github.com/grafana/grafana/issues/12801), thx [@gzzo](https://github.com/gzzo) and [@ofosos](https://github.com/ofosos) +* **Dataproxy**: Pass configured/auth headers to a Datasource [#10971](https://github.com/grafana/grafana/issues/10971), thx [@mrsiano](https://github.com/mrsiano) +* **CloudWatch**: GetMetricData support [#11487](https://github.com/grafana/grafana/issues/11487), thx [@mtanda](https://github.com/mtanda) +* **Postgres**: TimescaleDB support, e.g. use `time_bucket` for grouping by time when option enabled [#12680](https://github.com/grafana/grafana/pull/12680), thx [svenklemm](https://github.com/svenklemm) +* **Cleanup**: Make temp file time to live configurable [#11607](https://github.com/grafana/grafana/issues/11607), thx [@xapon](https://github.com/xapon) + +### Minor + +* **Alerting**: Its now possible to configure the default value for how to handle errors and no data in alerting. [#10424](https://github.com/grafana/grafana/issues/10424) +* **Alerting**: Fix diff and percent_diff reducers [#11563](https://github.com/grafana/grafana/issues/11563), thx [@jessetane](https://github.com/jessetane) +* **Alerting**: Fix rendering timeout which could cause notifications to not be sent due to rendering timing out [#12151](https://github.com/grafana/grafana/issues/12151) +* **Docker**: Make it possible to set a specific plugin url [#12861](https://github.com/grafana/grafana/pull/12861), thx [ClementGautier](https://github.com/ClementGautier) +* **GrafanaCli**: Fixed issue with grafana-cli install plugin resulting in corrupt http response from source error. Fixes [#13079](https://github.com/grafana/grafana/issues/13079) +* **Provisioning**: Should allow one default datasource per organisation [#12229](https://github.com/grafana/grafana/issues/12229) +* **Github OAuth**: Allow changes of user info at Github to be synched to Grafana when signing in [#11818](https://github.com/grafana/grafana/issues/11818), thx [@rwaweber](https://github.com/rwaweber) +* **OAuth**: Fix overriding tls_skip_verify_insecure using environment variable [#12747](https://github.com/grafana/grafana/issues/12747), thx [@jangaraj](https://github.com/jangaraj) +* **Prometheus**: Fix graph panel bar width issue in aligned prometheus queries [#12379](https://github.com/grafana/grafana/issues/12379) +* **Prometheus**: Heatmap - fix unhandled error when some points are missing [#12484](https://github.com/grafana/grafana/issues/12484) +* **Prometheus**: Add $__interval, $__interval_ms, $__range, $__range_s & $__range_ms support for dashboard and template queries [#12597](https://github.com/grafana/grafana/issues/12597) [#12882](https://github.com/grafana/grafana/issues/12882), thx [@roidelapluie](https://github.com/roidelapluie) +* **Elasticsearch**: For alerting/backend, support having index name to the right of pattern in index pattern [#12731](https://github.com/grafana/grafana/issues/12731) +* **Graphite**: Fix for quoting of int function parameters (when using variables) [#11927](https://github.com/grafana/grafana/pull/11927) +* **InfluxDB**: Support timeFilter in query templating for InfluxDB [#12598](https://github.com/grafana/grafana/pull/12598), thx [kichristensen](https://github.com/kichristensen) +* **Postgres/MySQL/MSSQL**: New $__unixEpochGroup and $__unixEpochGroupAlias macros [#12892](https://github.com/grafana/grafana/issues/12892), thx [@svenklemm](https://github.com/svenklemm) +* **Postgres/MySQL/MSSQL**: Add previous fill mode to $__timeGroup macro which will fill in previously seen value when point is missing [#12756](https://github.com/grafana/grafana/issues/12756), thx [@svenklemm](https://github.com/svenklemm) +* **Postgres/MySQL/MSSQL**: Use floor rounding in $__timeGroup macro function [#12460](https://github.com/grafana/grafana/issues/12460), thx [@svenklemm](https://github.com/svenklemm) +* **Postgres/MySQL/MSSQL**: Use metric column as prefix when returning multiple value columns [#12727](https://github.com/grafana/grafana/issues/12727), thx [@svenklemm](https://github.com/svenklemm) +* **Postgres/MySQL/MSSQL**: New $__timeGroupAlias macro. Postgres $__timeGroup no longer automatically adds time column alias [#12749](https://github.com/grafana/grafana/issues/12749), thx [@svenklemm](https://github.com/svenklemm) +* **Postgres/MySQL/MSSQL**: Escape single quotes in variables [#12785](https://github.com/grafana/grafana/issues/12785), thx [@eMerzh](https://github.com/eMerzh) +* **Postgres/MySQL/MSSQL**: Min time interval support [#13157](https://github.com/grafana/grafana/issues/13157), thx [@svenklemm](https://github.com/svenklemm) +* **MySQL/MSSQL**: Use datetime format instead of epoch for $__timeFilter, $__timeFrom and $__timeTo macros [#11618](https://github.com/grafana/grafana/issues/11618) [#11619](https://github.com/grafana/grafana/issues/11619), thx [@AustinWinstanley](https://github.com/AustinWinstanley) +* **Postgres**: Escape ssl mode parameter in connectionstring [#12644](https://github.com/grafana/grafana/issues/12644), thx [@yogyrahmawan](https://github.com/yogyrahmawan) +* **Cloudwatch**: Improved error handling [#12489](https://github.com/grafana/grafana/issues/12489), thx [@mtanda](https://github.com/mtanda) +* **Cloudwatch**: AppSync metrics and dimensions [#12300](https://github.com/grafana/grafana/issues/12300), thx [@franciscocpg](https://github.com/franciscocpg) +* **Cloudwatch**: Direct Connect metrics and dimensions [#12762](https://github.com/grafana/grafana/pulls/12762), thx [@mindriot88](https://github.com/mindriot88) +* **Cloudwatch**: Added BurstBalance metric to list of AWS RDS metrics [#12561](https://github.com/grafana/grafana/pulls/12561), thx [@activeshadow](https://github.com/activeshadow) +* **Cloudwatch**: Add new Redshift metrics and dimensions [#12063](https://github.com/grafana/grafana/pulls/12063), thx [@A21z](https://github.com/A21z) +* **Dashboard**: Fix selecting current dashboard from search should not reload dashboard [#12248](https://github.com/grafana/grafana/issues/12248) +* **Dashboard**: Use uid when linking to dashboards internally in a dashboard [#10705](https://github.com/grafana/grafana/issues/10705) +* **Graph**: Option to hide series from tooltip [#3341](https://github.com/grafana/grafana/issues/3341), thx [@mtanda](https://github.com/mtanda) +* **Singlestat**: Make colorization of prefix and postfix optional in singlestat [#11892](https://github.com/grafana/grafana/pull/11892), thx [@ApsOps](https://github.com/ApsOps) +* **Table**: Adjust header contrast for the light theme [#12668](https://github.com/grafana/grafana/issues/12668) +* **Table**: Fix link color when using light theme and thresholds in use [#12766](https://github.com/grafana/grafana/issues/12766) +* **Table**: Fix for useless horizontal scrollbar for table panel [#9964](https://github.com/grafana/grafana/issues/9964) +* **Table**: Make table sorting stable when null values exist [#12362](https://github.com/grafana/grafana/pull/12362), thx [@bz2](https://github.com/bz2) +* **Heatmap**: Fix broken tooltip and crosshair on Firefox [#12486](https://github.com/grafana/grafana/issues/12486) +* **Datasource**: Fix UI issue with secret fields after updating datasource [#11270](https://github.com/grafana/grafana/issues/11270) +* **Variables**: Skip unneeded extra query request when de-selecting variable values used for repeated panels [#8186](https://github.com/grafana/grafana/issues/8186), thx [@mtanda](https://github.com/mtanda) +* **Variables**: Limit amount of queries executed when updating variable that other variable(s) are dependent on [#11890](https://github.com/grafana/grafana/issues/11890) +* **Variables**: Support query variable refresh when another variable referenced in `Regex` field change its value [#12952](https://github.com/grafana/grafana/issues/12952), thx [@franciscocpg](https://github.com/franciscocpg) +* **Variables**: Support variables in query variable `Custom all value` field [#12965](https://github.com/grafana/grafana/issues/12965), thx [@franciscocpg](https://github.com/franciscocpg) +* **Units**: Change units to include characters for power of 2 and 3 [#12744](https://github.com/grafana/grafana/pull/12744), thx [@Worty](https://github.com/Worty) +* **Units**: Polish złoty currency [#12691](https://github.com/grafana/grafana/pull/12691), thx [@mwegrzynek](https://github.com/mwegrzynek) +* **Units**: Adds bitcoin axes unit. [#13125](https://github.com/grafana/grafana/pull/13125) +* **Api**: Delete nonexistent datasource should return 404 [#12313](https://github.com/grafana/grafana/issues/12313), thx [@AustinWinstanley](https://github.com/AustinWinstanley) +* **Logging**: Reopen log files after receiving a SIGHUP signal [#13112](https://github.com/grafana/grafana/pull/13112), thx [@filewalkwithme](https://github.com/filewalkwithme) +* **Login**: Show loading animation while waiting for authentication response on login [#12865](https://github.com/grafana/grafana/issues/12865) +* **UI**: Fix iOS home screen "app" icon and Windows 10 app experience [#12752](https://github.com/grafana/grafana/issues/12752), thx [@andig](https://github.com/andig) +* **Plugins**: Convert URL-like text to links in plugins readme [#12843](https://github.com/grafana/grafana/pull/12843), thx [pgiraud](https://github.com/pgiraud) + +### Breaking changes + +* Postgres datasource no longer automatically adds time column alias when using the $__timeGroup alias. However, there's code in place which should make this change backward compatible and shouldn't create any issues. +* Kiosk mode now also hides submenu (variables) +* ?inactive url parameter no longer supported, replaced with kiosk=tv url parameter + +### New experimental features + +These are new features that's still being worked on and are in an experimental phase. We encourage users to try these out and provide any feedback in related issue. + +* **Dashboard**: Auto fit dashboard panels to optimize space used for current TV / Monitor [#12768](https://github.com/grafana/grafana/issues/12768) + +### Tech + +* **Frontend**: Convert all Frontend Karma tests to Jest tests [#12224](https://github.com/grafana/grafana/issues/12224) +* **Backend**: Upgrade to golang 1.11 [#13030](https://github.com/grafana/grafana/issues/13030) + +# 5.2.4 (2018-09-07) + +* **GrafanaCli**: Fixed issue with grafana-cli install plugin resulting in corrupt http response from source error. Fixes [#13079](https://github.com/grafana/grafana/issues/13079) + +# 5.2.3 (2018-08-29) + +### Important fix for LDAP & OAuth login vulnerability + +See [security announcement](https://community.grafana.com/t/grafana-5-2-3-and-4-6-4-security-update/10050) for details. + +# 5.2.2 (2018-07-25) + +### Minor + +* **Prometheus**: Fix graph panel bar width issue in aligned prometheus queries [#12379](https://github.com/grafana/grafana/issues/12379) +* **Dashboard**: Dashboard links not updated when changing variables [#12506](https://github.com/grafana/grafana/issues/12506) +* **Postgres/MySQL/MSSQL**: Fix connection leak [#12636](https://github.com/grafana/grafana/issues/12636) [#9827](https://github.com/grafana/grafana/issues/9827) +* **Plugins**: Fix loading of external plugins [#12551](https://github.com/grafana/grafana/issues/12551) +* **Dashboard**: Remove unwanted scrollbars in embedded panels [#12589](https://github.com/grafana/grafana/issues/12589) +* **Prometheus**: Prevent error using $__interval_ms in query [#12533](https://github.com/grafana/grafana/pull/12533), thx [@mtanda](https://github.com/mtanda) + +# 5.2.1 (2018-06-29) + +### Minor + +* **Auth Proxy**: Important security fix for whitelist of IP address feature [#12444](https://github.com/grafana/grafana/pull/12444) +* **UI**: Fix - Grafana footer overlapping page [#12430](https://github.com/grafana/grafana/issues/12430) +* **Logging**: Errors should be reported before crashing [#12438](https://github.com/grafana/grafana/issues/12438) + +# 5.2.0-stable (2018-06-27) + +### Minor + +* **Plugins**: Handle errors correctly when loading datasource plugin [#12383](https://github.com/grafana/grafana/pull/12383) thx [@rozetko](https://github.com/rozetko) +* **Render**: Enhance error message if phantomjs executable is not found [#11868](https://github.com/grafana/grafana/issues/11868) +* **Dashboard**: Set correct text in drop down when variable is present in url [#11968](https://github.com/grafana/grafana/issues/11968) + +### 5.2.0-beta3 fixes + +* **LDAP**: Handle "dn" ldap attribute more gracefully [#12385](https://github.com/grafana/grafana/pull/12385), reverts [#10970](https://github.com/grafana/grafana/pull/10970) + +# 5.2.0-beta3 (2018-06-21) + +### Minor + +* **Build**: All rpm packages should be signed [#12359](https://github.com/grafana/grafana/issues/12359) + +# 5.2.0-beta2 (2018-06-20) + +### New Features + +* **Dashboard**: Import dashboard to folder [#10796](https://github.com/grafana/grafana/issues/10796) + +### Minor + +* **Permissions**: Important security fix for API keys with viewer role [#12343](https://github.com/grafana/grafana/issues/12343) +* **Dashboard**: Fix so panel titles doesn't wrap [#11074](https://github.com/grafana/grafana/issues/11074) +* **Dashboard**: Prevent double-click when saving dashboard [#11963](https://github.com/grafana/grafana/issues/11963) +* **Dashboard**: AutoFocus the add-panel search filter [#12189](https://github.com/grafana/grafana/pull/12189) thx [@ryantxu](https://github.com/ryantxu) +* **Units**: W/m2 (energy), l/h (flow) and kPa (pressure) [#11233](https://github.com/grafana/grafana/pull/11233), thx [@flopp999](https://github.com/flopp999) +* **Units**: Litre/min (flow) and milliLitre/min (flow) [#12282](https://github.com/grafana/grafana/pull/12282), thx [@flopp999](https://github.com/flopp999) +* **Alerting**: Fix mobile notifications for Microsoft Teams alert notifier [#11484](https://github.com/grafana/grafana/pull/11484), thx [@manacker](https://github.com/manacker) +* **Influxdb**: Add support for mode function [#12286](https://github.com/grafana/grafana/issues/12286) +* **Cloudwatch**: Fixes panic caused by bad timerange settings [#12199](https://github.com/grafana/grafana/issues/12199) +* **Auth Proxy**: Whitelist proxy IP address instead of client IP address [#10707](https://github.com/grafana/grafana/issues/10707) +* **User Management**: Make sure that a user always has a current org assigned [#11076](https://github.com/grafana/grafana/issues/11076) +* **Snapshots**: Fix: annotations not properly extracted leading to incorrect rendering of annotations [#12278](https://github.com/grafana/grafana/issues/12278) +* **LDAP**: Allow use of DN in group_search_filter_user_attribute and member_of [#3132](https://github.com/grafana/grafana/issues/3132), thx [@mmolnar](https://github.com/mmolnar) +* **Graph**: Fix legend decimals precision calculation [#11792](https://github.com/grafana/grafana/issues/11792) +* **Dashboard**: Make sure to process panels in collapsed rows when exporting dashboard [#12256](https://github.com/grafana/grafana/issues/12256) + +### 5.2.0-beta1 fixes + +* **Dashboard**: Dashboard link doesn't work when "As dropdown" option is checked [#12315](https://github.com/grafana/grafana/issues/12315) +* **Dashboard**: Fix regressions after save modal changes, including adhoc template issues [#12240](https://github.com/grafana/grafana/issues/12240) +* **Docker**: Config keys ending with _FILE are not respected [#170](https://github.com/grafana/grafana-docker/issues/170) + +# 5.2.0-beta1 (2018-06-05) + +### New Features + +* **Elasticsearch**: Alerting support [#5893](https://github.com/grafana/grafana/issues/5893), thx [@WPH95](https://github.com/WPH95) +* **Build**: Crosscompile and packages Grafana on arm, windows, linux and darwin [#11920](https://github.com/grafana/grafana/pull/11920), thx [@fg2it](https://github.com/fg2it) +* **Login**: Change admin password after first login [#11882](https://github.com/grafana/grafana/issues/11882) +* **Alert list panel**: Updated to support filtering alerts by name, dashboard title, folder, tags [#11500](https://github.com/grafana/grafana/issues/11500), [#8168](https://github.com/grafana/grafana/issues/8168), [#6541](https://github.com/grafana/grafana/issues/6541) + +### Minor + +* **Dashboard**: Modified time range and variables are now not saved by default [#10748](https://github.com/grafana/grafana/issues/10748), [#8805](https://github.com/grafana/grafana/issues/8805) +* **Graph**: Show invisible highest value bucket in histogram [#11498](https://github.com/grafana/grafana/issues/11498) +* **Dashboard**: Enable "Save As..." if user has edit permission [#11625](https://github.com/grafana/grafana/issues/11625) +* **Prometheus**: Query dates are now step-aligned [#10434](https://github.com/grafana/grafana/pull/10434) +* **Prometheus**: Table columns order now changes when rearrange queries [#11690](https://github.com/grafana/grafana/issues/11690), thx [@mtanda](https://github.com/mtanda) +* **Variables**: Fix variable interpolation when using multiple formatting types [#11800](https://github.com/grafana/grafana/issues/11800), thx [@svenklemm](https://github.com/svenklemm) +* **Dashboard**: Fix date selector styling for dark/light theme in time picker control [#11616](https://github.com/grafana/grafana/issues/11616) +* **Discord**: Alert notification channel type for Discord, [#7964](https://github.com/grafana/grafana/issues/7964) thx [@jereksel](https://github.com/jereksel), +* **InfluxDB**: Support SELECT queries in templating query, [#5013](https://github.com/grafana/grafana/issues/5013) +* **InfluxDB**: Support count distinct aggregation [#11645](https://github.com/grafana/grafana/issues/11645), thx [@kichristensen](https://github.com/kichristensen) +* **Dashboard**: JSON Model under dashboard settings can now be updated & changes saved, [#1429](https://github.com/grafana/grafana/issues/1429), thx [@jereksel](https://github.com/jereksel) +* **Security**: Fix XSS vulnerabilities in dashboard links [#11813](https://github.com/grafana/grafana/pull/11813) +* **Singlestat**: Fix "time of last point" shows local time when dashboard timezone set to UTC [#10338](https://github.com/grafana/grafana/issues/10338) +* **Prometheus**: Add support for passing timeout parameter to Prometheus [#11788](https://github.com/grafana/grafana/pull/11788), thx [@mtanda](https://github.com/mtanda) +* **Login**: Add optional option sign out url for generic oauth [#9847](https://github.com/grafana/grafana/issues/9847), thx [@roidelapluie](https://github.com/roidelapluie) +* **Login**: Use proxy server from environment variable if available [#9703](https://github.com/grafana/grafana/issues/9703), thx [@iyeonok](https://github.com/iyeonok) +* **Invite users**: Friendlier error message when smtp is not configured [#12087](https://github.com/grafana/grafana/issues/12087), thx [@thurt](https://github.com/thurt) +* **Graphite**: Don't send distributed tracing headers when using direct/browser access mode [#11494](https://github.com/grafana/grafana/issues/11494) +* **Sidenav**: Show create dashboard link for viewers if at least editor in one folder [#11858](https://github.com/grafana/grafana/issues/11858) +* **SQL**: Second epochs are now correctly converted to ms. [#12085](https://github.com/grafana/grafana/pull/12085) +* **Singlestat**: Fix singlestat threshold tooltip [#11971](https://github.com/grafana/grafana/issues/11971) +* **Dashboard**: Hide grid controls in fullscreen/low-activity views [#11771](https://github.com/grafana/grafana/issues/11771) +* **Dashboard**: Validate uid when importing dashboards [#11515](https://github.com/grafana/grafana/issues/11515) +* **Docker**: Support for env variables ending with _FILE [grafana-docker #166](https://github.com/grafana/grafana-docker/pull/166), thx [@efrecon](https://github.com/efrecon) +* **Alert list panel**: Show alerts for user with viewer role [#11167](https://github.com/grafana/grafana/issues/11167) +* **Provisioning**: Verify checksum of dashboards before updating to reduce load on database [#11670](https://github.com/grafana/grafana/issues/11670) +* **Provisioning**: Support symlinked files in dashboard provisioning config files [#11958](https://github.com/grafana/grafana/issues/11958) +* **Dashboard list panel**: Search dashboards by folder [#11525](https://github.com/grafana/grafana/issues/11525) +* **Sidenav**: Always show server admin link in sidenav if grafana admin [#11657](https://github.com/grafana/grafana/issues/11657) + +# 5.1.5 (2018-06-27) + +* **Docker**: Config keys ending with _FILE are not respected [#170](https://github.com/grafana/grafana-docker/issues/170) + +# 5.1.4 (2018-06-19) + +* **Permissions**: Important security fix for API keys with viewer role [#12343](https://github.com/grafana/grafana/issues/12343) + +# 5.1.3 (2018-05-16) + +* **Scroll**: Graph panel / legend texts shifts on the left each time we move scrollbar on firefox [#11830](https://github.com/grafana/grafana/issues/11830) + +# 5.1.2 (2018-05-09) + +* **Database**: Fix MySql migration issue [#11862](https://github.com/grafana/grafana/issues/11862) +* **Google Analytics**: Enable Google Analytics anonymizeIP setting for GDPR [#11656](https://github.com/grafana/grafana/pull/11656) + +# 5.1.1 (2018-05-07) + +* **LDAP**: LDAP login with MariaDB/MySQL database and dn>100 chars not possible [#11754](https://github.com/grafana/grafana/issues/11754) +* **Build**: AppVeyor Windows build missing version and commit info [#11758](https://github.com/grafana/grafana/issues/11758) +* **Scroll**: Scroll can't start in graphs on Chrome mobile [#11710](https://github.com/grafana/grafana/issues/11710) +* **Units**: Revert renaming of unit key ppm [#11743](https://github.com/grafana/grafana/issues/11743) + +# 5.1.0 (2018-04-26) + +* **Folders**: Default permissions on folder are not shown as inherited in its dashboards [#11668](https://github.com/grafana/grafana/issues/11668) +* **Templating**: Allow more than 20 previews when creating a variable [#11508](https://github.com/grafana/grafana/issues/11508) +* **Dashboard**: Row edit icon not shown [#11466](https://github.com/grafana/grafana/issues/11466) +* **SQL**: Unsupported data types for value column using time series query [#11703](https://github.com/grafana/grafana/issues/11703) +* **Prometheus**: Prometheus query inspector expands to be very large on autocomplete queries [#11673](https://github.com/grafana/grafana/issues/11673) + +# 5.1.0-beta1 (2018-04-20) * **MSSQL**: New Microsoft SQL Server data source [#10093](https://github.com/grafana/grafana/pull/10093), [#11298](https://github.com/grafana/grafana/pull/11298), thx [@linuxchips](https://github.com/linuxchips) * **Prometheus**: The heatmap panel now support Prometheus histograms [#10009](https://github.com/grafana/grafana/issues/10009) * **Postgres/MySQL**: Ability to insert 0s or nulls for missing intervals [#9487](https://github.com/grafana/grafana/issues/9487), thanks [@svenklemm](https://github.com/svenklemm) +* **Postgres/MySQL/MSSQL**: Fix precision for the time column in table mode [#11306](https://github.com/grafana/grafana/issues/11306) +* **Graph**: Align left and right Y-axes to one level [#1271](https://github.com/grafana/grafana/issues/1271) & [#2740](https://github.com/grafana/grafana/issues/2740) thx [@ilgizar](https://github.com/ilgizar) * **Graph**: Thresholds for Right Y axis [#7107](https://github.com/grafana/grafana/issues/7107), thx [@ilgizar](https://github.com/ilgizar) * **Graph**: Support multiple series stacking in histogram mode [#8151](https://github.com/grafana/grafana/issues/8151), thx [@mtanda](https://github.com/mtanda) * **Alerting**: Pausing/un alerts now updates new_state_date [#10942](https://github.com/grafana/grafana/pull/10942) * **Alerting**: Support Pagerduty notification channel using Pagerduty V2 API [#10531](https://github.com/grafana/grafana/issues/10531), thx [@jbaublitz](https://github.com/jbaublitz) * **Templating**: Add comma templating format [#10632](https://github.com/grafana/grafana/issues/10632), thx [@mtanda](https://github.com/mtanda) +* **Prometheus**: Show template variable candidate in query editor [#9210](https://github.com/grafana/grafana/issues/9210), thx [@mtanda](https://github.com/mtanda) * **Prometheus**: Support POST for query and query_range [#9859](https://github.com/grafana/grafana/pull/9859), thx [@mtanda](https://github.com/mtanda) +* **Alerting**: Add support for retries on alert queries [#5855](https://github.com/grafana/grafana/issues/5855), thx [@Thib17](https://github.com/Thib17) +* **Table**: Table plugin value mappings [#7119](https://github.com/grafana/grafana/issues/7119), thx [infernix](https://github.com/infernix) +* **IE11**: IE 11 compatibility [#11165](https://github.com/grafana/grafana/issues/11165) +* **Scrolling**: Better scrolling experience [#11053](https://github.com/grafana/grafana/issues/11053), [#11252](https://github.com/grafana/grafana/issues/11252), [#10836](https://github.com/grafana/grafana/issues/10836), [#11185](https://github.com/grafana/grafana/issues/11185), [#11168](https://github.com/grafana/grafana/issues/11168) +* **Docker**: Improved docker image (breaking changes regarding file ownership) [grafana-docker #141](https://github.com/grafana/grafana-docker/issues/141), thx [@Spindel](https://github.com/Spindel), [@ChristianKniep](https://github.com/ChristianKniep), [@brancz](https://github.com/brancz) and [@jangaraj](https://github.com/jangaraj) +* **Folders**: A folder admin cannot add user/team permissions for folder/its dashboards [#11173](https://github.com/grafana/grafana/issues/11173) +* **Provisioning**: Improved workflow for provisioned dashboards [#10883](https://github.com/grafana/grafana/issues/10883) ### Minor + * **OpsGenie**: Add triggered alerts as description [#11046](https://github.com/grafana/grafana/pull/11046), thx [@llamashoes](https://github.com/llamashoes) * **Cloudwatch**: Support high resolution metrics [#10925](https://github.com/grafana/grafana/pull/10925), thx [@mtanda](https://github.com/mtanda) * **Cloudwatch**: Add dimension filtering to CloudWatch `dimension_values()` [#10029](https://github.com/grafana/grafana/issues/10029), thx [@willyhutw](https://github.com/willyhutw) * **Units**: Second to HH:mm:ss formatter [#11107](https://github.com/grafana/grafana/issues/11107), thx [@gladdiologist](https://github.com/gladdiologist) * **Singlestat**: Add color to prefix and postfix in singlestat panel [#11143](https://github.com/grafana/grafana/pull/11143), thx [@ApsOps](https://github.com/ApsOps) +* **Dashboards**: Version cleanup fails on old databases with many entries [#11278](https://github.com/grafana/grafana/issues/11278) +* **Server**: Adjust permissions of unix socket [#11343](https://github.com/grafana/grafana/pull/11343), thx [@corny](https://github.com/corny) +* **Shortcuts**: Add shortcut for duplicate panel [#11102](https://github.com/grafana/grafana/issues/11102) +* **AuthProxy**: Support IPv6 in Auth proxy white list [#11330](https://github.com/grafana/grafana/pull/11330), thx [@corny](https://github.com/corny) +* **SMTP**: Don't connect to STMP server using TLS unless configured. [#7189](https://github.com/grafana/grafana/issues/7189) +* **Prometheus**: Escape backslash in labels correctly. [#10555](https://github.com/grafana/grafana/issues/10555), thx [@roidelapluie](https://github.com/roidelapluie) +* **Variables**: Case-insensitive sorting for template values [#11128](https://github.com/grafana/grafana/issues/11128) thx [@cross](https://github.com/cross) +* **Annotations (native)**: Change default limit from 10 to 100 when querying api [#11569](https://github.com/grafana/grafana/issues/11569), thx [@flopp999](https://github.com/flopp999) +* **MySQL/Postgres/MSSQL**: PostgreSQL datasource generates invalid query with dates before 1970 [#11530](https://github.com/grafana/grafana/issues/11530) thx [@ryantxu](https://github.com/ryantxu) +* **Kiosk**: Adds url parameter for starting a dashboard in inactive mode [#11228](https://github.com/grafana/grafana/issues/11228), thx [@towolf](https://github.com/towolf) +* **Dashboard**: Enable closing timepicker using escape key [#11332](https://github.com/grafana/grafana/issues/11332) +* **Datasources**: Rename direct access mode in the data source settings [#11391](https://github.com/grafana/grafana/issues/11391) +* **Search**: Display dashboards in folder indented [#11073](https://github.com/grafana/grafana/issues/11073) +* **Units**: Use B/s instead Bps for Bytes per second [#9342](https://github.com/grafana/grafana/pull/9342), thx [@mayli](https://github.com/mayli) +* **Units**: Radiation units [#11001](https://github.com/grafana/grafana/issues/11001), thx [@victorclaessen](https://github.com/victorclaessen) +* **Units**: Timeticks unit [#11183](https://github.com/grafana/grafana/pull/11183), thx [@jtyr](https://github.com/jtyr) +* **Units**: Concentration units and "Normal cubic metre" [#11211](https://github.com/grafana/grafana/issues/11211), thx [@flopp999](https://github.com/flopp999) +* **Units**: New currency - Czech koruna [#11384](https://github.com/grafana/grafana/pull/11384), thx [@Rohlik](https://github.com/Rohlik) +* **Avatar**: Fix DISABLE_GRAVATAR option [#11095](https://github.com/grafana/grafana/issues/11095) +* **Heatmap**: Disable log scale when using time time series buckets [#10792](https://github.com/grafana/grafana/issues/10792) +* **Provisioning**: Remove `id` from json when provisioning dashboards, [#11138](https://github.com/grafana/grafana/issues/11138) +* **Prometheus**: tooltip for legend format not showing properly [#11516](https://github.com/grafana/grafana/issues/11516), thx [@svenklemm](https://github.com/svenklemm) +* **Playlist**: Empty playlists cannot be deleted [#11133](https://github.com/grafana/grafana/issues/11133), thx [@kichristensen](https://github.com/kichristensen) +* **Switch Orgs**: Alphabetic order in Switch Organization modal [#11556](https://github.com/grafana/grafana/issues/11556) +* **Postgres**: improve `$__timeFilter` macro [#11578](https://github.com/grafana/grafana/issues/11578), thx [@svenklemm](https://github.com/svenklemm) +* **Permission list**: Improved ux [#10747](https://github.com/grafana/grafana/issues/10747) +* **Dashboard**: Sizing and positioning of settings menu icons [#11572](https://github.com/grafana/grafana/pull/11572) +* **Dashboard**: Add search filter/tabs to new panel control [#10427](https://github.com/grafana/grafana/issues/10427) +* **Folders**: User with org viewer role should not be able to save/move dashboards in/to general folder [#11553](https://github.com/grafana/grafana/issues/11553) +* **Influxdb**: Don't assume the first column in table response is time. [#11476](https://github.com/grafana/grafana/issues/11476), thx [@hahnjo](https://github.com/hahnjo) -# 5.0.4 (unreleased) -* **Dashboard** Fixed bug where collapsed panels could not be directly linked to/renderer [#11114](https://github.com/grafana/grafana/issues/11114) & [#11086](https://github.com/grafana/grafana/issues/11086) +### Tech +* Backend code simplification [#11613](https://github.com/grafana/grafana/pull/11613), thx [@knweiss](https://github.com/knweiss) +* Add codespell to CI [#11602](https://github.com/grafana/grafana/pull/11602), thx [@mjtrangoni](https://github.com/mjtrangoni) +* Migrated JavaScript files to TypeScript + +# 5.0.4 (2018-03-28) + +* **Docker** Can't start Grafana on Kubernetes 1.7.14, 1.8.9, or 1.9.4 [#140 in grafana-docker repo](https://github.com/grafana/grafana-docker/issues/140) thx [@suquant](https://github.com/suquant) +* **Dashboard** Fixed bug where collapsed panels could not be directly linked to/renderer [#11114](https://github.com/grafana/grafana/issues/11114) & [#11086](https://github.com/grafana/grafana/issues/11086) & [#11296](https://github.com/grafana/grafana/issues/11296) +* **Dashboard** Provisioning dashboard with alert rules should create alerts [#11247](https://github.com/grafana/grafana/issues/11247) +* **Snapshots** For snapshots, the Graph panel renders the legend incorrectly on right hand side [#11318](https://github.com/grafana/grafana/issues/11318) +* **Alerting** Link back to Grafana returns wrong URL if root_path contains sub-path components [#11403](https://github.com/grafana/grafana/issues/11403) +* **Alerting** Incorrect default value for upload images setting for alert notifiers [#11413](https://github.com/grafana/grafana/pull/11413) # 5.0.3 (2018-03-16) * **Mysql**: Mysql panic occurring occasionally upon Grafana dashboard access (a bigger patch than the one in 5.0.2) [#11155](https://github.com/grafana/grafana/issues/11155) @@ -37,7 +444,7 @@ * **Dashboards**: Changing templated value from dropdown is causing unsaved changes [#11063](https://github.com/grafana/grafana/issues/11063) * **Prometheus**: Fixes bundled Prometheus 2.0 dashboard [#11016](https://github.com/grafana/grafana/issues/11016), thx [@roidelapluie](https://github.com/roidelapluie) * **Sidemenu**: Profile menu "invisible" when gravatar is disabled [#11097](https://github.com/grafana/grafana/issues/11097) -* **Dashboard**: Fixes a bug with resizeable handles for panels [#11103](https://github.com/grafana/grafana/issues/11103) +* **Dashboard**: Fixes a bug with resizable handles for panels [#11103](https://github.com/grafana/grafana/issues/11103) * **Alerting**: Telegram inline image mode fails when caption too long [#10975](https://github.com/grafana/grafana/issues/10975) * **Alerting**: Fixes silent failing validation [#11145](https://github.com/grafana/grafana/pull/11145) * **OAuth**: Only use jwt token if it contains an email address [#11127](https://github.com/grafana/grafana/pull/11127) @@ -101,7 +508,7 @@ Grafana v5.0 is going to be the biggest and most foundational release Grafana ha ### New Major Features - **Dashboards** Dashboard folders, [#1611](https://github.com/grafana/grafana/issues/1611) - **Teams** User groups (teams) implemented. Can be used in folder & dashboard permission list. -- **Dashboard grid**: Panels are now layed out in a two dimensional grid (with x, y, w, h). [#9093](https://github.com/grafana/grafana/issues/9093). +- **Dashboard grid**: Panels are now laid out in a two dimensional grid (with x, y, w, h). [#9093](https://github.com/grafana/grafana/issues/9093). - **Templating**: Vertical repeat direction for panel repeats. - **UX**: Major update to page header and navigation - **Dashboard settings**: Combine dashboard settings views into one with side menu, [#9750](https://github.com/grafana/grafana/issues/9750) @@ -135,7 +542,7 @@ Dashboard panels and rows are positioned using a gridPos object `{x: 0, y: 0, w: * **Dashboard history**: New config file option versions_to_keep sets how many versions per dashboard to store, [#9671](https://github.com/grafana/grafana/issues/9671) * **Dashboard as cfg**: Load dashboards from file into Grafana on startup/change [#9654](https://github.com/grafana/grafana/issues/9654) [#5269](https://github.com/grafana/grafana/issues/5269) * **Prometheus**: Grafana can now send alerts to Prometheus Alertmanager while firing [#7481](https://github.com/grafana/grafana/issues/7481), thx [@Thib17](https://github.com/Thib17) and [@mtanda](https://github.com/mtanda) -* **Table**: Support multiple table formated queries in table panel [#9170](https://github.com/grafana/grafana/issues/9170), thx [@davkal](https://github.com/davkal) +* **Table**: Support multiple table formatted queries in table panel [#9170](https://github.com/grafana/grafana/issues/9170), thx [@davkal](https://github.com/davkal) * **Security**: Protect against brute force (frequent) login attempts [#7616](https://github.com/grafana/grafana/issues/7616) ## Minor @@ -157,7 +564,7 @@ Dashboard panels and rows are positioned using a gridPos object `{x: 0, y: 0, w: * **Sensu**: Send alert message to sensu output [#9551](https://github.com/grafana/grafana/issues/9551), thx [@cjchand](https://github.com/cjchand) * **Singlestat**: suppress error when result contains no datapoints [#9636](https://github.com/grafana/grafana/issues/9636), thx [@utkarshcmu](https://github.com/utkarshcmu) * **Postgres/MySQL**: Control quoting in SQL-queries when using template variables [#9030](https://github.com/grafana/grafana/issues/9030), thanks [@svenklemm](https://github.com/svenklemm) -* **Pagerduty**: Pagerduty dont auto resolve incidents by default anymore. [#10222](https://github.com/grafana/grafana/issues/10222) +* **Pagerduty**: Pagerduty don't auto resolve incidents by default anymore. [#10222](https://github.com/grafana/grafana/issues/10222) * **Cloudwatch**: Fix for multi-valued templated queries. [#9903](https://github.com/grafana/grafana/issues/9903) ## Tech @@ -174,6 +581,12 @@ The following properties have been deprecated and will be removed in a future re - `uri` property in `GET /api/search` -> Use new `url` or `uid` property instead - `meta.slug` property in `GET /api/dashboards/uid/:uid` and `GET /api/dashboards/db/:slug` -> Use new `meta.url` or `dashboard.uid` property instead +# 4.6.4 (2018-08-29) + +### Important fix for LDAP & OAuth login vulnerability + +See [security announcement](https://community.grafana.com/t/grafana-5-2-3-and-4-6-4-security-update/10050) for details. + # 4.6.3 (2017-12-14) ## Fixes @@ -185,7 +598,7 @@ The following properties have been deprecated and will be removed in a future re # 4.6.2 (2017-11-16) ## Important -* **Prometheus**: Fixes bug with new prometheus alerts in Grafana. Make sure to download this version if your using Prometheus for alerting. More details in the issue. [#9777](https://github.com/grafana/grafana/issues/9777) +* **Prometheus**: Fixes bug with new prometheus alerts in Grafana. Make sure to download this version if you're using Prometheus for alerting. More details in the issue. [#9777](https://github.com/grafana/grafana/issues/9777) ## Fixes * **Color picker**: Bug after using textbox input field to change/paste color string [#9769](https://github.com/grafana/grafana/issues/9769) @@ -235,7 +648,7 @@ The following properties have been deprecated and will be removed in a future re * **Annotations**: Add support for creating annotations from graph panel [#8197](https://github.com/grafana/grafana/pull/8197) * **GCS**: Adds support for Google Cloud Storage [#8370](https://github.com/grafana/grafana/issues/8370) thx [@chuhlomin](https://github.com/chuhlomin) * **Prometheus**: Adds /metrics endpoint for exposing Grafana metrics. [#9187](https://github.com/grafana/grafana/pull/9187) -* **Graph**: Add support for local formating in axis. [#1395](https://github.com/grafana/grafana/issues/1395), thx [@m0nhawk](https://github.com/m0nhawk) +* **Graph**: Add support for local formatting in axis. [#1395](https://github.com/grafana/grafana/issues/1395), thx [@m0nhawk](https://github.com/m0nhawk) * **Jaeger**: Add support for open tracing using jaeger in Grafana. [#9213](https://github.com/grafana/grafana/pull/9213) * **Unit types**: New date & time unit types added, useful in singlestat to show dates & times. [#3678](https://github.com/grafana/grafana/issues/3678), [#6710](https://github.com/grafana/grafana/issues/6710), [#2764](https://github.com/grafana/grafana/issues/2764) * **CLI**: Make it possible to install plugins from any url [#5873](https://github.com/grafana/grafana/issues/5873) @@ -272,7 +685,7 @@ The following properties have been deprecated and will be removed in a future re * **Graphite**: Fix for Grafana internal metrics to Graphite sending NaN values [#9279](https://github.com/grafana/grafana/issues/9279) * **HTTP API**: Fix for HEAD method requests [#9307](https://github.com/grafana/grafana/issues/9307) * **Templating**: Fix for duplicate template variable queries when refresh is set to time range change [#9185](https://github.com/grafana/grafana/issues/9185) -* **Metrics**: dont write NaN values to graphite [#9279](https://github.com/grafana/grafana/issues/9279) +* **Metrics**: don't write NaN values to graphite [#9279](https://github.com/grafana/grafana/issues/9279) # 4.5.1 (2017-09-15) @@ -309,12 +722,12 @@ The following properties have been deprecated and will be removed in a future re ### Breaking change * **InfluxDB/Elasticsearch**: The panel & data source option named "Group by time interval" is now named "Min time interval" and does now always define a lower limit for the auto group by time. Without having to use `>` prefix (that prefix still works). This should in theory have close to zero actual impact on existing dashboards. It does mean that if you used this setting to define a hard group by time interval of, say "1d", if you zoomed to a time range wide enough the time range could increase above the "1d" range as the setting is now always considered a lower limit. -* **Elasticsearch**: Elasticsearch metric queries without date histogram now return table formated data making table panel much easier to use for this use case. Should not break/change existing dashboards with stock panels but external panel plugins can be affected. +* **Elasticsearch**: Elasticsearch metric queries without date histogram now return table formatted data making table panel much easier to use for this use case. Should not break/change existing dashboards with stock panels but external panel plugins can be affected. ## Changes * **InfluxDB**: Change time range filter for absolute time ranges to be inclusive instead of exclusive [#8319](https://github.com/grafana/grafana/issues/8319), thx [@Oxydros](https://github.com/Oxydros) -* **InfluxDB**: Added paranthesis around tag filters in queries [#9131](https://github.com/grafana/grafana/pull/9131) +* **InfluxDB**: Added parenthesis around tag filters in queries [#9131](https://github.com/grafana/grafana/pull/9131) ## Bug Fixes @@ -326,7 +739,7 @@ The following properties have been deprecated and will be removed in a future re ## Bug Fixes -* **Search**: Fix for issue that casued search view to hide when you clicked starred or tags filters, fixes [#8981](https://github.com/grafana/grafana/issues/8981) +* **Search**: Fix for issue that caused search view to hide when you clicked starred or tags filters, fixes [#8981](https://github.com/grafana/grafana/issues/8981) * **Modals**: ESC key now closes modal again, fixes [#8981](https://github.com/grafana/grafana/issues/8988), thx [@j-white](https://github.com/j-white) # 4.4.2 (2017-08-01) @@ -665,12 +1078,12 @@ due to too many connections/file handles on the data source backend. This proble ### Enhancements * **Login**: Adds option to disable username/password logins, closes [#4674](https://github.com/grafana/grafana/issues/4674) * **SingleStat**: Add seriename as option in singlestat panel, closes [#4740](https://github.com/grafana/grafana/issues/4740) -* **Localization**: Week start day now dependant on browser locale setting, closes [#3003](https://github.com/grafana/grafana/issues/3003) +* **Localization**: Week start day now dependent on browser locale setting, closes [#3003](https://github.com/grafana/grafana/issues/3003) * **Templating**: Update panel repeats for variables that change on time refresh, closes [#5021](https://github.com/grafana/grafana/issues/5021) * **Templating**: Add support for numeric and alphabetical sorting of variable values, closes [#2839](https://github.com/grafana/grafana/issues/2839) * **Elasticsearch**: Support to set Precision Threshold for Unique Count metric, closes [#4689](https://github.com/grafana/grafana/issues/4689) * **Navigation**: Add search to org swithcer, closes [#2609](https://github.com/grafana/grafana/issues/2609) -* **Database**: Allow database config using one propertie, closes [#5456](https://github.com/grafana/grafana/pull/5456) +* **Database**: Allow database config using one property, closes [#5456](https://github.com/grafana/grafana/pull/5456) * **Graphite**: Add support for groupByNodes, closes [#5613](https://github.com/grafana/grafana/pull/5613) * **Influxdb**: Add support for elapsed(), closes [#5827](https://github.com/grafana/grafana/pull/5827) * **OpenTSDB**: Add support for explicitTags for OpenTSDB>=2.3, closes [#6360](https://github.com/grafana/grafana/pull/6361) @@ -737,7 +1150,7 @@ due to too many connections/file handles on the data source backend. This proble * **Datasource**: Pending data source requests are cancelled before new ones are issues (Graphite & Prometheus), closes [#5321](https://github.com/grafana/grafana/issues/5321) ### Breaking changes -* **Logging** : Changed default logging output format (now structured into message, and key value pairs, with logger key acting as component). You can also no change in config to json log ouput. +* **Logging** : Changed default logging output format (now structured into message, and key value pairs, with logger key acting as component). You can also no change in config to json log output. * **Graphite** : The Graph panel no longer have a Graphite PNG option. closes [#5367](https://github.com/grafana/grafana/issues/5367) ### Bug fixes @@ -755,7 +1168,7 @@ due to too many connections/file handles on the data source backend. This proble * **Annotations**: Annotations can now use a template variable as data source, closes [#5054](https://github.com/grafana/grafana/issues/5054) * **Time picker**: Fixed issue timepicker and UTC when reading time from URL, fixes [#5078](https://github.com/grafana/grafana/issues/5078) * **CloudWatch**: Support for Multiple Account by AssumeRole, closes [#3522](https://github.com/grafana/grafana/issues/3522) -* **Singlestat**: Fixed alignment and minium height issue, fixes [#5113](https://github.com/grafana/grafana/issues/5113), fixes [#4679](https://github.com/grafana/grafana/issues/4679) +* **Singlestat**: Fixed alignment and minimum height issue, fixes [#5113](https://github.com/grafana/grafana/issues/5113), fixes [#4679](https://github.com/grafana/grafana/issues/4679) * **Share modal**: Fixed link when using grafana under dashboard sub url, fixes [#5109](https://github.com/grafana/grafana/issues/5109) * **Prometheus**: Fixed bug in query editor that caused it not to load when reloading page, fixes [#5107](https://github.com/grafana/grafana/issues/5107) * **Elasticsearch**: Fixed bug when template variable query returns numeric values, fixes [#5097](https://github.com/grafana/grafana/issues/5097), fixes [#5088](https://github.com/grafana/grafana/issues/5088) @@ -772,7 +1185,7 @@ due to too many connections/file handles on the data source backend. This proble * **Graph**: Fixed broken PNG rendering in graph panel, fixes [#5025](https://github.com/grafana/grafana/issues/5025) * **Graph**: Fixed broken xaxis on graph panel, fixes [#5024](https://github.com/grafana/grafana/issues/5024) -* **Influxdb**: Fixes crash when hiding middle serie, fixes [#5005](https://github.com/grafana/grafana/issues/5005) +* **Influxdb**: Fixes crash when hiding middle series, fixes [#5005](https://github.com/grafana/grafana/issues/5005) # 3.0.1 Stable (2016-05-11) @@ -784,7 +1197,7 @@ due to too many connections/file handles on the data source backend. This proble ### Bug fixes * **Dashboard title**: Fixed max dashboard title width (media query) for large screens, fixes [#4859](https://github.com/grafana/grafana/issues/4859) * **Annotations**: Fixed issue with entering annotation edit view, fixes [#4857](https://github.com/grafana/grafana/issues/4857) -* **Remove query**: Fixed issue with removing query for data sources without collapsable query editors, fixes [#4856](https://github.com/grafana/grafana/issues/4856) +* **Remove query**: Fixed issue with removing query for data sources without collapsible query editors, fixes [#4856](https://github.com/grafana/grafana/issues/4856) * **Graphite PNG**: Fixed issue graphite png rendering option, fixes [#4864](https://github.com/grafana/grafana/issues/4864) * **InfluxDB**: Fixed issue missing plus group by iconn, fixes [#4862](https://github.com/grafana/grafana/issues/4862) * **Graph**: Fixes missing line mode for thresholds, fixes [#4902](https://github.com/grafana/grafana/pull/4902) @@ -800,11 +1213,11 @@ due to too many connections/file handles on the data source backend. This proble ### Bug fixes * **InfluxDB 0.12**: Fixed issue templating and `show tag values` query only returning tags for first measurement, fixes [#4726](https://github.com/grafana/grafana/issues/4726) -* **Templating**: Fixed issue with regex formating when matching multiple values, fixes [#4755](https://github.com/grafana/grafana/issues/4755) +* **Templating**: Fixed issue with regex formatting when matching multiple values, fixes [#4755](https://github.com/grafana/grafana/issues/4755) * **Templating**: Fixed issue with custom all value and escaping, fixes [#4736](https://github.com/grafana/grafana/issues/4736) * **Dashlist**: Fixed issue dashboard list panel and caching tags, fixes [#4768](https://github.com/grafana/grafana/issues/4768) * **Graph**: Fixed issue with unneeded scrollbar in legend for Firefox, fixes [#4760](https://github.com/grafana/grafana/issues/4760) -* **Table panel**: Fixed issue table panel formating string array properties, fixes [#4791](https://github.com/grafana/grafana/issues/4791) +* **Table panel**: Fixed issue table panel formatting string array properties, fixes [#4791](https://github.com/grafana/grafana/issues/4791) * **grafana-cli**: Improve error message when failing to install plugins due to corrupt response, fixes [#4651](https://github.com/grafana/grafana/issues/4651) * **Singlestat**: Fixes prefix an postfix for gauges, fixes [#4812](https://github.com/grafana/grafana/issues/4812) * **Singlestat**: Fixes auto-refresh on change for some options, fixes [#4809](https://github.com/grafana/grafana/issues/4809) @@ -896,7 +1309,7 @@ slack channel (link to slack channel in readme). ### Bug fixes * **Playlist**: Fix for memory leak when running a playlist, closes [#3794](https://github.com/grafana/grafana/pull/3794) * **InfluxDB**: Fix for InfluxDB and table panel when using Format As Table and having group by time, fixes [#3928](https://github.com/grafana/grafana/issues/3928) -* **Panel Time shift**: Fix for panel time range and using dashboard times liek `Today` and `This Week`, fixes [#3941](https://github.com/grafana/grafana/issues/3941) +* **Panel Time shift**: Fix for panel time range and using dashboard times like `Today` and `This Week`, fixes [#3941](https://github.com/grafana/grafana/issues/3941) * **Row repeat**: Repeated rows will now appear next to each other and not by the bottom of the dashboard, fixes [#3942](https://github.com/grafana/grafana/issues/3942) * **Png renderer**: Fix for phantomjs path on windows, fixes [#3657](https://github.com/grafana/grafana/issues/3657) @@ -920,7 +1333,7 @@ slack channel (link to slack channel in readme). ### Bug Fixes * **metric editors**: Fix for clicking typeahead auto dropdown option, fixes [#3428](https://github.com/grafana/grafana/issues/3428) * **influxdb**: Fixed issue showing Group By label only on first query, fixes [#3453](https://github.com/grafana/grafana/issues/3453) -* **logging**: Add more verbose info logging for http reqeusts, closes [#3405](https://github.com/grafana/grafana/pull/3405) +* **logging**: Add more verbose info logging for http requests, closes [#3405](https://github.com/grafana/grafana/pull/3405) # 2.6.0-Beta1 (2015-12-04) @@ -947,7 +1360,7 @@ slack channel (link to slack channel in readme). **New Feature: Mix data sources** - A built in data source is now available named `-- Mixed --`, When picked in the metrics tab, -it allows you to add queries of differnet data source types & instances to the same graph/panel! +it allows you to add queries of different data source types & instances to the same graph/panel! [Issue #436](https://github.com/grafana/grafana/issues/436) **New Feature: Elasticsearch Metrics Query Editor and Viz Support** @@ -986,7 +1399,7 @@ it allows you to add queries of differnet data source types & instances to the s - [Issue #2564](https://github.com/grafana/grafana/issues/2564). Templating: Another atempt at fixing #2534 (Init multi value template var used in repeat panel from url) - [Issue #2620](https://github.com/grafana/grafana/issues/2620). Graph: multi series tooltip did no highlight correct point when stacking was enabled and series were of different resolution - [Issue #2636](https://github.com/grafana/grafana/issues/2636). InfluxDB: Do no show template vars in dropdown for tag keys and group by keys -- [Issue #2604](https://github.com/grafana/grafana/issues/2604). InfluxDB: More alias options, can now use `$[0-9]` syntax to reference part of a measurement name (seperated by dots) +- [Issue #2604](https://github.com/grafana/grafana/issues/2604). InfluxDB: More alias options, can now use `$[0-9]` syntax to reference part of a measurement name (separated by dots) **Breaking Changes** - Notice to makers/users of custom data sources, there is a minor breaking change in 2.2 that @@ -1068,7 +1481,7 @@ Grunt & Watch tasks: - [Issue #1826](https://github.com/grafana/grafana/issues/1826). User role 'Viewer' are now prohibited from entering edit mode (and doing other transient dashboard edits). A new role `Read Only Editor` will replace the old Viewer behavior - [Issue #1928](https://github.com/grafana/grafana/issues/1928). HTTP API: GET /api/dashboards/db/:slug response changed property `model` to `dashboard` to match the POST request nameing - Backend render URL changed from `/render/dashboard/solo` `render/dashboard-solo/` (in order to have consistent dashboard url `/dashboard/:type/:slug`) -- Search HTTP API response has changed (simplified), tags list moved to seperate HTTP resource URI +- Search HTTP API response has changed (simplified), tags list moved to separate HTTP resource URI - Datasource HTTP api breaking change, ADD datasource is now POST /api/datasources/, update is now PUT /api/datasources/:id **Fixes** @@ -1085,7 +1498,7 @@ Grunt & Watch tasks: # 2.0.2 (2015-04-22) **Fixes** -- [Issue #1832](https://github.com/grafana/grafana/issues/1832). Graph Panel + Legend Table mode: Many series casued zero height graph, now legend will never reduce the height of the graph below 50% of row height. +- [Issue #1832](https://github.com/grafana/grafana/issues/1832). Graph Panel + Legend Table mode: Many series caused zero height graph, now legend will never reduce the height of the graph below 50% of row height. - [Issue #1846](https://github.com/grafana/grafana/issues/1846). Snapshots: Fixed issue with snapshoting dashboards with an interval template variable - [Issue #1848](https://github.com/grafana/grafana/issues/1848). Panel timeshift: You can now use panel timeshift without a relative time override @@ -1127,7 +1540,7 @@ Grunt & Watch tasks: **Fixes** - [Issue #1649](https://github.com/grafana/grafana/issues/1649). HTTP API: grafana /render calls nows with api keys -- [Issue #1667](https://github.com/grafana/grafana/issues/1667). Datasource proxy & session timeout fix (casued 401 Unauthorized error after a while) +- [Issue #1667](https://github.com/grafana/grafana/issues/1667). Datasource proxy & session timeout fix (caused 401 Unauthorized error after a while) - [Issue #1707](https://github.com/grafana/grafana/issues/1707). Unsaved changes: Do not show for snapshots, scripted and file based dashboards - [Issue #1703](https://github.com/grafana/grafana/issues/1703). Unsaved changes: Do not show for users with role `Viewer` - [Issue #1675](https://github.com/grafana/grafana/issues/1675). Data source proxy: Fixed issue with Gzip enabled and data source proxy @@ -1140,14 +1553,14 @@ Grunt & Watch tasks: **Important Note** -Grafana 2.x is fundamentally different from 1.x; it now ships with an integrated backend server. Please read the [Documentation](http://docs.grafana.org) for more detailed about this SIGNIFCANT change to Grafana +Grafana 2.x is fundamentally different from 1.x; it now ships with an integrated backend server. Please read the [Documentation](http://docs.grafana.org) for more detailed about this SIGNIFICANT change to Grafana **New features** - [Issue #1623](https://github.com/grafana/grafana/issues/1623). Share Dashboard: Dashboard snapshot sharing (dash and data snapshot), save to local or save to public snapshot dashboard snapshots.raintank.io site -- [Issue #1622](https://github.com/grafana/grafana/issues/1622). Share Panel: The share modal now has an embed option, gives you an iframe that you can use to embedd a single graph on another web site -- [Issue #718](https://github.com/grafana/grafana/issues/718). Dashboard: When saving a dashboard and another user has made changes inbetween the user is promted with a warning if he really wants to overwrite the other's changes +- [Issue #1622](https://github.com/grafana/grafana/issues/1622). Share Panel: The share modal now has an embed option, gives you an iframe that you can use to embed a single graph on another web site +- [Issue #718](https://github.com/grafana/grafana/issues/718). Dashboard: When saving a dashboard and another user has made changes in between the user is prompted with a warning if he really wants to overwrite the other's changes - [Issue #1331](https://github.com/grafana/grafana/issues/1331). Graph & Singlestat: New axis/unit format selector and more units (kbytes, Joule, Watt, eV), and new design for graph axis & grid tab and single stat options tab views -- [Issue #1241](https://github.com/grafana/grafana/issues/1242). Timepicker: New option in timepicker (under dashboard settings), to change ``now`` to be for example ``now-1m``, usefull when you want to ignore last minute because it contains incomplete data +- [Issue #1241](https://github.com/grafana/grafana/issues/1242). Timepicker: New option in timepicker (under dashboard settings), to change ``now`` to be for example ``now-1m``, useful when you want to ignore last minute because it contains incomplete data - [Issue #171](https://github.com/grafana/grafana/issues/171). Panel: Different time periods, panels can override dashboard relative time and/or add a time shift - [Issue #1488](https://github.com/grafana/grafana/issues/1488). Dashboard: Clone dashboard / Save as - [Issue #1458](https://github.com/grafana/grafana/issues/1458). User: persisted user option for dark or light theme (no longer an option on a dashboard) @@ -1178,7 +1591,7 @@ Grafana 2.x is fundamentally different from 1.x; it now ships with an integrated **OpenTSDB breaking change** - [Issue #1438](https://github.com/grafana/grafana/issues/1438). OpenTSDB: Automatic downsample interval passed to OpenTSDB (depends on timespan and graph width) -- NOTICE, Downsampling is now enabled by default, so if you have not picked a downsample aggregator in your metric query do so or your graphs will be missleading +- NOTICE, Downsampling is now enabled by default, so if you have not picked a downsample aggregator in your metric query do so or your graphs will be misleading - This will make Grafana a lot quicker for OpenTSDB users when viewing large time spans without having to change the downsample interval manually. **Tech** @@ -1209,7 +1622,7 @@ Grafana 2.x is fundamentally different from 1.x; it now ships with an integrated - [Issue #1114](https://github.com/grafana/grafana/issues/1114). Graphite: Lexer fix, allow equal sign (=) in metric paths - [Issue #1136](https://github.com/grafana/grafana/issues/1136). Graph: Fix to legend value Max and negative values - [Issue #1150](https://github.com/grafana/grafana/issues/1150). SinglestatPanel: Fixed absolute drilldown link issue -- [Issue #1123](https://github.com/grafana/grafana/issues/1123). Firefox: Workaround for Firefox bug, casued input text fields to not be selectable and not have placeable cursor +- [Issue #1123](https://github.com/grafana/grafana/issues/1123). Firefox: Workaround for Firefox bug, caused input text fields to not be selectable and not have placeable cursor - [Issue #1108](https://github.com/grafana/grafana/issues/1108). Graph: Fix for tooltip series order when series draw order was changed with zindex property # 1.9.0-rc1 (2014-11-17) @@ -1286,7 +1699,7 @@ Read this [blog post](https://grafana.com/blog/2014/09/11/grafana-1.8.0-rc1-rele - [Issue #234](https://github.com/grafana/grafana/issues/234). Templating: Interval variable type for time intervals summarize/group by parameter, included "auto" option, and auto step counts option. - [Issue #262](https://github.com/grafana/grafana/issues/262). Templating: Ability to use template variables for function parameters via custom variable type, can be used as parameter for movingAverage or scaleToSeconds for example - [Issue #312](https://github.com/grafana/grafana/issues/312). Templating: Can now use template variables in panel titles -- [Issue #613](https://github.com/grafana/grafana/issues/613). Templating: Full support for InfluxDB, filter by part of series names, extract series substrings, nested queries, multipe where clauses! +- [Issue #613](https://github.com/grafana/grafana/issues/613). Templating: Full support for InfluxDB, filter by part of series names, extract series substrings, nested queries, multiple where clauses! - Template variables can be initialized from url, with var-my_varname=value, breaking change, before it was just my_varname. - Templating and url state sync has some issues that are not solved for this release, see [Issue #772](https://github.com/grafana/grafana/issues/772) for more details. @@ -1375,7 +1788,7 @@ Read this [blog post](https://grafana.com/blog/2014/09/11/grafana-1.8.0-rc1-rele - [Issue #136](https://github.com/grafana/grafana/issues/136). Graph: New legend display option "Align as table" - [Issue #556](https://github.com/grafana/grafana/issues/556). Graph: New legend display option "Right side", will show legend to the right of the graph - [Issue #604](https://github.com/grafana/grafana/issues/604). Graph: New axis format, 'bps' (SI unit in steps of 1000) useful for network gear metics -- [Issue #626](https://github.com/grafana/grafana/issues/626). Graph: Downscale y axis to more precise unit, value of 0.1 for seconds format will be formated as 100 ms. Thanks @kamaradclimber +- [Issue #626](https://github.com/grafana/grafana/issues/626). Graph: Downscale y axis to more precise unit, value of 0.1 for seconds format will be formatted as 100 ms. Thanks @kamaradclimber - [Issue #618](https://github.com/grafana/grafana/issues/618). OpenTSDB: Series alias option to override metric name returned from opentsdb. Thanks @heldr **Documentation** @@ -1405,13 +1818,13 @@ Read this [blog post](https://grafana.com/blog/2014/09/11/grafana-1.8.0-rc1-rele - [Issue #522](https://github.com/grafana/grafana/issues/522). Series names and column name typeahead cache fix - [Issue #504](https://github.com/grafana/grafana/issues/504). Fixed influxdb issue with raw query that caused wrong value column detection - [Issue #526](https://github.com/grafana/grafana/issues/526). Default property that marks which datasource is default in config.js is now optional -- [Issue #342](https://github.com/grafana/grafana/issues/342). Auto-refresh caused 2 refreshes (and hence mulitple queries) each time (at least in firefox) +- [Issue #342](https://github.com/grafana/grafana/issues/342). Auto-refresh caused 2 refreshes (and hence multiple queries) each time (at least in firefox) # 1.6.0 (2014-06-16) #### New features or improvements - [Issue #427](https://github.com/grafana/grafana/issues/427). New Y-axis formater for metric values that represent seconds, Thanks @jippi -- [Issue #390](https://github.com/grafana/grafana/issues/390). Allow special characters in serie names (influxdb datasource), Thanks @majst01 +- [Issue #390](https://github.com/grafana/grafana/issues/390). Allow special characters in series names (influxdb datasource), Thanks @majst01 - [Issue #428](https://github.com/grafana/grafana/issues/428). Refactoring of filterSrv, Thanks @Tetha - [Issue #445](https://github.com/grafana/grafana/issues/445). New config for playlist feature. Set playlist_timespan to set default playlist interval, Thanks @rmca - [Issue #461](https://github.com/grafana/grafana/issues/461). New graphite function definition added isNonNull, Thanks @tmonk42 @@ -1432,13 +1845,13 @@ Read this [blog post](https://grafana.com/blog/2014/09/11/grafana-1.8.0-rc1-rele - [Issue #475](https://github.com/grafana/grafana/issues/475). Add panel icon and Row edit button is replaced by the Row edit menu - New graphs now have a default empty query - Add Row button now creates a row with default height of 250px (no longer opens dashboard settings modal) -- Clean up of config.sample.js, graphiteUrl removed (still works, but depricated, removed in future) +- Clean up of config.sample.js, graphiteUrl removed (still works, but deprecated, removed in future) Use datasources config instead. panel_names removed from config.js. Use plugins.panels to add custom panels - Graphite panel is now renamed graph (Existing dashboards will still work) #### Fixes - [Issue #126](https://github.com/grafana/grafana/issues/126). Graphite query lexer change, can now handle regex parameters for aliasSub function -- [Issue #447](https://github.com/grafana/grafana/issues/447). Filter option loading when having muliple nested filters now works better. Options are now reloaded correctly and there are no multiple renders/refresh inbetween. +- [Issue #447](https://github.com/grafana/grafana/issues/447). Filter option loading when having muliple nested filters now works better. Options are now reloaded correctly and there are no multiple renders/refresh in between. - [Issue #412](https://github.com/grafana/grafana/issues/412). After a filter option is changed and a nested template param is reloaded, if the current value exists after the options are reloaded the current selected value is kept. - [Issue #460](https://github.com/grafana/grafana/issues/460). Legend Current value did not display when value was zero - [Issue #328](https://github.com/grafana/grafana/issues/328). Fix to series toggling bug that caused annotations to be hidden when toggling/hiding series. @@ -1603,3 +2016,4 @@ Thanks to everyone who contributed fixes and provided feedback :+1: # 1.0.0 (2014-01-19) First public release + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..8b2ba090fe1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ + +# Contributing + +Grafana uses GitHub to manage contributions. +Contributions take the form of pull requests that will be reviewed by the core team. + +* If you are a new contributor see: [Steps to Contribute](#steps-to-contribute) + +* If you have a trivial fix or improvement, go ahead and create a pull request. + +* If you plan to do something more involved, discuss your idea on the respective [issue](https://github.com/grafana/grafana/issues) or create a [new issue](https://github.com/grafana/grafana/issues/new) if it does not exist. This will avoid unnecessary work and surely give you and us a good deal of inspiration. + + +## Steps to Contribute + +Should you wish to work on a GitHub issue, check first if it is not already assigned to someone. If it is free, you claim it by commenting on the issue that you want to work on it. This is to prevent duplicated efforts from contributors on the same issue. + +Please check the [`beginner friendly`](https://github.com/grafana/grafana/issues?q=is%3Aopen+is%3Aissue+label%3A%22beginner+friendly%22) label to find issues that are good for getting started. If you have questions about one of the issues, with or without the tag, please comment on them and one of the core team or the original poster will clarify it. + + + +## Setup + +Follow the setup guide in README.md + +### Rebuild frontend assets on source change +``` +yarn watch +``` + +### Rerun tests on source change +``` +yarn jest +``` + +### Run tests for backend assets before commit +``` +test -z "$(gofmt -s -l . | grep -v -E 'vendor/(github.com|golang.org|gopkg.in)' | tee /dev/stderr)" +``` + +### Run tests for frontend assets before commit +``` +yarn test +go test -v ./pkg/... +``` + + +## Pull Request Checklist + +* Branch from the master branch and, if needed, rebase to the current master branch before submitting your pull request. If it doesn't merge cleanly with master you may be asked to rebase your changes. + +* Commits should be as small as possible, while ensuring that each commit is correct independently (i.e., each commit should compile and pass tests). + +* If your patch is not getting reviewed or you need a specific person to review it, you can @-reply a reviewer asking for a review in the pull request or a comment. + +* Add tests relevant to the fixed bug or new feature. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..28dd71952af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,82 @@ +# Golang build container +FROM golang:1.11 + +WORKDIR $GOPATH/src/github.com/grafana/grafana + +COPY Gopkg.toml Gopkg.lock ./ +COPY vendor vendor + +ARG DEP_ENSURE="" +RUN if [ ! -z "${DEP_ENSURE}" ]; then \ + go get -u github.com/golang/dep/cmd/dep && \ + dep ensure --vendor-only; \ + fi + +COPY pkg pkg +COPY build.go build.go +COPY package.json package.json + +RUN go run build.go build + +# Node build container +FROM node:8 + +WORKDIR /usr/src/app/ + +COPY package.json yarn.lock ./ +RUN yarn install --pure-lockfile --no-progress + +COPY Gruntfile.js tsconfig.json tslint.json ./ +COPY public public +COPY scripts scripts +COPY emails emails + +ENV NODE_ENV production +RUN ./node_modules/.bin/grunt build + +# Final container +FROM debian:stretch-slim + +ARG GF_UID="472" +ARG GF_GID="472" + +ENV PATH=/usr/share/grafana/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + GF_PATHS_CONFIG="/etc/grafana/grafana.ini" \ + GF_PATHS_DATA="/var/lib/grafana" \ + GF_PATHS_HOME="/usr/share/grafana" \ + GF_PATHS_LOGS="/var/log/grafana" \ + GF_PATHS_PLUGINS="/var/lib/grafana/plugins" \ + GF_PATHS_PROVISIONING="/etc/grafana/provisioning" + +WORKDIR $GF_PATHS_HOME + +RUN apt-get update && apt-get install -qq -y libfontconfig ca-certificates && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +COPY conf ./conf + +RUN mkdir -p "$GF_PATHS_HOME/.aws" && \ + groupadd -r -g $GF_GID grafana && \ + useradd -r -u $GF_UID -g grafana grafana && \ + mkdir -p "$GF_PATHS_PROVISIONING/datasources" \ + "$GF_PATHS_PROVISIONING/dashboards" \ + "$GF_PATHS_LOGS" \ + "$GF_PATHS_PLUGINS" \ + "$GF_PATHS_DATA" && \ + cp "$GF_PATHS_HOME/conf/sample.ini" "$GF_PATHS_CONFIG" && \ + cp "$GF_PATHS_HOME/conf/ldap.toml" /etc/grafana/ldap.toml && \ + chown -R grafana:grafana "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS" && \ + chmod 777 "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS" + +COPY --from=0 /go/src/github.com/grafana/grafana/bin/linux-amd64/grafana-server /go/src/github.com/grafana/grafana/bin/linux-amd64/grafana-cli ./bin/ +COPY --from=1 /usr/src/app/public ./public +COPY --from=1 /usr/src/app/tools ./tools +COPY tools/phantomjs/render.js ./tools/phantomjs/render.js + +EXPOSE 3000 + +COPY ./packaging/docker/run.sh /run.sh + +USER grafana +ENTRYPOINT [ "/run.sh" ] diff --git a/Gopkg.lock b/Gopkg.lock index ebadad8331b..1e06d4cbc5d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,30 +2,39 @@ [[projects]] + digest = "1:f8ad8a53fa865a70efbe215b0ca34735523f50ea39e0efde319ab6fc80089b44" name = "cloud.google.com/go" packages = ["compute/metadata"] - revision = "767c40d6a2e058483c25fa193e963a22da17236d" - version = "v0.18.0" + pruneopts = "NUT" + revision = "056a55f54a6cc77b440b31a56a5e7c3982d32811" + version = "v0.22.0" [[projects]] + digest = "1:167b6f65a6656de568092189ae791253939f076df60231fdd64588ac703892a1" name = "github.com/BurntSushi/toml" packages = ["."] + pruneopts = "NUT" revision = "b26d9c308763d68093482582cea63d69be07a0f0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:7d23e6e1889b8bb4bbb37a564708fdab4497ce232c3a99d66406c975b642a6ff" name = "github.com/Unknwon/com" packages = ["."] + pruneopts = "NUT" revision = "7677a1d7c1137cd3dd5ba7a076d0c898a1ef4520" [[projects]] - name = "github.com/apache/thrift" - packages = ["lib/go/thrift"] - revision = "b2a4d4ae21c789b689dd162deb819665567f481c" - version = "0.10.0" + branch = "master" + digest = "1:1610787cd9726e29d8fecc2a80e43e4fced008a1f560fec6688fc4d946f17835" + name = "github.com/VividCortex/mysqlerr" + packages = ["."] + pruneopts = "NUT" + revision = "6c6b55f8796f578c870b7e19bafb16103bc40095" [[projects]] + digest = "1:58294d68772aab5a8941b7d5d228eff7cccf63f895e914bc9bc38fda80471ea5" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -38,15 +47,20 @@ "aws/credentials/ec2rolecreds", "aws/credentials/endpointcreds", "aws/credentials/stscreds", + "aws/csm", "aws/defaults", "aws/ec2metadata", "aws/endpoints", "aws/request", "aws/session", "aws/signer/v4", + "internal/sdkio", + "internal/sdkrand", "internal/shareddefaults", "private/protocol", "private/protocol/ec2query", + "private/protocol/eventstream", + "private/protocol/eventstream/eventstreamapi", "private/protocol/query", "private/protocol/query/queryutil", "private/protocol/rest", @@ -56,377 +70,523 @@ "service/ec2", "service/ec2/ec2iface", "service/s3", - "service/sts" + "service/sts", ] - revision = "decd990ddc5dcdf2f73309cbcab90d06b996ca28" - version = "v1.12.67" + pruneopts = "NUT" + revision = "fde4ded7becdeae4d26bf1212916aabba79349b4" + version = "v1.14.12" [[projects]] branch = "master" + digest = "1:79cad073c7be02632d3fa52f62486848b089f560db1e94536de83a408c0f4726" name = "github.com/benbjohnson/clock" packages = ["."] + pruneopts = "NUT" revision = "7dc76406b6d3c05b5f71a86293cbcf3c4ea03b19" [[projects]] branch = "master" + digest = "1:707ebe952a8b3d00b343c01536c79c73771d100f63ec6babeaed5c79e2b8a8dd" name = "github.com/beorn7/perks" packages = ["quantile"] - revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9" + pruneopts = "NUT" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] branch = "master" + digest = "1:433a2ff0ef4e2f8634614aab3174783c5ff80120b487712db96cc3712f409583" name = "github.com/bmizerany/assert" packages = ["."] + pruneopts = "NUT" revision = "b7ed37b82869576c289d7d97fb2bbd8b64a0cb28" [[projects]] branch = "master" + digest = "1:d8f9145c361920507a4f85ffb7f70b96beaedacba2ce8c00aa663adb08689d3e" name = "github.com/bradfitz/gomemcache" packages = ["memcache"] + pruneopts = "NUT" revision = "1952afaa557dc08e8e0d89eafab110fb501c1a2b" [[projects]] branch = "master" + digest = "1:8ecb89af7dfe3ac401bdb0c9390b134ef96a97e85f732d2b0604fb7b3977839f" name = "github.com/codahale/hdrhistogram" packages = ["."] + pruneopts = "NUT" revision = "3a0bb77429bd3a61596f5e8a3172445844342120" [[projects]] + digest = "1:5dba68a1600a235630e208cb7196b24e58fcbb77bb7a6bec08fcd23f081b0a58" name = "github.com/codegangsta/cli" packages = ["."] + pruneopts = "NUT" revision = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1" version = "v1.20.0" [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "NUT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:1b318d2dd6cea8a1a8d8ec70348852303bd3e491df74e8bca6e32eb5a4d06970" name = "github.com/denisenkom/go-mssqldb" packages = [ ".", - "internal/cp" + "internal/cp", ] + pruneopts = "NUT" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" [[projects]] + branch = "master" + digest = "1:2da5f11ad66ff01a27a5c3dba4620b7eee2327be75b32c9ee9f87c9a8001ecbf" + name = "github.com/facebookgo/inject" + packages = ["."] + pruneopts = "NUT" + revision = "cc1aa653e50f6a9893bcaef89e673e5b24e1e97b" + +[[projects]] + branch = "master" + digest = "1:1108df7f658c90db041e0d6174d55be689aaeb0585913b9c3c7aab51a3a6b2b1" + name = "github.com/facebookgo/structtag" + packages = ["."] + pruneopts = "NUT" + revision = "217e25fb96916cc60332e399c9aa63f5c422ceed" + +[[projects]] + digest = "1:ade392a843b2035effb4b4a2efa2c3bab3eb29b992e98bacf9c898b0ecb54e45" name = "github.com/fatih/color" packages = ["."] - revision = "570b54cabe6b8eb0bc2dfce68d964677d63b5260" - version = "v1.5.0" + pruneopts = "NUT" + revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" + version = "v1.7.0" [[projects]] + digest = "1:e05711632e1515319b014e8fe4cbe1d30ab024c473403f60cf0fdeb4c586a474" name = "github.com/go-ini/ini" packages = ["."] - revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a" - version = "v1.32.0" + pruneopts = "NUT" + revision = "6529cf7c58879c08d927016dde4477f18a0634cb" + version = "v1.36.0" [[projects]] + digest = "1:7e1c00b9959544fa1ccca7cf0407a5b29ac6d5201059c4fac6f599cb99bfd24d" name = "github.com/go-ldap/ldap" packages = ["."] + pruneopts = "NUT" revision = "bb7a9ca6e4fbc2129e3db588a34bc970ffe811a9" version = "v2.5.1" [[projects]] branch = "master" + digest = "1:682a0aca743a1a4a36697f3d7f86c0ed403c4e3a780db9935f633242855eac9c" name = "github.com/go-macaron/binding" packages = ["."] + pruneopts = "NUT" revision = "ac54ee249c27dca7e76fad851a4a04b73bd1b183" [[projects]] branch = "master" + digest = "1:6326b27f8e0c8e135c8674ddbc619fae879664ac832e8e6fa6a23ce0d279ed4d" name = "github.com/go-macaron/gzip" packages = ["."] + pruneopts = "NUT" revision = "cad1c6580a07c56f5f6bc52d66002a05985c5854" [[projects]] branch = "master" + digest = "1:fb8711b648d1ff03104fc1d9593a13cb1d5120be7ba2b01641c14ccae286a9e3" name = "github.com/go-macaron/inject" packages = ["."] + pruneopts = "NUT" revision = "d8a0b8677191f4380287cfebd08e462217bac7ad" [[projects]] branch = "master" + digest = "1:21577aafe885f088e8086a3415f154c63c0b7ce956a6994df2ac5776bc01b7e3" name = "github.com/go-macaron/session" packages = [ ".", "memcache", - "mysql", "postgres", - "redis" + "redis", ] - revision = "b8e286a0dba8f4999042d6b258daf51b31d08938" + pruneopts = "NUT" + revision = "068d408f9c54c7fa7fcc5e2bdd3241ab21280c9e" [[projects]] + digest = "1:fddd4bada6100d6fc49a9f32f18ba5718db45a58e4b00aa6377e1cfbf06af34f" name = "github.com/go-sql-driver/mysql" packages = ["."] + pruneopts = "NUT" revision = "2cc627ac8defc45d65066ae98f898166f580f9a4" [[projects]] + digest = "1:a1efdbc2762667c8a41cbf02b19a0549c846bf2c1d08cad4f445e3344089f1f0" name = "github.com/go-stack/stack" packages = ["."] + pruneopts = "NUT" revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" version = "v1.7.0" [[projects]] - branch = "master" + digest = "1:b9d4f09cdaaa9e7dca5ed0b501ca5519eb2168dd01fc5d174c54edfe42a7d5ed" name = "github.com/go-xorm/builder" packages = ["."] - revision = "488224409dd8aa2ce7a5baf8d10d55764a913738" + pruneopts = "NUT" + revision = "bad0a612f0d6277b953910822ab5dfb30dd18237" + version = "v0.2.0" [[projects]] + digest = "1:30fb106b0cd6d64ea6fccded579c8f7d788460092f885fcc8f3edd02fc2085a4" name = "github.com/go-xorm/core" packages = ["."] - revision = "e8409d73255791843585964791443dbad877058c" + pruneopts = "NUT" + revision = "da1adaf7a28ca792961721a34e6e04945200c890" + version = "v0.5.7" [[projects]] + digest = "1:007d1354e4f44e6a393337e7623bcf911dfe75d6ef30fb767a6a0b65d302f5ed" name = "github.com/go-xorm/xorm" packages = ["."] - revision = "6687a2b4e824f4d87f2d65060ec5cb0d896dff1e" + pruneopts = "NUT" + revision = "1933dd69e294c0a26c0266637067f24dbb25770c" + version = "v0.6.4" [[projects]] branch = "master" + digest = "1:ffbb19fb66f140b5ea059428d1f84246a055d1bc3d9456c1e5c3d143611f03d0" name = "github.com/golang/protobuf" packages = [ "proto", "ptypes", "ptypes/any", "ptypes/duration", - "ptypes/timestamp" + "ptypes/timestamp", ] - revision = "c65a0412e71e8b9b3bfd22925720d23c0f054237" + pruneopts = "NUT" + revision = "927b65914520a8b7d44f5c9057611cfec6b2e2d0" [[projects]] branch = "master" + digest = "1:f14d1b50e0075fb00177f12a96dd7addf93d1e2883c25befd17285b779549795" name = "github.com/gopherjs/gopherjs" packages = ["js"] - revision = "178c176a91fe05e3e6c58fa5c989bad19e6cdcb3" + pruneopts = "NUT" + revision = "8dffc02ea1cb8398bb73f30424697c60fcf8d4c5" [[projects]] + digest = "1:3b708ebf63bfa9ba3313bedb8526bc0bb284e51474e65e958481476a9d4a12aa" name = "github.com/gorilla/websocket" packages = ["."] + pruneopts = "NUT" revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" version = "v1.2.0" [[projects]] + digest = "1:4e771d1c6e15ca4516ad971c34205c822b5cff2747179679d7b321e4e1bfe431" name = "github.com/gosimple/slug" packages = ["."] + pruneopts = "NUT" revision = "e9f42fa127660e552d0ad2b589868d403a9be7c6" version = "v1.1.1" [[projects]] branch = "master" - name = "github.com/grafana/grafana_plugin_model" - packages = ["go/datasource"] - revision = "dfe5dc0a6ce05825ba7fe2d0323d92e631bffa89" + digest = "1:08e53c69cd267ef7d71eeae5d953153d0d2bc1b8e0b498731fe9acaead7001b6" + name = "github.com/grafana/grafana-plugin-model" + packages = [ + "go/datasource", + "go/renderer", + ] + pruneopts = "NUT" + revision = "84176c64269d8060f99e750ee8aba6f062753336" [[projects]] branch = "master" + digest = "1:58ba5285227b0f635652cd4aa82c4cfd00b590191eadd823462f0c9f64e3ae07" name = "github.com/hashicorp/go-hclog" packages = ["."] - revision = "5bcb0f17e36442247290887cc914a6e507afa5c4" + pruneopts = "NUT" + revision = "69ff559dc25f3b435631604f573a5fa1efdb6433" [[projects]] + digest = "1:532090ffc3b05a7e4c0229dd2698d79149f2e0683df993224a8b202f607fb605" name = "github.com/hashicorp/go-plugin" packages = ["."] - revision = "3e6d191694b5a3a2b99755f31b47fa209e4bcd09" + pruneopts = "NUT" + revision = "e8d22c780116115ae5624720c9af0c97afe4f551" [[projects]] branch = "master" + digest = "1:8925116d1edcd85fc0c014e1aa69ce12892489b48ee633a605c46d893b8c151f" name = "github.com/hashicorp/go-version" packages = ["."] - revision = "4fe82ae3040f80a03d04d2cccb5606a626b8e1ee" + pruneopts = "NUT" + revision = "23480c0665776210b5fbbac6eaaee40e3e6a96b7" [[projects]] branch = "master" + digest = "1:8deb0c5545c824dfeb0ac77ab8eb67a3d541eab76df5c85ce93064ef02d44cd0" name = "github.com/hashicorp/yamux" packages = ["."] - revision = "683f49123a33db61abfb241b7ac5e4af4dc54d55" + pruneopts = "NUT" + revision = "7221087c3d281fda5f794e28c2ea4c6e4d5c4558" [[projects]] + digest = "1:efbe016b6d198cf44f1db0ed2fbdf1b36ebf1f6956cc9b76d6affa96f022d368" name = "github.com/inconshreveable/log15" packages = ["."] + pruneopts = "NUT" revision = "0decfc6c20d9ca0ad143b0e89dcaa20f810b4fb3" version = "v2.13" [[projects]] + digest = "1:ac6d01547ec4f7f673311b4663909269bfb8249952de3279799289467837c3cc" name = "github.com/jmespath/go-jmespath" packages = ["."] + pruneopts = "NUT" revision = "0b12d6b5" [[projects]] + digest = "1:6ddab442e52381bab82fb6c07ef3f4b565ff7ec4b8fae96d8dd4b8573a460597" name = "github.com/jtolds/gls" packages = ["."] + pruneopts = "NUT" revision = "77f18212c9c7edc9bd6a33d383a7b545ce62f064" version = "v4.2.1" [[projects]] + digest = "1:1da1796a71eb70f1e3e085984d044f67840bb0326816ec8276231aa87b1b9fc3" name = "github.com/klauspost/compress" packages = [ "flate", - "gzip" + "gzip", ] + pruneopts = "NUT" revision = "6c8db69c4b49dd4df1fff66996cf556176d0b9bf" version = "v1.2.1" [[projects]] + digest = "1:5e55a8699c9ff7aba1e4c8952aeda209685d88d4cb63a8766c338e333b8e65d6" name = "github.com/klauspost/cpuid" packages = ["."] + pruneopts = "NUT" revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da" version = "v1.1" [[projects]] + digest = "1:b95da1293525625ef6f07be79d537b9bf2ecd7901efcf9a92193edafbd55b9ef" name = "github.com/klauspost/crc32" packages = ["."] + pruneopts = "NUT" revision = "cb6bfca970f6908083f26f39a79009d608efd5cd" version = "v1.1" [[projects]] - branch = "master" + digest = "1:7b21c7fc5551b46d1308b4ffa9e9e49b66c7a8b0ba88c0130474b0e7a20d859f" name = "github.com/kr/pretty" packages = ["."] - revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4" + pruneopts = "NUT" + revision = "73f6ac0b30a98e433b289500d779f50c1a6f0712" + version = "v0.1.0" [[projects]] - branch = "master" + digest = "1:c3a7836b5904db0f8b609595b619916a6831cb35b8b714aec39f96d00c6155d8" name = "github.com/kr/text" packages = ["."] - revision = "7cafcd837844e784b526369c9bce262804aebc60" + pruneopts = "NUT" + revision = "e2ffdb16a802fe2bb95e2e35ff34f0e53aeef34f" + version = "v0.1.0" [[projects]] branch = "master" + digest = "1:7a1e592f0349d56fac8ce47f28469e4e7f4ce637cb26f40c88da9dff25db1c98" name = "github.com/lib/pq" packages = [ ".", - "oid" + "oid", ] - revision = "61fe37aa2ee24fabcdbe5c4ac1d4ac566f88f345" + pruneopts = "NUT" + revision = "d34b9ff171c21ad295489235aec8b6626023cd04" [[projects]] + digest = "1:08c231ec84231a7e23d67e4b58f975e1423695a32467a362ee55a803f9de8061" name = "github.com/mattn/go-colorable" packages = ["."] + pruneopts = "NUT" revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" version = "v0.0.9" [[projects]] + digest = "1:bc4f7eec3b7be8c6cb1f0af6c1e3333d5bb71072951aaaae2f05067b0803f287" name = "github.com/mattn/go-isatty" packages = ["."] + pruneopts = "NUT" revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" [[projects]] + digest = "1:536979f1c56397dbf91c2785159b37dec37e35d3bffa3cd1cfe66d25f51f8088" name = "github.com/mattn/go-sqlite3" packages = ["."] - revision = "6c771bb9887719704b210e87e934f08be014bdb1" - version = "v1.6.0" + pruneopts = "NUT" + revision = "323a32be5a2421b8c7087225079c6c900ec397cd" + version = "v1.7.0" [[projects]] + digest = "1:5985ef4caf91ece5d54817c11ea25f182697534f8ae6521eadcd628c142ac4b6" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] + pruneopts = "NUT" revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:18b773b92ac82a451c1276bd2776c1e55ce057ee202691ab33c8d6690efcc048" name = "github.com/mitchellh/go-testing-interface" packages = ["."] + pruneopts = "NUT" revision = "a61a99592b77c9ba629d254a693acffaeb4b7e28" [[projects]] + digest = "1:3b517122f3aad1ecce45a630ea912b3092b4729f25532a911d0cb2935a1f9352" + name = "github.com/oklog/run" + packages = ["."] + pruneopts = "NUT" + revision = "4dadeb3030eda0273a12382bb2348ffc7c9d1a39" + version = "v1.0.0" + +[[projects]] + digest = "1:7da29c22bcc5c2ffb308324377dc00b5084650348c2799e573ed226d8cc9faf0" name = "github.com/opentracing/opentracing-go" packages = [ ".", "ext", - "log" + "log", ] + pruneopts = "NUT" revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" [[projects]] + digest = "1:748946761cf99c8b73cef5a3c0ee3e040859dd713a20cece0d0e0dc04e6ceca7" name = "github.com/patrickmn/go-cache" packages = ["."] + pruneopts = "NUT" revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0" version = "v2.1.0" [[projects]] + digest = "1:5cf3f025cbee5951a4ee961de067c8a89fc95a5adabead774f82822efabab121" + name = "github.com/pkg/errors" + packages = ["."] + pruneopts = "NUT" + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + digest = "1:4759bed95e3a52febc18c071db28790a5c6e9e106ee201a37add6f6a056f8f9c" name = "github.com/prometheus/client_golang" packages = [ "api", "api/prometheus/v1", "prometheus", - "prometheus/promhttp" + "prometheus/promhttp", ] + pruneopts = "NUT" revision = "967789050ba94deca04a5e84cce8ad472ce313c1" version = "v0.9.0-pre1" [[projects]] branch = "master" + digest = "1:32d10bdfa8f09ecf13598324dba86ab891f11db3c538b6a34d1c3b5b99d7c36b" name = "github.com/prometheus/client_model" packages = ["go"] + pruneopts = "NUT" revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" [[projects]] branch = "master" + digest = "1:768b555b86742de2f28beb37f1dedce9a75f91f871d75b5717c96399c1a78c08" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", - "model" + "model", ] - revision = "89604d197083d4781071d3c65855d24ecfb0a563" + pruneopts = "NUT" + revision = "d811d2e9bf898806ecfb6ef6296774b13ffc314c" [[projects]] branch = "master" + digest = "1:c4a213a8d73fbb0b13f717ba7996116602ef18ecb42b91d77405877914cb0349" name = "github.com/prometheus/procfs" packages = [ ".", "internal/util", - "nfsd", - "xfs" + "nfs", + "xfs", ] - revision = "85fadb6e89903ef7cca6f6a804474cd5ea85b6e1" + pruneopts = "NUT" + revision = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e" [[projects]] branch = "master" + digest = "1:16e2136a67ec44aa2d1d6b0fd65394b3c4a8b2a1b6730c77967f7b7b06b179b2" name = "github.com/rainycape/unidecode" packages = ["."] + pruneopts = "NUT" revision = "cb7f23ec59bec0d61b19c56cd88cee3d0cc1870c" [[projects]] - branch = "master" + digest = "1:d917313f309bda80d27274d53985bc65651f81a5b66b820749ac7f8ef061fd04" name = "github.com/sergi/go-diff" packages = ["diffmatchpatch"] + pruneopts = "NUT" revision = "1744e2970ca51c86172c8190fadad617561ed6e7" + version = "v1.0.0" [[projects]] + digest = "1:1f0b284a6858827de4c27c66b49b2b25df3e16b031c2b57b7892273131e7dd2b" name = "github.com/smartystreets/assertions" packages = [ ".", "internal/go-render/render", - "internal/oglematchers" + "internal/oglematchers", ] - revision = "0b37b35ec7434b77e77a4bb29b79677cced992ea" - version = "1.8.1" + pruneopts = "NUT" + revision = "7678a5452ebea5b7090a6b163f844c133f523da2" + version = "1.8.3" [[projects]] + digest = "1:7efd0b2309cdd6468029fa30c808c50a820c9344df07e1a4bbdaf18f282907aa" name = "github.com/smartystreets/goconvey" packages = [ "convey", "convey/gotest", - "convey/reporting" + "convey/reporting", ] + pruneopts = "NUT" revision = "9e8dc3f972df6c8fcc0375ef492c24d0bb204857" version = "1.6.3" [[projects]] branch = "master" + digest = "1:a66add8dd963bfc72649017c1b321198f596cb4958cb1a11ff91a1be8691020b" name = "github.com/teris-io/shortid" packages = ["."] + pruneopts = "NUT" revision = "771a37caa5cf0c81f585d7b6df4dfc77e0615b5c" [[projects]] + digest = "1:3d48c38e0eca8c66df62379c5ae7a83fb5cd839b94f241354c07ba077da7bc45" name = "github.com/uber/jaeger-client-go" packages = [ ".", @@ -434,89 +594,111 @@ "internal/baggage", "internal/baggage/remote", "internal/spanlog", + "internal/throttler", + "internal/throttler/remote", "log", "rpcmetrics", + "thrift", "thrift-gen/agent", "thrift-gen/baggage", "thrift-gen/jaeger", "thrift-gen/sampling", "thrift-gen/zipkincore", - "utils" + "utils", ] - revision = "3ac96c6e679cb60a74589b0d0aa7c70a906183f7" - version = "v2.11.2" + pruneopts = "NUT" + revision = "b043381d944715b469fd6b37addfd30145ca1758" + version = "v2.14.0" [[projects]] + digest = "1:0f09db8429e19d57c8346ad76fbbc679341fa86073d3b8fb5ac919f0357d8f4c" name = "github.com/uber/jaeger-lib" packages = ["metrics"] - revision = "7f95f4f7e80028096410abddaae2556e4c61b59f" - version = "v1.3.1" + pruneopts = "NUT" + revision = "ed3a127ec5fef7ae9ea95b01b542c47fbd999ce5" + version = "v1.5.0" [[projects]] + digest = "1:4c7d12ad3ef47bb03892a52e2609dc9a9cff93136ca9c7d31c00b79fcbc23c7b" name = "github.com/yudai/gojsondiff" packages = [ ".", - "formatter" + "formatter", ] + pruneopts = "NUT" revision = "7b1b7adf999dab73a6eb02669c3d82dbb27a3dd6" version = "1.0.0" [[projects]] branch = "master" + digest = "1:e50cbf8eba568d59b71e08c22c2a77809ed4646ae06ef4abb32b3d3d3fdb1a77" name = "github.com/yudai/golcs" packages = ["."] + pruneopts = "NUT" revision = "ecda9a501e8220fae3b4b600c3db4b0ba22cfc68" [[projects]] branch = "master" + digest = "1:758f363e0dff33cf00b234be2efb12f919d79b42d5ae3909ff9eb69ef2c3cca5" name = "golang.org/x/crypto" packages = [ + "ed25519", + "ed25519/internal/edwards25519", "md4", - "pbkdf2" + "pbkdf2", ] - revision = "3d37316aaa6bd9929127ac9a527abf408178ea7b" + pruneopts = "NUT" + revision = "1a580b3eff7814fc9b40602fd35256c63b50f491" [[projects]] branch = "master" + digest = "1:0b3fee9c4472022a0982ee0d81e08b3cc3e595f50befd7a4b358b48540d9d8c5" name = "golang.org/x/net" packages = [ "context", "context/ctxhttp", + "http/httpguts", "http2", "http2/hpack", "idna", "internal/timeseries", - "lex/httplex", - "trace" + "trace", ] - revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec" + pruneopts = "NUT" + revision = "2491c5de3490fced2f6cff376127c667efeed857" [[projects]] branch = "master" + digest = "1:46bd4e66bfce5e77f08fc2e8dcacc3676e679241ce83d9c150ff0397d686dd44" name = "golang.org/x/oauth2" packages = [ ".", "google", "internal", "jws", - "jwt" + "jwt", ] - revision = "b28fcf2b08a19742b43084fb40ab78ac6c3d8067" + pruneopts = "NUT" + revision = "cdc340f7c179dbbfa4afd43b7614e8fcadde4269" [[projects]] branch = "master" + digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239" name = "golang.org/x/sync" packages = ["errgroup"] - revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5" + pruneopts = "NUT" + revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" [[projects]] branch = "master" + digest = "1:ec21c5bf0572488865b93e30ffd9132afbf85bec0b20c2d6cbcf349cf2031ed5" name = "golang.org/x/sys" packages = ["unix"] - revision = "af50095a40f9041b3b38960738837185c26e9419" + pruneopts = "NUT" + revision = "7c87d13f8e835d2fb3a70a2912c811ed0c1d241b" [[projects]] - branch = "master" + digest = "1:e7071ed636b5422cc51c0e3a6cebc229d6c9fffc528814b519a980641422d619" name = "golang.org/x/text" packages = [ "collate", @@ -532,11 +714,14 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] - revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3" + pruneopts = "NUT" + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" [[projects]] + digest = "1:dbd5568923513ee74aa626d027e2a8a352cf8f35df41d19f4e34491d1858c38b" name = "google.golang.org/appengine" packages = [ ".", @@ -549,18 +734,22 @@ "internal/modules", "internal/remote_api", "internal/urlfetch", - "urlfetch" + "urlfetch", ] + pruneopts = "NUT" revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:3c24554c312721e98fa6b76403e7100cf974eb46b1255ea7fc6471db9a9ce498" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "a8101f21cf983e773d0c1133ebc5424792003214" + pruneopts = "NUT" + revision = "7bb2a897381c9c5ab2aeb8614f758d7766af68ff" [[projects]] + digest = "1:840b77b6eb539b830bb760b6e30b688ed2ff484bd83466fce2395835ed9367fe" name = "google.golang.org/grpc" packages = [ ".", @@ -571,6 +760,7 @@ "connectivity", "credentials", "encoding", + "encoding/proto", "grpclb/grpc_lb_v1/messages", "grpclog", "health", @@ -586,62 +776,167 @@ "stats", "status", "tap", - "transport" + "transport", ] - revision = "6b51017f791ae1cfbec89c52efdf444b13b550ef" - version = "v1.9.2" + pruneopts = "NUT" + revision = "1e2570b1b19ade82d8dbb31bba4e65e9f9ef5b34" + version = "v1.11.1" [[projects]] branch = "v3" + digest = "1:1244a9b3856f70d5ffb74bbfd780fc9d47f93f2049fa265c6fb602878f507bf8" name = "gopkg.in/alexcesaro/quotedprintable.v3" packages = ["."] + pruneopts = "NUT" revision = "2caba252f4dc53eaf6b553000885530023f54623" [[projects]] + digest = "1:aea6e9483c167cc6fdf1274c442558c5dda8fd3373372be04d98c79100868da1" name = "gopkg.in/asn1-ber.v1" packages = ["."] + pruneopts = "NUT" revision = "379148ca0225df7a432012b8df0355c2a2063ac0" version = "v1.2" [[projects]] + digest = "1:24bfc2e8bf971485cb5ba0f0e5b08a1b806cca5828134df76b32d1ea50f2ab49" name = "gopkg.in/bufio.v1" packages = ["."] + pruneopts = "NUT" revision = "567b2bfa514e796916c4747494d6ff5132a1dfce" version = "v1" [[projects]] - branch = "v2" - name = "gopkg.in/gomail.v2" - packages = ["."] - revision = "81ebce5c23dfd25c6c67194b37d3dd3f338c98b1" - -[[projects]] + digest = "1:e05711632e1515319b014e8fe4cbe1d30ab024c473403f60cf0fdeb4c586a474" name = "gopkg.in/ini.v1" packages = ["."] - revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a" - version = "v1.32.0" + pruneopts = "NUT" + revision = "6529cf7c58879c08d927016dde4477f18a0634cb" + version = "v1.36.0" [[projects]] + digest = "1:3b0cf3a465fd07f76e5fc1a9d0783c662dac0de9fc73d713ebe162768fd87b5f" name = "gopkg.in/macaron.v1" packages = ["."] - revision = "75f2e9b42e99652f0d82b28ccb73648f44615faa" - version = "v1.2.4" + pruneopts = "NUT" + revision = "c1be95e6d21e769e44e1ec33cec9da5837861c10" + version = "v1.3.1" [[projects]] + branch = "v2" + digest = "1:d52332f9e9f2c6343652e13aa3fd40cfd03353520c9a48d90f21215d3012d50f" + name = "gopkg.in/mail.v2" + packages = ["."] + pruneopts = "NUT" + revision = "5bc5c8bb07bd8d2803831fbaf8cbd630fcde2c68" + +[[projects]] + digest = "1:00126f697efdcab42f07c89ac8bf0095fb2328aef6464e070055154088cea859" name = "gopkg.in/redis.v2" packages = ["."] + pruneopts = "NUT" revision = "e6179049628164864e6e84e973cfb56335748dea" version = "v2.3.2" +[[projects]] + digest = "1:a50fabe7a46692dc7c656310add3d517abe7914df02afd151ef84da884605dc8" + name = "gopkg.in/square/go-jose.v2" + packages = [ + ".", + "cipher", + "json", + ] + pruneopts = "NUT" + revision = "ef984e69dd356202fd4e4910d4d9c24468bdf0b8" + version = "v2.1.9" + [[projects]] branch = "v2" + digest = "1:7c95b35057a0ff2e19f707173cc1a947fa43a6eb5c4d300d196ece0334046082" name = "gopkg.in/yaml.v2" packages = ["."] - revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" + pruneopts = "NUT" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5e65aeace832f1b4be17e7ff5d5714513c40f31b94b885f64f98f2332968d7c6" + input-imports = [ + "github.com/BurntSushi/toml", + "github.com/Unknwon/com", + "github.com/VividCortex/mysqlerr", + "github.com/aws/aws-sdk-go/aws", + "github.com/aws/aws-sdk-go/aws/awserr", + "github.com/aws/aws-sdk-go/aws/awsutil", + "github.com/aws/aws-sdk-go/aws/credentials", + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds", + "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds", + "github.com/aws/aws-sdk-go/aws/defaults", + "github.com/aws/aws-sdk-go/aws/ec2metadata", + "github.com/aws/aws-sdk-go/aws/endpoints", + "github.com/aws/aws-sdk-go/aws/request", + "github.com/aws/aws-sdk-go/aws/session", + "github.com/aws/aws-sdk-go/service/cloudwatch", + "github.com/aws/aws-sdk-go/service/ec2", + "github.com/aws/aws-sdk-go/service/ec2/ec2iface", + "github.com/aws/aws-sdk-go/service/s3", + "github.com/aws/aws-sdk-go/service/sts", + "github.com/benbjohnson/clock", + "github.com/bmizerany/assert", + "github.com/codegangsta/cli", + "github.com/davecgh/go-spew/spew", + "github.com/denisenkom/go-mssqldb", + "github.com/facebookgo/inject", + "github.com/fatih/color", + "github.com/go-ldap/ldap", + "github.com/go-macaron/binding", + "github.com/go-macaron/gzip", + "github.com/go-macaron/session", + "github.com/go-macaron/session/memcache", + "github.com/go-macaron/session/postgres", + "github.com/go-macaron/session/redis", + "github.com/go-sql-driver/mysql", + "github.com/go-stack/stack", + "github.com/go-xorm/core", + "github.com/go-xorm/xorm", + "github.com/gorilla/websocket", + "github.com/gosimple/slug", + "github.com/grafana/grafana-plugin-model/go/datasource", + "github.com/grafana/grafana-plugin-model/go/renderer", + "github.com/hashicorp/go-hclog", + "github.com/hashicorp/go-plugin", + "github.com/hashicorp/go-version", + "github.com/inconshreveable/log15", + "github.com/lib/pq", + "github.com/mattn/go-isatty", + "github.com/mattn/go-sqlite3", + "github.com/opentracing/opentracing-go", + "github.com/opentracing/opentracing-go/ext", + "github.com/opentracing/opentracing-go/log", + "github.com/patrickmn/go-cache", + "github.com/pkg/errors", + "github.com/prometheus/client_golang/api", + "github.com/prometheus/client_golang/api/prometheus/v1", + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/prometheus/client_model/go", + "github.com/prometheus/common/expfmt", + "github.com/prometheus/common/model", + "github.com/smartystreets/goconvey/convey", + "github.com/teris-io/shortid", + "github.com/uber/jaeger-client-go/config", + "github.com/yudai/gojsondiff", + "github.com/yudai/gojsondiff/formatter", + "golang.org/x/net/context/ctxhttp", + "golang.org/x/oauth2", + "golang.org/x/oauth2/google", + "golang.org/x/oauth2/jwt", + "golang.org/x/sync/errgroup", + "gopkg.in/ini.v1", + "gopkg.in/macaron.v1", + "gopkg.in/mail.v2", + "gopkg.in/square/go-jose.v2", + "gopkg.in/yaml.v2", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index df163e01ed3..e3cbdeabb5d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -36,7 +36,7 @@ ignored = [ [[constraint]] name = "github.com/aws/aws-sdk-go" - version = "1.12.65" + version = "1.13.56" [[constraint]] branch = "master" @@ -85,13 +85,11 @@ ignored = [ [[constraint]] name = "github.com/go-xorm/core" - revision = "e8409d73255791843585964791443dbad877058c" - #version = "0.5.7" //keeping this since we would rather depend on version then commit + version = "=0.5.7" [[constraint]] name = "github.com/go-xorm/xorm" - revision = "6687a2b4e824f4d87f2d65060ec5cb0d896dff1e" - #version = "0.6.4" //keeping this since we would rather depend on version then commit + version = "=0.6.4" [[constraint]] name = "github.com/gorilla/websocket" @@ -103,12 +101,16 @@ ignored = [ [[constraint]] branch = "master" - name = "github.com/grafana/grafana_plugin_model" + name = "github.com/grafana/grafana-plugin-model" [[constraint]] branch = "master" name = "github.com/hashicorp/go-hclog" +[[constraint]] + name = "github.com/hashicorp/go-plugin" + revision = "e8d22c780116115ae5624720c9af0c97afe4f551" + [[constraint]] branch = "master" name = "github.com/hashicorp/go-version" @@ -127,7 +129,7 @@ ignored = [ [[constraint]] name = "github.com/mattn/go-sqlite3" - version = "1.6.0" + version = "1.7.0" [[constraint]] name = "github.com/opentracing/opentracing-go" @@ -174,7 +176,7 @@ ignored = [ name = "golang.org/x/sync" [[constraint]] - name = "gopkg.in/gomail.v2" + name = "gopkg.in/mail.v2" branch = "v2" [[constraint]] @@ -201,3 +203,11 @@ ignored = [ [[constraint]] name = "github.com/denisenkom/go-mssqldb" revision = "270bc3860bb94dd3a3ffd047377d746c5e276726" + +[[constraint]] + name = "github.com/VividCortex/mysqlerr" + branch = "master" + +[[constraint]] + name = "gopkg.in/square/go-jose.v2" + version = "2.1.9" diff --git a/Gruntfile.js b/Gruntfile.js index a0607ef49dc..de3e68d4a92 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -1,4 +1,3 @@ -/* jshint node:true */ 'use strict'; module.exports = function (grunt) { var os = require('os'); @@ -10,8 +9,17 @@ module.exports = function (grunt) { destDir: 'dist', tempDir: 'tmp', platform: process.platform.replace('win32', 'windows'), + enterprise: false, }; + if (grunt.option('platform')) { + config.platform = grunt.option('platform'); + } + + if (grunt.option('enterprise')) { + config.enterprise = true; + } + if (grunt.option('arch')) { config.arch = grunt.option('arch'); } else { @@ -22,7 +30,6 @@ module.exports = function (grunt) { } } - config.coverage = grunt.option('coverage'); config.phjs = grunt.option('phjsToRelease'); config.pkg.version = grunt.option('pkgVer') || config.pkg.version; diff --git a/Makefile b/Makefile index 6f7beb837d8..fcb740d2fac 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ +-include local/Makefile + all: deps build deps-go: go run build.go setup -deps-js: - yarn install --pure-lockfile --no-progress +deps-js: node_modules deps: deps-js @@ -22,6 +23,15 @@ build-js: build: build-go build-js +build-docker-dev: + @echo "\033[92mInfo:\033[0m the frontend code is expected to be built already." + go run build.go -goos linux -pkg-arch amd64 ${OPT} build package-only latest + cp dist/grafana-latest.linux-x64.tar.gz packaging/docker + cd packaging/docker && docker build --tag grafana/grafana:dev . + +build-docker-full: + docker build --tag grafana/grafana:dev . + test-go: go test -v ./pkg/... @@ -33,5 +43,9 @@ test: test-go test-js run: ./bin/grafana-server -protoc: - protoc -I pkg/tsdb/models pkg/tsdb/models/*.proto --go_out=plugins=grpc:pkg/tsdb/models/. \ No newline at end of file +clean: + rm -rf node_modules + rm -rf public/build + +node_modules: package.json yarn.lock + yarn install --pure-lockfile --no-progress diff --git a/NOTICE.md b/NOTICE.md index ca148971b62..899b2a3c3f9 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,5 +1,5 @@ -Copyright 2014-2017 Grafana Labs +Copyright 2014-2018 Grafana Labs This software is based on Kibana: Copyright 2012-2013 Elasticsearch BV diff --git a/PLUGIN_DEV.md b/PLUGIN_DEV.md index 9d831a95697..168b21dbd88 100644 --- a/PLUGIN_DEV.md +++ b/PLUGIN_DEV.md @@ -6,9 +6,10 @@ upgrading Grafana please check here before creating an issue. ## Links -- [Datasource plugin written in typescript](https://github.com/grafana/typescript-template-datasource) -- [Simple json dataource plugin](https://github.com/grafana/simple-json-datasource) +- [Datasource plugin written in TypeScript](https://github.com/grafana/typescript-template-datasource) +- [Simple JSON datasource plugin](https://github.com/grafana/simple-json-datasource) - [Plugin development guide](http://docs.grafana.org/plugins/developing/development/) +- [Webpack Grafana plugin template project](https://github.com/CorpGlory/grafana-plugin-template-webpack) ## Changes in v4.6 diff --git a/README.md b/README.md index 9a05633c391..5882ea8a6a3 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ the latest master builds [here](https://grafana.com/grafana/download) ### Dependencies -- Go 1.10 +- Go (Latest Stable) - NodeJS LTS ### Building the backend @@ -39,20 +39,24 @@ go run build.go build For this you need nodejs (v.6+). +To build the assets, rebuild on file change, and serve them by Grafana's webserver (http://localhost:3000): ```bash npm install -g yarn yarn install --pure-lockfile -npm run watch +yarn watch ``` -Run tests +Build the assets, rebuild on file change with Hot Module Replacement (HMR), and serve them by webpack-dev-server (http://localhost:3333): ```bash -npm run jest +yarn start +# OR set a theme +env GRAFANA_THEME=light yarn start ``` +Note: HMR for Angular is not supported. If you edit files in the Angular part of the app, the whole page will reload. -Run karma tests +Run tests ```bash -npm run karma +yarn jest ``` ### Recompile backend on source change @@ -65,6 +69,27 @@ bra run Open grafana in your browser (default: `http://localhost:3000`) and login with admin user (default: `user/pass = admin/admin`). +### Building a Docker image + +There are two different ways to build a Grafana docker image. If you're machine is setup for Grafana development and you run linux/amd64 you can build just the image. Otherwise, there is the option to build Grafana completely within Docker. + +Run the image you have built using: `docker run --rm -p 3000:3000 grafana/grafana:dev` + +#### Building on linux/amd64 (fast) + +1. Build the frontend `go run build.go build-frontend` +2. Build the docker image `make build-docker-dev` + +The resulting image will be tagged as `grafana/grafana:dev` + +#### Building anywhere (slower) + +Choose this option to build on platforms other than linux/amd64 and/or not have to setup the Grafana development environment. + +1. `make build-docker-full` or `docker build -t grafana/grafana:dev .` + +The resulting image will be tagged as `grafana/grafana:dev` + ### Dev config Create a custom.ini in the conf directory to override default configuration options. @@ -80,28 +105,24 @@ In your custom.ini uncomment (remove the leading `;`) sign. And set `app_mode = #### Frontend Execute all frontend tests ```bash -npm run test +yarn test ``` -Writing & watching frontend tests (we have two test runners) +Writing & watching frontend tests -- jest for all new tests that do not require browser context (React+more) - - Start watcher: `npm run jest` - - Jest will run all test files that end with the name ".jest.ts" -- karma + mocha is used for testing angularjs components. We do want to migrate these test to jest over time (if possible). - - Start watcher: `npm run karma` - - Karma+Mocha runs all files that end with the name "_specs.ts". +- Start watcher: `yarn jest` +- Jest will run all test files that end with the name ".test.ts" #### Backend ```bash # Run Golang tests using sqlite3 as database (default) -go test ./pkg/... +go test ./pkg/... # Run Golang tests using mysql as database - convenient to use /docker/blocks/mysql_tests -GRAFANA_TEST_DB=mysql go test ./pkg/... +GRAFANA_TEST_DB=mysql go test ./pkg/... # Run Golang tests using postgres as database - convenient to use /docker/blocks/postgres_tests -GRAFANA_TEST_DB=postgres go test ./pkg/... +GRAFANA_TEST_DB=postgres go test ./pkg/... ``` ## Contribute @@ -117,5 +138,5 @@ plugin development. ## License -Grafana is distributed under Apache 2.0 License. +Grafana is distributed under [Apache 2.0 License](https://github.com/grafana/grafana/blob/master/LICENSE.md). diff --git a/ROADMAP.md b/ROADMAP.md index e7bed99489e..891bc9f790b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,40 +1,27 @@ -# Roadmap (2018-02-22) +# Roadmap (2018-08-07) This roadmap is a tentative plan for the core development team. Things change constantly as PRs come in and priorities change. But it will give you an idea of our current vision and plan. - -### Short term (1-2 months) - -- v5.1 - - Build speed improvements & integration test execution - - Kubernetes friendly docker container - - Enterprise LDAP - - Provisioning workflow - - MSSQL datasource -### Mid term (2-4 months) - -- v5.2 - - Azure monitor backend rewrite - - Elasticsearch alerting - - First login registration view - - Backend plugins? (alert notifiers, auth) - - Crossplatform builds - - IFQL Initial support +### Short term (1-2 months) + - PRs & Bugs + - Multi-Stat panel + - Metrics & Log Explore UI + +### Mid term (2-4 months) + - React Panels + - Change visualization (panel type) on the fly. + - Templating Query Editor UI Plugin hook + - Backend plugins ### Long term (4 - 8 months) - -- Alerting improvements (silence, per series tracking, etc) -- Progress on React migration -- Change visualization (panel type) on the fly. -- Multi stat panel (vertical version of singlestat with bars/graph mode with big number etc) -- Repeat panel by query results + - Alerting improvements (silence, per series tracking, etc) + - Progress on React migration ### In a distant future far far away - -- Meta queries -- Integrated light weight TSDB -- Web socket & live data sources + - Meta queries + - Integrated light weight TSDB + - Web socket & live data sources ### Outside contributions We know this is being worked on right now by contributors (and we hope to merge it when it's ready). diff --git a/UPGRADING_DEPENDENCIES.md b/UPGRADING_DEPENDENCIES.md new file mode 100644 index 00000000000..f3d2adbd71a --- /dev/null +++ b/UPGRADING_DEPENDENCIES.md @@ -0,0 +1,89 @@ +# Guide to Upgrading Dependencies + +Upgrading Go or Node.js requires making changes in many different files. See below for a list and explanation for each. + +## Go + +- CircleCi +- `grafana/build-container` +- Appveyor +- Dockerfile + +## Node.js + +- CircleCI +- `grafana/build-container` +- Appveyor +- Dockerfile + +## Go Dependencies + +Updated using `dep`. + +- `Gopkg.toml` +- `Gopkg.lock` + +## Node.js Dependencies + +Updated using `yarn`. + +- `package.json` + +## Where to make changes + +### CircleCI + +Our builds run on CircleCI through our build script. + +#### Files + +- `.circleci/config.yml`. + +#### Dependencies + +- nodejs +- golang +- grafana/build-container (our custom docker build container) + +### grafana/build-container + +The main build step (in CircleCI) is built using a custom build container that comes pre-baked with some of the neccesary dependencies. + +Link: [grafana-build-container](https://github.com/grafana/grafana-build-container) + +#### Dependencies + +- fpm +- nodejs +- golang +- crosscompiling (several compilers) + +### Appveyor + +Master and release builds trigger test runs on Appveyors build environment so that tests will run on Windows. + +#### Files: + +- `appveyor.yml` + +#### Dependencies + +- nodejs +- golang + +### Dockerfile + +There is a Docker build for Grafana in the root of the project that allows anyone to build Grafana just using Docker. + +#### Files + +- `Dockerfile` + +#### Dependencies + +- nodejs +- golang + +### Local developer environments + +Please send out a notice in the grafana-dev slack channel when updating Go or Node.js to make it easier for everyone to update their local developer environments. \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml index 5d67edca9d9..4bbd3668e19 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,9 +5,9 @@ os: Windows Server 2012 R2 clone_folder: c:\gopath\src\github.com\grafana\grafana environment: - nodejs_version: "6" - GOPATH: c:\gopath - GOVERSION: 1.9.2 + nodejs_version: "8" + GOPATH: C:\gopath + GOVERSION: 1.11 install: - rmdir c:\go /s /q @@ -38,16 +38,3 @@ artifacts: - path: grafana-*windows-*.* name: binzip type: zip - -deploy: - - provider: Environment - name: GrafanaReleaseMaster - on: - buildType: master - - - provider: Environment - name: GrafanaReleaseRelease - on: - buildType: release - - diff --git a/build.go b/build.go index c38c452f61f..dc789670f62 100644 --- a/build.go +++ b/build.go @@ -16,55 +16,67 @@ import ( "os/exec" "path" "path/filepath" - "regexp" "runtime" "strconv" "strings" "time" ) +const ( + windows = "windows" + linux = "linux" +) + var ( - versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`) - goarch string - goos string - gocc string - gocxx string - cgo string - pkgArch string - version string = "v1" + //versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`) + goarch string + goos string + gocc string + cgo bool + pkgArch string + version string = "v1" // deb & rpm does not support semver so have to handle their version a little differently linuxPackageVersion string = "v1" linuxPackageIteration string = "" race bool phjsToRelease string workingDir string - includeBuildNumber bool = true - buildNumber int = 0 + includeBuildId bool = true + buildId string = "0" binaries []string = []string{"grafana-server", "grafana-cli"} + isDev bool = false + enterprise bool = false ) -const minGoVersion = 1.8 - func main() { log.SetOutput(os.Stdout) log.SetFlags(0) ensureGoPath() + var buildIdRaw string + flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH") flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS") flag.StringVar(&gocc, "cc", "", "CC") - flag.StringVar(&gocxx, "cxx", "", "CXX") - flag.StringVar(&cgo, "cgo-enabled", "", "CGO_ENABLED") + flag.BoolVar(&cgo, "cgo-enabled", cgo, "Enable cgo") flag.StringVar(&pkgArch, "pkg-arch", "", "PKG ARCH") flag.StringVar(&phjsToRelease, "phjs", "", "PhantomJS binary") flag.BoolVar(&race, "race", race, "Use race detector") - flag.BoolVar(&includeBuildNumber, "includeBuildNumber", includeBuildNumber, "IncludeBuildNumber in package name") - flag.IntVar(&buildNumber, "buildNumber", 0, "Build number from CI system") + flag.BoolVar(&includeBuildId, "includeBuildId", includeBuildId, "IncludeBuildId in package name") + flag.BoolVar(&enterprise, "enterprise", enterprise, "Build enterprise version of Grafana") + flag.StringVar(&buildIdRaw, "buildId", "0", "Build ID from CI system") + flag.BoolVar(&isDev, "dev", isDev, "optimal for development, skips certain steps") flag.Parse() + buildId = shortenBuildId(buildIdRaw) + readVersionFromPackageJson() + if pkgArch == "" { + pkgArch = goarch + } + log.Printf("Version: %s, Linux Version: %s, Package Iteration: %s\n", version, linuxPackageVersion, linuxPackageIteration) if flag.NArg() == 0 { @@ -92,18 +104,28 @@ func main() { build("grafana-server", "./pkg/cmd/grafana-server", []string{}) case "build": - clean() + //clean() for _, binary := range binaries { build(binary, "./pkg/cmd/"+binary, []string{}) } + case "build-frontend": + grunt(gruntBuildArg("build")...) + case "test": test("./pkg/...") grunt("test") case "package": - grunt(gruntBuildArg("release")...) - if runtime.GOOS != "windows" { + grunt(gruntBuildArg("build")...) + grunt(gruntBuildArg("package")...) + if goos == linux { + createLinuxPackages() + } + + case "package-only": + grunt(gruntBuildArg("package")...) + if goos == linux { createLinuxPackages() } @@ -137,9 +159,9 @@ func makeLatestDistCopies() { } latestMapping := map[string]string{ - ".deb": "dist/grafana_latest_amd64.deb", - ".rpm": "dist/grafana-latest-1.x86_64.rpm", - ".tar.gz": "dist/grafana-latest.linux-x64.tar.gz", + "_amd64.deb": "dist/grafana_latest_amd64.deb", + ".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm", + ".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz", } for _, file := range files { @@ -179,9 +201,9 @@ func readVersionFromPackageJson() { } // add timestamp to iteration - if includeBuildNumber { - if buildNumber != 0 { - linuxPackageIteration = fmt.Sprintf("%d%s", buildNumber, linuxPackageIteration) + if includeBuildId { + if buildId != "0" { + linuxPackageIteration = fmt.Sprintf("%s%s", buildId, linuxPackageIteration) } else { linuxPackageIteration = fmt.Sprintf("%d%s", time.Now().Unix(), linuxPackageIteration) } @@ -210,6 +232,10 @@ type linuxPackageOptions struct { } func createDebPackages() { + previousPkgArch := pkgArch + if pkgArch == "armv7" { + pkgArch = "armhf" + } createPackage(linuxPackageOptions{ packageType: "deb", homeDir: "/usr/share/grafana", @@ -227,9 +253,17 @@ func createDebPackages() { depends: []string{"adduser", "libfontconfig"}, }) + pkgArch = previousPkgArch } func createRpmPackages() { + previousPkgArch := pkgArch + switch { + case pkgArch == "armv7": + pkgArch = "armhfp" + case pkgArch == "arm64": + pkgArch = "aarch64" + } createPackage(linuxPackageOptions{ packageType: "rpm", homeDir: "/usr/share/grafana", @@ -247,6 +281,7 @@ func createRpmPackages() { depends: []string{"/sbin/service", "fontconfig", "freetype", "urw-fonts"}, }) + pkgArch = previousPkgArch } func createLinuxPackages() { @@ -284,19 +319,34 @@ func createPackage(options linuxPackageOptions) { "-s", "dir", "--description", "Grafana", "-C", packageRoot, - "--vendor", "Grafana", "--url", "https://grafana.com", - "--license", "\"Apache 2.0\"", "--maintainer", "contact@grafana.com", "--config-files", options.initdScriptFilePath, "--config-files", options.etcDefaultFilePath, "--config-files", options.systemdServiceFilePath, "--after-install", options.postinstSrc, - "--name", "grafana", + "--version", linuxPackageVersion, "-p", "./dist", } + name := "grafana" + if enterprise { + name += "-enterprise" + args = append(args, "--replaces", "grafana") + } + args = append(args, "--name", name) + + description := "Grafana" + if enterprise { + description += " Enterprise" + } + args = append(args, "--vendor", description) + + if !enterprise { + args = append(args, "--license", "\"Apache 2.0\"") + } + if options.packageType == "rpm" { args = append(args, "--rpm-posttrans", "packaging/rpm/control/posttrans") } @@ -324,20 +374,6 @@ func createPackage(options linuxPackageOptions) { runPrint("fpm", append([]string{"-t", options.packageType}, args...)...) } -func verifyGitRepoIsClean() { - rs, err := runError("git", "ls-files", "--modified") - if err != nil { - log.Fatalf("Failed to check if git tree was clean, %v, %v\n", string(rs), err) - return - } - count := len(string(rs)) - if count > 0 { - log.Fatalf("Git repository has modified files, aborting") - } - - log.Println("Git repository is clean") -} - func ensureGoPath() { if os.Getenv("GOPATH") == "" { cwd, err := os.Getwd() @@ -350,12 +386,8 @@ func ensureGoPath() { } } -func ChangeWorkingDir(dir string) { - os.Chdir(dir) -} - func grunt(params ...string) { - if runtime.GOOS == "windows" { + if runtime.GOOS == windows { runPrint(`.\node_modules\.bin\grunt`, params...) } else { runPrint("./node_modules/.bin/grunt", params...) @@ -364,7 +396,7 @@ func grunt(params ...string) { func gruntBuildArg(task string) []string { args := []string{task} - if includeBuildNumber { + if includeBuildId { args = append(args, fmt.Sprintf("--pkgVer=%v-%v", linuxPackageVersion, linuxPackageIteration)) } else { args = append(args, fmt.Sprintf("--pkgVer=%v", version)) @@ -375,6 +407,12 @@ func gruntBuildArg(task string) []string { if phjsToRelease != "" { args = append(args, fmt.Sprintf("--phjsToRelease=%v", phjsToRelease)) } + if enterprise { + args = append(args, "--enterprise") + } + + args = append(args, fmt.Sprintf("--platform=%v", goos)) + return args } @@ -389,12 +427,19 @@ func test(pkg string) { } func build(binaryName, pkg string, tags []string) { - binary := "./bin/" + binaryName - if goos == "windows" { + binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName) + if isDev { + //don't include os and arch in output path in dev environment + binary = fmt.Sprintf("./bin/%s", binaryName) + } + + if goos == windows { binary += ".exe" } - rmr(binary, binary+".md5") + if !isDev { + rmr(binary, binary+".md5") + } args := []string{"build", "-ldflags", ldflags()} if len(tags) > 0 { args = append(args, "-tags", strings.Join(tags, ",")) @@ -405,16 +450,22 @@ func build(binaryName, pkg string, tags []string) { args = append(args, "-o", binary) args = append(args, pkg) - setBuildEnv() - runPrint("go", "version") + if !isDev { + setBuildEnv() + runPrint("go", "version") + fmt.Printf("Targeting %s/%s\n", goos, goarch) + } + runPrint("go", args...) - // Create an md5 checksum of the binary, to be included in the archive for - // automatic upgrades. - err := md5File(binary) - if err != nil { - log.Fatal(err) + if !isDev { + // Create an md5 checksum of the binary, to be included in the archive for + // automatic upgrades. + err := md5File(binary) + if err != nil { + log.Fatal(err) + } } } @@ -424,6 +475,7 @@ func ldflags() string { b.WriteString(fmt.Sprintf(" -X main.version=%s", version)) b.WriteString(fmt.Sprintf(" -X main.commit=%s", getGitSha())) b.WriteString(fmt.Sprintf(" -X main.buildstamp=%d", buildStamp())) + b.WriteString(fmt.Sprintf(" -X main.buildBranch=%s", getGitBranch())) return b.String() } @@ -435,6 +487,10 @@ func rmr(paths ...string) { } func clean() { + if isDev { + return + } + rmr("dist") rmr("tmp") rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/grafana", goos, goarch))) @@ -442,6 +498,14 @@ func clean() { func setBuildEnv() { os.Setenv("GOOS", goos) + if goos == windows { + // require windows >=7 + os.Setenv("CGO_CFLAGS", "-D_WIN32_WINNT=0x0601") + } + if goarch != "amd64" || goos != linux { + // needed for all other archs + cgo = true + } if strings.HasPrefix(goarch, "armv") { os.Setenv("GOARCH", "arm") os.Setenv("GOARM", goarch[4:]) @@ -451,15 +515,20 @@ func setBuildEnv() { if goarch == "386" { os.Setenv("GO386", "387") } - if cgo != "" { - os.Setenv("CGO_ENABLED", cgo) + if cgo { + os.Setenv("CGO_ENABLED", "1") } if gocc != "" { os.Setenv("CC", gocc) } - if gocxx != "" { - os.Setenv("CXX", gocxx) +} + +func getGitBranch() string { + v, err := runError("git", "rev-parse", "--abbrev-ref", "HEAD") + if err != nil { + return "master" } + return string(v) } func getGitSha() string { @@ -479,24 +548,6 @@ func buildStamp() int64 { return s } -func buildArch() string { - os := goos - if os == "darwin" { - os = "macosx" - } - return fmt.Sprintf("%s-%s", os, goarch) -} - -func run(cmd string, args ...string) []byte { - bs, err := runError(cmd, args...) - if err != nil { - log.Println(cmd, strings.Join(args, " ")) - log.Println(string(bs)) - log.Fatal(err) - } - return bytes.TrimSpace(bs) -} - func runError(cmd string, args ...string) ([]byte, error) { ecmd := exec.Command(cmd, args...) bs, err := ecmd.CombinedOutput() @@ -550,7 +601,7 @@ func shaFilesInDist() { return nil } - if strings.Contains(path, ".sha256") == false { + if !strings.Contains(path, ".sha256") { err := shaFile(path) if err != nil { log.Printf("Failed to create sha file. error: %v\n", err) @@ -585,3 +636,11 @@ func shaFile(file string) error { return out.Close() } + +func shortenBuildId(buildId string) string { + buildId = strings.Replace(buildId, "-", "", -1) + if len(buildId) < 9 { + return buildId + } + return buildId[0:8] +} diff --git a/circle.yml b/circle.yml deleted file mode 100644 index cfa8b762e49..00000000000 --- a/circle.yml +++ /dev/null @@ -1,135 +0,0 @@ -version: 2 - -jobs: - test-frontend: - docker: - - image: circleci/node:6.11.4 - steps: - - checkout - - run: - name: install yarn - command: 'sudo npm install -g yarn --quiet' - - restore_cache: - key: dependency-cache-{{ checksum "yarn.lock" }} - # Could we skip this step if the cache has been restored? `[ -d node_modules ] || yarn install ...` should be able to apply to build step as well - - run: - name: yarn install - command: 'yarn install --pure-lockfile --no-progress' - - save_cache: - key: dependency-cache-{{ checksum "yarn.lock" }} - paths: - - node_modules - - run: - name: frontend tests - command: './scripts/circle-test-frontend.sh' - - test-backend: - docker: - - image: circleci/golang:1.10 - working_directory: /go/src/github.com/grafana/grafana - steps: - - checkout - - run: - name: build backend and run go tests - command: './scripts/circle-test-backend.sh' - - build: - docker: - - image: grafana/build-container:v0.1 - working_directory: /go/src/github.com/grafana/grafana - steps: - - checkout - - run: - name: build and package grafana - command: './scripts/build/build.sh' - - run: - name: sign packages - command: './scripts/build/sign_packages.sh' - - run: - name: sha-sum packages - command: 'go run build.go sha-dist' - - run: - name: Build Grafana.com publisher - command: 'go build -o scripts/publish scripts/build/publish.go' - - persist_to_workspace: - root: . - paths: - - dist/grafana* - - scripts/*.sh - - scripts/publish - - deploy-master: - docker: - - image: circleci/python:2.7-stretch - steps: - - attach_workspace: - at: . - - run: - name: install awscli - command: 'sudo pip install awscli' - - run: - name: deploy to s3 - command: 'aws s3 sync ./dist s3://$BUCKET_NAME/master' - - run: - name: Trigger Windows build - command: './scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} master' - - run: - name: Trigger Docker build - command: './scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN}' - - run: - name: Publish to Grafana.com - command: './scripts/publish -apiKey ${GRAFANA_COM_API_KEY}' - - deploy-release: - docker: - - image: circleci/python:2.7-stretch - steps: - - attach_workspace: - at: dist - - run: - name: install awscli - command: 'sudo pip install awscli' - - run: - name: deploy to s3 - command: 'aws s3 sync ./dist s3://$BUCKET_NAME/release' - - run: - name: Trigger Windows build - command: './scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} release' - - run: - name: Trigger Docker build - command: './scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN} ${CIRCLE_TAG}' - -workflows: - version: 2 - test-and-build: - jobs: - - build: - filters: - tags: - only: /.*/ - - test-frontend: - filters: - tags: - only: /.*/ - - test-backend: - filters: - tags: - only: /.*/ - - deploy-master: - requires: - - test-backend - - test-frontend - - build - filters: - branches: - only: master - - deploy-release: - requires: - - test-backend - - test-frontend - - build - filters: - branches: - ignore: /.*/ - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 82a86e0232b..00000000000 --- a/codecov.yml +++ /dev/null @@ -1,13 +0,0 @@ -coverage: - precision: 2 - round: down - range: "50...100" - - status: - project: yes - patch: yes - changes: no - -comment: - layout: "diff" - behavior: "once" diff --git a/conf/defaults.ini b/conf/defaults.ini index 4a2240f1924..679a6a88eb7 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -14,6 +14,9 @@ instance_name = ${HOSTNAME} # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) data = data +# Temporary files in `data` directory older than given duration will be removed +temp_data_lifetime = 24h + # Directory where grafana can store logs logs = data/log @@ -82,6 +85,9 @@ max_idle_conn = 2 # Max conn setting default is 0 (mean not set) max_open_conn = +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +conn_max_lifetime = 14400 + # Set to true to log the sql calls and execution times. log_queries = @@ -125,6 +131,9 @@ cookie_secure = false session_life_time = 86400 gc_interval_time = 86400 +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +conn_max_lifetime = 14400 + #################################### Data proxy ########################### [dataproxy] @@ -204,6 +213,9 @@ allow_org_create = false # Set to true to automatically assign new users to the default organization (id 1) auto_assign_org = true +# Set this value to automatically add new users to the provided organization (if auto_assign_org above is set to true) +auto_assign_org_id = 1 + # Default role new users will be automatically assigned (if auto_assign_org above is set to true) auto_assign_org_role = Viewer @@ -231,6 +243,9 @@ disable_login_form = false # Set to true to disable the signout link in the side menu. useful if you use auth.proxy disable_signout_menu = false +# URL to redirect the user to after sign out +signout_redirect_url = + #################################### Anonymous Auth ###################### [auth.anonymous] # enable anonymous access @@ -255,6 +270,18 @@ api_url = https://api.github.com/user team_ids = allowed_organizations = +#################################### GitLab Auth ######################### +[auth.gitlab] +enabled = false +allow_sign_up = true +client_id = some_id +client_secret = some_secret +scopes = api +auth_url = https://gitlab.com/oauth/authorize +token_url = https://gitlab.com/oauth/token +api_url = https://gitlab.com/api/v4 +allowed_groups = + #################################### Google Auth ######################### [auth.google] enabled = false @@ -294,11 +321,16 @@ allow_sign_up = true client_id = some_id client_secret = some_secret scopes = user:email +email_attribute_name = email:primary auth_url = token_url = api_url = team_ids = allowed_organizations = +tls_skip_verify_insecure = false +tls_client_cert = +tls_client_key = +tls_client_ca = #################################### Basic Auth ########################## [auth.basic] @@ -312,6 +344,7 @@ header_property = username auto_sign_up = true ldap_sync_ttl = 60 whitelist = +headers = #################################### Auth LDAP ########################### [auth.ldap] @@ -436,6 +469,21 @@ enabled = true # Makes it possible to turn off alert rule execution but alerting UI is visible execute_alerts = true +# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state) +error_or_timeout = alerting + +# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) +nodata_or_nullvalues = no_data + +# Alert notifications can include images, but rendering many images at the same time can overload the server +# This limit will protect the server from render overloading and make sure notifications are sent out quickly +concurrent_render_limit = 5 + +#################################### Explore ############################# +[explore] +# Enable the Explore section +enabled = false + #################################### Internal Grafana Metrics ############ # Metrics available at HTTP API Url /metrics [metrics] @@ -502,3 +550,15 @@ container_name = [external_image_storage.local] # does not require any configuration + +[rendering] +# Options to configure external image rendering server like https://github.com/grafana/grafana-image-renderer +server_url = +callback_url = + +[panels] +enable_alpha = false + +[enterprise] +license_path = + diff --git a/conf/ldap.toml b/conf/ldap.toml index 166d85eabb1..b684f2556d5 100644 --- a/conf/ldap.toml +++ b/conf/ldap.toml @@ -15,6 +15,9 @@ start_tls = false ssl_skip_verify = false # set to the path to your root CA certificate or leave unset to use system defaults # root_ca_cert = "/path/to/certificate.crt" +# Authentication against LDAP servers requiring client certificates +# client_cert = "/path/to/client.crt" +# client_key = "/path/to/client.key" # Search user bind dn bind_dn = "cn=admin,dc=grafana,dc=org" @@ -28,37 +31,11 @@ search_filter = "(cn=%s)" # An array of base dns to search through search_base_dns = ["dc=grafana,dc=org"] -# In POSIX LDAP schemas, without memberOf attribute a secondary query must be made for groups. -# This is done by enabling group_search_filter below. You must also set member_of= "cn" -# in [servers.attributes] below. - -# Users with nested/recursive group membership and an LDAP server that supports LDAP_MATCHING_RULE_IN_CHAIN -# can set group_search_filter, group_search_filter_user_attribute, group_search_base_dns and member_of -# below in such a way that the user's recursive group membership is considered. -# -# Nested Groups + Active Directory (AD) Example: -# -# AD groups store the Distinguished Names (DNs) of members, so your filter must -# recursively search your groups for the authenticating user's DN. For example: -# -# group_search_filter = "(member:1.2.840.113556.1.4.1941:=%s)" -# group_search_filter_user_attribute = "distinguishedName" -# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] -# -# [servers.attributes] -# ... -# member_of = "distinguishedName" - -## Group search filter, to retrieve the groups of which the user is a member (only set if memberOf attribute is not available) +## For Posix or LDAP setups that does not support member_of attribute you can define the below settings +## Please check grafana LDAP docs for examples # group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" -## Group search filter user attribute defines what user attribute gets substituted for %s in group_search_filter. -## Defaults to the value of username in [server.attributes] -## Valid options are any of your values in [servers.attributes] -## If you are using nested groups you probably want to set this and member_of in -## [servers.attributes] to "distinguishedName" -# group_search_filter_user_attribute = "distinguishedName" -## An array of the base DNs to search through for groups. Typically uses ou=groups # group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] +# group_search_filter_user_attribute = "uid" # Specify names of the ldap attributes your ldap uses [servers.attributes] @@ -72,6 +49,8 @@ email = "email" [[servers.group_mappings]] group_dn = "cn=admins,dc=grafana,dc=org" org_role = "Admin" +# To make user an instance admin (Grafana Admin) uncomment line below +# grafana_admin = true # The Grafana organization database id, optional, if left out the default org (id 1) will be used # org_id = 1 diff --git a/conf/provisioning/datasources/sample.yaml b/conf/provisioning/datasources/sample.yaml index 877e229183d..37487dc4b3b 100644 --- a/conf/provisioning/datasources/sample.yaml +++ b/conf/provisioning/datasources/sample.yaml @@ -40,11 +40,14 @@ apiVersion: 1 # graphiteVersion: "1.1" # tlsAuth: true # tlsAuthWithCACert: true +# httpHeaderName1: "Authorization" # # json object of data that will be encrypted. # secureJsonData: # tlsCACert: "..." # tlsClientCert: "..." # tlsClientKey: "..." +# # +# httpHeaderValue1: "Bearer xf5yhfkpsnmgo" # version: 1 # # allow users to edit datasources from the UI. # editable: false diff --git a/conf/sample.ini b/conf/sample.ini index 34b28ccf3fe..c6b716a731d 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -14,6 +14,9 @@ # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) ;data = /var/lib/grafana +# Temporary files in `data` directory older than given duration will be removed +;temp_data_lifetime = 24h + # Directory where grafana can store logs ;logs = /var/log/grafana @@ -64,7 +67,7 @@ #################################### Database #################################### [database] # You can configure the database connection by specifying type, host, name, user and password -# as seperate properties or as on string using the url propertie. +# as separate properties or as on string using the url properties. # Either "mysql", "postgres" or "sqlite3", it's your choice ;type = sqlite3 @@ -90,6 +93,9 @@ # Max conn setting default is 0 (mean not set) ;max_open_conn = +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + # Set to true to log the sql calls and execution times. log_queries = @@ -214,6 +220,9 @@ log_queries = # Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false ;disable_signout_menu = false +# URL to redirect the user to after sign out +;signout_redirect_url = + #################################### Anonymous Auth ########################## [auth.anonymous] # enable anonymous access @@ -263,6 +272,10 @@ log_queries = ;api_url = https://foo.bar/user ;team_ids = ;allowed_organizations = +;tls_skip_verify_insecure = false +;tls_client_cert = +;tls_client_key = +;tls_client_ca = #################################### Grafana.com Auth #################### [auth.grafana_com] @@ -281,6 +294,7 @@ log_queries = ;auto_sign_up = true ;ldap_sync_ttl = 60 ;whitelist = 192.168.1.1, 192.168.2.1 +;headers = Email:X-User-Email, Name:X-User-Name #################################### Basic Auth ########################## [auth.basic] @@ -374,6 +388,21 @@ log_queries = # Makes it possible to turn off alert rule execution but alerting UI is visible ;execute_alerts = true +# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state) +;error_or_timeout = alerting + +# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) +;nodata_or_nullvalues = no_data + +# Alert notifications can include images, but rendering many images at the same time can overload the server +# This limit will protect the server from render overloading and make sure notifications are sent out quickly +;concurrent_render_limit = 5 + +#################################### Explore ############################# +[explore] +# Enable the Explore section +;enabled = false + #################################### Internal Grafana Metrics ########################## # Metrics available at HTTP API Url /metrics [metrics] @@ -407,7 +436,7 @@ log_queries = ;sampler_param = 1 #################################### Grafana.com integration ########################## -# Url used to to import dashboards directly from Grafana.com +# Url used to import dashboards directly from Grafana.com [grafana_com] ;url = https://grafana.com @@ -442,3 +471,13 @@ log_queries = [external_image_storage.local] # does not require any configuration + +[rendering] +# Options to configure external image rendering server like https://github.com/grafana/grafana-image-renderer +;server_url = +;callback_url = + +[enterprise] +# Path to a valid Grafana Enterprise license.jwt file +;license_path = + diff --git a/devenv/README.md b/devenv/README.md new file mode 100644 index 00000000000..9abf3596776 --- /dev/null +++ b/devenv/README.md @@ -0,0 +1,16 @@ +This folder contains useful scripts and configuration for... + +* Configuring dev datasources in Grafana +* Configuring dev & test scenarios dashboards. + +```bash +./setup.sh +``` + +After restarting grafana server there should now be a number of datasources named `gdev-` provisioned as well as a dashboard folder named `gdev dashboards`. This folder contains dashboard & panel features tests dashboards. + +# Dev dashboards + +Please update these dashboards or make new ones as new panels & dashboards features are developed or new bugs are found. The dashboards are located in the `devenv/dev-dashboards` folder. + + diff --git a/scripts/benchmarks/ab/ab_test.sh b/devenv/benchmarks/ab/ab_test.sh similarity index 100% rename from scripts/benchmarks/ab/ab_test.sh rename to devenv/benchmarks/ab/ab_test.sh diff --git a/devenv/bulk-dashboards/bulk-dashboards.yaml b/devenv/bulk-dashboards/bulk-dashboards.yaml new file mode 100644 index 00000000000..65557901f42 --- /dev/null +++ b/devenv/bulk-dashboards/bulk-dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: 'Bulk dashboards' + folder: 'Bulk dashboards' + type: file + options: + path: devenv/bulk-dashboards + diff --git a/devenv/bulk-dashboards/bulkdash.jsonnet b/devenv/bulk-dashboards/bulkdash.jsonnet new file mode 100644 index 00000000000..4c82fd36f69 --- /dev/null +++ b/devenv/bulk-dashboards/bulkdash.jsonnet @@ -0,0 +1,1140 @@ +{ + "annotations": { + "enable": false, + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "aliasColors": { + "cpu": "#E24D42", + "memory": "#1f78c1", + "statsd.fakesite.counters.session_start.desktop.count": "#6ED0E0" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 3, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 4, + "interactive": true, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": true, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [ + { + "alias": "cpu", + "fill": 0, + "lines": true, + "yaxis": 2, + "zindex": 2 + }, + { + "alias": "memory", + "pointradius": 2, + "points": true + } + ], + "spaceLength": 10, + "spyable": true, + "stack": false, + "steppedLine": false, + "targets": [ + { + "hide": false, + "refId": "A", + "target": "alias(movingAverage(scaleToSeconds(apps.fakesite.web_server_01.counters.request_status.code_302.count, 10), 20), 'cpu')" + }, + { + "refId": "B", + "target": "alias(statsd.fakesite.counters.session_start.desktop.count, 'memory')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "Memory / CPU", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "aliasColors": { + "logins": "#5195ce", + "logins (-1 day)": "#447EBC", + "logins (-1 hour)": "#705da0" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 1, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 3, + "interactive": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "alias(movingAverage(scaleToSeconds(apps.fakesite.web_server_01.counters.requests.count, 1), 2), 'logins')" + }, + { + "refId": "B", + "target": "alias(movingAverage(timeShift(scaleToSeconds(apps.fakesite.web_server_01.counters.requests.count, 1), '1h'), 2), 'logins (-1 hour)')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": "1h", + "timezone": "browser", + "title": "logins", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.4)" + } + ], + "thresholds": "200,270", + "title": "Memory", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "apps.backend.backend_02.counters.requests.count" + } + ], + "thresholds": "100,270", + "title": "Sign ups", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 17, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "apps.backend.backend_04.counters.requests.count" + } + ], + "thresholds": "100,270", + "title": "Sign outs", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.7)" + } + ], + "thresholds": "100,270", + "title": "Logins", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "web_server_01": "#badff4", + "web_server_02": "#5195ce", + "web_server_03": "#1f78c1", + "web_server_04": "#0a437c" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 6, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 11, + "w": 16, + "x": 0, + "y": 7 + }, + "id": 2, + "interactive": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(movingAverage(scaleToSeconds(apps.fakesite.*.counters.requests.count, 1), 2), 2)" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "server requests", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 7 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.8)" + } + ], + "thresholds": "200,270", + "title": "Logouts", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_03.counters.requests.count, 0.3)" + } + ], + "thresholds": "100,270", + "title": "Support calls", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 12 + }, + "id": 26, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.2)" + } + ], + "thresholds": "200,270", + "title": "Google hits", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 12 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.2)" + } + ], + "thresholds": "200,270", + "title": "Google hits", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "upper_25": "#F9E2D2", + "upper_50": "#F2C96D", + "upper_75": "#EAB839" + }, + "annotate": { + "enable": false + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 1, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 5, + "interactive": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "legend_counts": true, + "lines": false, + "linewidth": 2, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(summarize(statsd.fakesite.timers.ads_timer.*, '4min', 'avg'), 4)" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "client side full page load", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "demo" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "2h", + " 6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Big Dashboard", + "uid": "000000003", + "version": 16 +} diff --git a/devenv/bulk_alerting_dashboards/bulk_alerting_dashboards.yaml b/devenv/bulk_alerting_dashboards/bulk_alerting_dashboards.yaml new file mode 100644 index 00000000000..1ede5dcd30a --- /dev/null +++ b/devenv/bulk_alerting_dashboards/bulk_alerting_dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: 'Bulk alerting dashboards' + folder: 'Bulk alerting dashboards' + type: file + options: + path: devenv/bulk_alerting_dashboards + diff --git a/devenv/bulk_alerting_dashboards/bulkdash_alerting.jsonnet b/devenv/bulk_alerting_dashboards/bulkdash_alerting.jsonnet new file mode 100644 index 00000000000..a7acd57745d --- /dev/null +++ b/devenv/bulk_alerting_dashboards/bulkdash_alerting.jsonnet @@ -0,0 +1,168 @@ +{ + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 65 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "10s", + "handler": 1, + "name": "bulk alerting", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "$$hashKey": "object:117", + "expr": "go_goroutines", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 50 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "New dashboard", + "uid": null, + "version": 0 +} \ No newline at end of file diff --git a/docker/create_docker_compose.sh b/devenv/create_docker_compose.sh similarity index 94% rename from docker/create_docker_compose.sh rename to devenv/create_docker_compose.sh index 9d28ede8e7e..5da9e8f5c8f 100755 --- a/docker/create_docker_compose.sh +++ b/devenv/create_docker_compose.sh @@ -1,13 +1,13 @@ #!/bin/bash -blocks_dir=blocks +blocks_dir=docker/blocks docker_dir=docker template_dir=templates grafana_config_file=conf.tmp grafana_config=config -compose_header_file=compose_header.yml +compose_header_file=docker/compose_header.yml fig_file=docker-compose.yaml fig_config=docker-compose.yaml diff --git a/devenv/dashboards.yaml b/devenv/dashboards.yaml new file mode 100644 index 00000000000..226c1a8b335 --- /dev/null +++ b/devenv/dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: 'gdev dashboards' + folder: 'gdev dashboards' + type: file + options: + path: devenv/dev-dashboards + diff --git a/devenv/datasources.yaml b/devenv/datasources.yaml new file mode 100644 index 00000000000..a4e9bf05641 --- /dev/null +++ b/devenv/datasources.yaml @@ -0,0 +1,104 @@ +apiVersion: 1 + +datasources: + - name: gdev-graphite + type: graphite + access: proxy + url: http://localhost:8080 + jsonData: + graphiteVersion: "1.1" + + - name: gdev-prometheus + type: prometheus + access: proxy + isDefault: true + url: http://localhost:9090 + + - name: gdev-testdata + type: testdata + + - name: gdev-influxdb + type: influxdb + access: proxy + database: site + user: grafana + password: grafana + url: http://localhost:8086 + jsonData: + timeInterval: "15s" + + - name: gdev-opentsdb + type: opentsdb + access: proxy + url: http://localhost:4242 + jsonData: + tsdbResolution: 1 + tsdbVersion: 1 + + - name: gdev-elasticsearch-metrics + type: elasticsearch + access: proxy + database: "[metrics-]YYYY.MM.DD" + url: http://localhost:9200 + jsonData: + interval: Daily + timeField: "@timestamp" + + - name: gdev-mysql + type: mysql + url: localhost:3306 + database: grafana + user: grafana + password: password + + - name: gdev-mysql-ds-tests + type: mysql + url: localhost:3306 + database: grafana_ds_tests + user: grafana + password: password + + - name: gdev-mssql + type: mssql + url: localhost:1433 + database: grafana + user: grafana + secureJsonData: + password: Password! + + - name: gdev-mssql-ds-tests + type: mssql + url: localhost:1433 + database: grafanatest + user: grafana + secureJsonData: + password: Password! + + - name: gdev-postgres + type: postgres + url: localhost:5432 + database: grafana + user: grafana + secureJsonData: + password: password + jsonData: + sslmode: "disable" + + - name: gdev-postgres-ds-tests + type: postgres + url: localhost:5432 + database: grafanadstest + user: grafanatest + secureJsonData: + password: grafanatest + jsonData: + sslmode: "disable" + + - name: gdev-cloudwatch + type: cloudwatch + editable: true + jsonData: + authType: credentials + defaultRegion: eu-west-2 + + diff --git a/docker/blocks/mssql/dashboard.json b/devenv/dev-dashboards/datasource_tests_mssql_fakedata.json similarity index 92% rename from docker/blocks/mssql/dashboard.json rename to devenv/dev-dashboards/datasource_tests_mssql_fakedata.json index ce9aa141a75..e810a686134 100644 --- a/docker/blocks/mssql/dashboard.json +++ b/devenv/dev-dashboards/datasource_tests_mssql_fakedata.json @@ -1,40 +1,4 @@ { - "__inputs": [ - { - "name": "DS_MSSQL", - "label": "MSSQL", - "description": "", - "type": "datasource", - "pluginId": "mssql", - "pluginName": "MSSQL" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "mssql", - "name": "MSSQL", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "5.0.0" - } - ], "annotations": { "list": [ { @@ -52,8 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": null, - "iteration": 1520976748896, + "iteration": 1532618661457, "links": [], "panels": [ { @@ -63,7 +26,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL}", + "datasource": "gdev-mssql", "fill": 2, "gridPos": { "h": 9, @@ -149,14 +112,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL}", + "datasource": "gdev-mssql", "fill": 2, "gridPos": { "h": 18, @@ -234,14 +201,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL}", + "datasource": "gdev-mssql", "fill": 2, "gridPos": { "h": 9, @@ -313,11 +284,15 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "columns": [], - "datasource": "${DS_MSSQL}", + "datasource": "gdev-mssql", "fontSize": "100%", "gridPos": { "h": 10, @@ -371,13 +346,13 @@ ], "schemaVersion": 16, "style": "dark", - "tags": [], + "tags": ["gdev", "mssql", "fake-data-gen"], "templating": { "list": [ { "allValue": null, "current": {}, - "datasource": "${DS_MSSQL}", + "datasource": "gdev-mssql", "hide": 0, "includeAll": false, "label": "Datacenter", @@ -387,6 +362,7 @@ "query": "SELECT DISTINCT datacenter FROM grafana_metric", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], @@ -397,7 +373,7 @@ { "allValue": null, "current": {}, - "datasource": "${DS_MSSQL}", + "datasource": "gdev-mssql", "hide": 0, "includeAll": true, "label": "Hostname", @@ -407,6 +383,7 @@ "query": "SELECT DISTINCT hostname FROM grafana_metric WHERE datacenter='$datacenter'", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], @@ -499,6 +476,7 @@ ], "query": "1s,10s,30s,1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "refresh": 2, + "skipUrlSync": false, "type": "interval" } ] @@ -533,7 +511,7 @@ ] }, "timezone": "", - "title": "Grafana Fake Data Gen - MSSQL", + "title": "Datasource tests - MSSQL", "uid": "86Js1xRmk", - "version": 11 + "version": 1 } \ No newline at end of file diff --git a/devenv/dev-dashboards/datasource_tests_mssql_unittest.json b/devenv/dev-dashboards/datasource_tests_mssql_unittest.json new file mode 100644 index 00000000000..b2d757ae188 --- /dev/null +++ b/devenv/dev-dashboards/datasource_tests_mssql_unittest.json @@ -0,0 +1,2902 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "gdev-mssql-ds-tests", + "enable": false, + "hide": false, + "iconColor": "#6ed0e0", + "limit": 100, + "name": "Deploys", + "rawQuery": "SELECT\n $__time(time_sec),\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='deploy'\n ORDER BY 1 ASC\n ", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "gdev-mssql-ds-tests", + "enable": false, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Tickets", + "rawQuery": "SELECT\n $__time(time_sec),\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='ticket'\n ORDER BY 1 ASC\n ", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "gdev-mssql-ds-tests", + "enable": false, + "hide": false, + "iconColor": "#7eb26d", + "limit": 100, + "name": "Metric Values timeEpoch macro", + "rawQuery": "SELECT \n $__timeEpoch(time), \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "gdev-mssql-ds-tests", + "enable": false, + "hide": false, + "iconColor": "#1f78c1", + "limit": 100, + "name": "Metric Values native time", + "rawQuery": "SELECT \n time, \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", + "showIn": 0, + "tags": [], + "type": "tags" + } + ] + }, + "description": "Run the mssql unit tests to generate the data backing this dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534507501976, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "gdev-mssql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT * from mssql_types", + "refId": "A" + } + ], + "title": "Data types", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-mssql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 32, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as bigint) as time", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as bigint) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-mssql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 33, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as datetime) as time", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as datetime) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-mssql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 34, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT GETDATE() as time", + "refId": "A", + "target": "" + } + ], + "title": "GETDATE() as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-mssql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 35, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT GETUTCDATE() as time", + "refId": "A", + "target": "" + } + ], + "title": "GETUTCDATE() as time", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m'), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '5m') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m without fill", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', NULL), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '5m') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m with fill(NULL) and null as zero", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', 10.0), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '5m') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m with fill(10.0)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', previous), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '5m') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m with fill(previous) and null as zero", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize'), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '$summarize') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize without fill", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', NULL), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '$summarize') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize with fill(NULL)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', 100.0), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '$summarize') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize with fill(100.0)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', previous), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '$summarize') ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize with fill(previous)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__timeGroupAlias(time, '$summarize'), \n measurement as metric, \n avg(valueOne) as valueOne,\n avg(valueTwo) as valueTwo\nFROM\n metric_values \nWHERE\n $__timeFilter(time) AND\n ($metric = 'ALL' OR measurement = $metric)\nGROUP BY \n $__timeGroup(time, '$summarize'), \n measurement \nORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column using timeGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "MovingAverageValueOne", + "dashes": true, + "lines": false + }, + { + "alias": "MovingAverageValueTwo", + "dashes": true, + "lines": false, + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__timeGroupAlias(time, '$summarize'), \n avg(valueOne) as valueOne, \n avg(valueTwo) as valueTwo \nFROM\n metric_values \nWHERE \n $__timeFilter(time) AND \n ($metric = 'ALL' OR measurement = $metric)\nGROUP BY \n $__timeGroup(time, '$summarize')\nORDER BY 1", + "refId": "A" + }, + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n time,\n avg(valueOne) OVER (ORDER BY time ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as MovingAverageValueOne,\n avg(valueTwo) OVER (ORDER BY time ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as MovingAverageValueTwo\nFROM\n metric_values \nWHERE \n $__timeFilter(time) AND \n ($metric = 'ALL' OR measurement = $metric)\nORDER BY 1", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column using timeGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__unixEpochGroupAlias(timeInt32, '$summarize'), \n measurement as metric, \n avg(valueOne) as valueOne,\n avg(valueTwo) as valueTwo\nFROM\n metric_values \nWHERE\n $__unixEpochFilter(timeInt32) AND\n ($metric = 'ALL' OR measurement = $metric)\nGROUP BY \n $__unixEpochGroup(timeInt32, '$summarize'), \n measurement \nORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column using unixEpochGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "MovingAverageValueOne", + "dashes": true, + "lines": false + }, + { + "alias": "MovingAverageValueTwo", + "dashes": true, + "lines": false, + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__unixEpochGroupAlias(timeInt32, '$summarize'), \n avg(valueOne) as valueOne,\n avg(valueTwo) as valueTwo\nFROM\n metric_values \nWHERE\n $__unixEpochFilter(timeInt32) AND\n ($metric = 'ALL' OR measurement = $metric)\nGROUP BY \n $__unixEpochGroup(timeInt32, '$summarize')\nORDER BY 1", + "refId": "A" + }, + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n time,\n avg(valueOne) OVER (ORDER BY time ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as MovingAverageValueOne,\n avg(valueTwo) OVER (ORDER BY time ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as MovingAverageValueTwo\nFROM\n metric_values \nWHERE \n $__timeFilter(time) AND \n ($metric = 'ALL' OR measurement = $metric)\nORDER BY 1", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column using unixEpochGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "DECLARE\n @from int = $__unixEpochFrom(), \n @to int = $__unixEpochTo(), \n @interval nvarchar(50) = '$summarize', \n @metric nvarchar(200) = $metric\n \nEXEC dbo.sp_test_epoch @from, @to, @interval, @metric", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Stored procedure support using epoch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "DECLARE\n @from datetime = $__timeFrom(), \n @to datetime = $__timeTo(), \n @interval nvarchar(50) = '$summarize', \n @metric nvarchar(200) = $metric\n \nEXEC dbo.sp_test_datetime @from, @to, @interval, @metric", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Stored procedure support using datetime", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - series mode", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - series mode", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - histogram", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 50, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - histogram", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 100, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - histogram stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - histogram stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - histogram stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mssql-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - histogram stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "gdev", + "mssql" + ], + "templating": { + "list": [ + { + "allValue": "'ALL'", + "current": { + "selected": true, + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "gdev-mssql-ds-tests", + "hide": 0, + "includeAll": true, + "label": "Metric", + "multi": false, + "name": "metric", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "Metric A", + "value": "Metric A" + }, + { + "selected": false, + "text": "Metric B", + "value": "Metric B" + } + ], + "query": "SELECT DISTINCT measurement FROM metric_values", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": "Interval", + "name": "summarize", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + } + ], + "query": "1s,10s,30s,1m,5m,10m", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "2018-03-15T12:30:00.000Z", + "to": "2018-03-15T13:55:01.000Z" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Datasource tests - MSSQL (unit test)", + "uid": "GlAqcPgmz", + "version": 2 +} \ No newline at end of file diff --git a/docker/blocks/mysql/dashboard.json b/devenv/dev-dashboards/datasource_tests_mysql_fakedata.json similarity index 90% rename from docker/blocks/mysql/dashboard.json rename to devenv/dev-dashboards/datasource_tests_mysql_fakedata.json index e2b791f82e6..ebeb452fc4c 100644 --- a/docker/blocks/mysql/dashboard.json +++ b/devenv/dev-dashboards/datasource_tests_mysql_fakedata.json @@ -1,40 +1,4 @@ { - "__inputs": [ - { - "name": "DS_MYSQL", - "label": "Mysql", - "description": "", - "type": "datasource", - "pluginId": "mysql", - "pluginName": "MySQL" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "mysql", - "name": "MySQL", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], "annotations": { "list": [ { @@ -52,8 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": null, - "iteration": 1518602729468, + "iteration": 1532620738041, "links": [], "panels": [ { @@ -63,7 +26,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MYSQL}", + "datasource": "gdev-mysql", "fill": 2, "gridPos": { "h": 9, @@ -118,7 +81,7 @@ ], "thresholds": [], "timeFrom": null, - "timeShift": "1h", + "timeShift": null, "title": "Average logins / $summarize", "tooltip": { "shared": true, @@ -150,14 +113,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MYSQL}", + "datasource": "gdev-mysql", "fill": 2, "gridPos": { "h": 18, @@ -204,7 +171,7 @@ ], "thresholds": [], "timeFrom": null, - "timeShift": "1h", + "timeShift": null, "title": "Average payments started/ended / $summarize", "tooltip": { "shared": true, @@ -236,14 +203,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MYSQL}", + "datasource": "gdev-mysql", "fill": 2, "gridPos": { "h": 9, @@ -284,7 +255,7 @@ ], "thresholds": [], "timeFrom": null, - "timeShift": "1h", + "timeShift": null, "title": "Max CPU / $summarize", "tooltip": { "shared": true, @@ -316,11 +287,15 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "columns": [], - "datasource": "${DS_MYSQL}", + "datasource": "gdev-mysql", "fontSize": "100%", "gridPos": { "h": 9, @@ -369,7 +344,7 @@ "target": "" } ], - "timeShift": "1h", + "timeShift": null, "title": "Values", "transform": "table", "type": "table" @@ -378,6 +353,7 @@ "schemaVersion": 16, "style": "dark", "tags": [ + "gdev", "fake-data-gen", "mysql" ], @@ -385,8 +361,11 @@ "list": [ { "allValue": null, - "current": {}, - "datasource": "${DS_MYSQL}", + "current": { + "text": "America", + "value": "America" + }, + "datasource": "gdev-mysql", "hide": 0, "includeAll": false, "label": "Datacenter", @@ -396,6 +375,7 @@ "query": "SELECT DISTINCT datacenter FROM grafana_metric", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], @@ -405,8 +385,11 @@ }, { "allValue": null, - "current": {}, - "datasource": "${DS_MYSQL}", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "gdev-mysql", "hide": 0, "includeAll": true, "label": "Hostname", @@ -416,6 +399,7 @@ "query": "SELECT DISTINCT hostname FROM grafana_metric WHERE datacenter='$datacenter'", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], @@ -428,7 +412,6 @@ "auto_count": 5, "auto_min": "10s", "current": { - "selected": true, "text": "1m", "value": "1m" }, @@ -509,6 +492,7 @@ ], "query": "1s,10s,30s,1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "refresh": 2, + "skipUrlSync": false, "type": "interval" } ] @@ -543,7 +527,7 @@ ] }, "timezone": "", - "title": "Grafana Fake Data Gen - MySQL", + "title": "Datasource tests - MySQL", "uid": "DGsCac3kz", - "version": 6 + "version": 8 } \ No newline at end of file diff --git a/docker/blocks/mssql_tests/dashboard.json b/devenv/dev-dashboards/datasource_tests_mysql_unittest.json similarity index 70% rename from docker/blocks/mssql_tests/dashboard.json rename to devenv/dev-dashboards/datasource_tests_mysql_unittest.json index 323a61bb49a..0255f3c0c91 100644 --- a/docker/blocks/mssql_tests/dashboard.json +++ b/devenv/dev-dashboards/datasource_tests_mysql_unittest.json @@ -1,40 +1,4 @@ { - "__inputs": [ - { - "name": "DS_MSSQL_TEST", - "label": "MSSQL Test", - "description": "", - "type": "datasource", - "pluginId": "mssql", - "pluginName": "Microsoft SQL Server" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "mssql", - "name": "Microsoft SQL Server", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "5.0.0" - } - ], "annotations": { "list": [ { @@ -47,53 +11,65 @@ "type": "dashboard" }, { - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "enable": false, "hide": false, "iconColor": "#6ed0e0", "limit": 100, "name": "Deploys", - "rawQuery": "SELECT\n time_sec as time,\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='deploy'\n ORDER BY 1 ASC\n ", + "rawQuery": "SELECT\n time_sec,\n description as text,\n tags\n FROM event\n WHERE $__unixEpochFilter(time_sec) AND tags='deploy'\n ORDER BY 1 ASC\n ", "showIn": 0, "tags": [], "type": "tags" }, { - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "enable": false, "hide": false, "iconColor": "rgba(255, 96, 96, 1)", "limit": 100, "name": "Tickets", - "rawQuery": "SELECT\n time_sec as time,\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='ticket'\n ORDER BY 1 ASC\n ", + "rawQuery": "SELECT\n time_sec as time,\n description as text,\n tags\n FROM event\n WHERE $__unixEpochFilter(time_sec) AND tags='ticket'\n ORDER BY 1 ASC\n ", "showIn": 0, "tags": [], "type": "tags" }, { - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "enable": false, "hide": false, "iconColor": "#7eb26d", "limit": 100, - "name": "Metric Values", - "rawQuery": "SELECT \n time, \n measurement as text, \n '' as tags\nFROM\n metric_values \nORDER BY 1", + "name": "Metric Values timeEpoch macro", + "rawQuery": "SELECT \n $__timeEpoch(time), \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "gdev-mysql-ds-tests", + "enable": false, + "hide": false, + "iconColor": "#1f78c1", + "limit": 100, + "name": "Metric Values native time", + "rawQuery": "SELECT \n time, \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", "showIn": 0, "tags": [], "type": "tags" } ] }, + "description": "Run the mysql unit tests to generate the data backing this dashboard", "editable": true, "gnetId": null, "graphTooltip": 0, - "id": null, - "iteration": 1521481503341, + "iteration": 1534508678095, "links": [], "panels": [ { "columns": [], - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fontSize": "100%", "gridPos": { "h": 4, @@ -130,7 +106,7 @@ { "alias": "", "format": "table", - "rawSql": "SELECT * from mssql_types", + "rawSql": "SELECT * from mysql_types", "refId": "A" } ], @@ -138,18 +114,234 @@ "transform": "table", "type": "table" }, + { + "columns": [], + "datasource": "gdev-mysql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 32, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time_sec", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as unsigned integer) as time_sec", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as unsigned integer) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-mysql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 33, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time_sec", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as datetime) as time_sec", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as datetime) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-mysql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 34, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time_sec", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(NOW() as datetime) as time_sec", + "refId": "A", + "target": "" + } + ], + "title": "cast()NOW() as datetime) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-mysql-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 35, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT NOW() as time", + "refId": "A", + "target": "" + } + ], + "title": "NOW() as time", + "transform": "table", + "type": "table" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { - "h": 9, - "w": 8, + "h": 6, + "w": 6, "x": 0, - "y": 4 + "y": 7 }, "id": 7, "legend": { @@ -177,7 +369,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '5m') ORDER BY 1", + "rawSql": "SELECT $__timeGroupAlias(time, '5m'), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", "refId": "A" } ], @@ -215,20 +407,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 4 + "h": 6, + "w": 6, + "x": 6, + "y": 7 }, "id": 9, "legend": { @@ -256,7 +452,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '5m') ORDER BY 1", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', NULL), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", "refId": "A" } ], @@ -294,20 +490,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 4 + "h": 6, + "w": 6, + "x": 12, + "y": 7 }, "id": 10, "legend": { @@ -335,7 +535,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeGroup(time, '5m', 10.0) AS time, avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '5m') ORDER BY 1", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', 10.0), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", "refId": "A" } ], @@ -373,18 +573,105 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mysql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', previous), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m with fill(previous)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { - "h": 9, - "w": 8, + "h": 6, + "w": 6, "x": 0, "y": 13 }, @@ -414,7 +701,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeGroup(time, '$summarize') AS time, avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '$summarize') ORDER BY 1", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize'), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", "refId": "A" } ], @@ -452,19 +739,23 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { - "h": 9, - "w": 8, - "x": 8, + "h": 6, + "w": 6, + "x": 6, "y": 13 }, "id": 12, @@ -493,7 +784,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeGroup(time, '$summarize', NULL) AS time, sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '$summarize') ORDER BY 1", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', NULL), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", "refId": "A" } ], @@ -531,19 +822,23 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { - "h": 9, - "w": 8, - "x": 16, + "h": 6, + "w": 6, + "x": 12, "y": 13 }, "id": 13, @@ -572,7 +867,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeGroup(time, '$summarize', 100.0) AS time, sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY $__timeGroup(time, '$summarize') ORDER BY 1", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', 100.0), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", "refId": "A" } ], @@ -610,20 +905,107 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mysql-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', previous), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize with fill(previous)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 22 + "y": 19 }, "id": 27, "legend": { @@ -655,14 +1037,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n measurement + ' - value one' as metric, \n avg(valueOne) as valueOne\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nGROUP BY \n $__timeGroup(time, '$summarize'), \n measurement \nORDER BY 1", + "rawSql": "SELECT \n $__timeGroupAlias(time, '$summarize'), \n measurement as metric, \n avg(valueOne) as valueOne,\n avg(valueTwo) as valueTwo\nFROM\n metric_values \nWHERE\n $__timeFilter(time) AND\n measurement IN($metric)\nGROUP BY 1, 2\nORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n measurement + ' - value two' as metric, \n avg(valueTwo) as valueTwo \nFROM\n metric_values \nGROUP BY \n $__timeGroup(time, '$summarize'), \n measurement \nORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -699,20 +1075,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 22 + "y": 19 }, "id": 5, "legend": { @@ -734,7 +1114,19 @@ "pointradius": 3, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "MovingAverageValueOne", + "dashes": true, + "lines": false + }, + { + "alias": "MovingAverageValueTwo", + "dashes": true, + "lines": false, + "yaxis": 1 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, @@ -742,7 +1134,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n avg(valueOne) as valueOne, \n avg(valueTwo) as valueTwo \nFROM\n metric_values \nGROUP BY \n $__timeGroup(time, '$summarize')\nORDER BY 1", + "rawSql": "SELECT \n $__timeGroupAlias(time, '$summarize'), \n avg(valueOne) as valueOne, \n avg(valueTwo) as valueTwo \nFROM\n metric_values \nWHERE \n $__timeFilter(time) AND \n measurement IN($metric)\nGROUP BY 1\nORDER BY 1", "refId": "A" } ], @@ -780,20 +1172,208 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 27 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__unixEpochGroupAlias(timeInt32, '$summarize'), \n measurement, \n avg(valueOne) as valueOne,\n avg(valueTwo) as valueTwo\nFROM\n metric_values \nWHERE\n $__unixEpochFilter(timeInt32) AND\n measurement in($metric)\nGROUP BY 1, 2\nORDER BY 1, 2", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column using unixEpochGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mysql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "MovingAverageValueOne", + "dashes": true, + "lines": false + }, + { + "alias": "MovingAverageValueTwo", + "dashes": true, + "lines": false, + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__unixEpochGroupAlias(timeInt32, '$summarize'), \n avg(valueOne) as valueOne,\n avg(valueTwo) as valueTwo\nFROM\n metric_values \nWHERE\n $__unixEpochFilter(timeInt32) AND\n measurement in($metric)\nGROUP BY 1\nORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column using unixEpochGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-mysql-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 }, "id": 4, "legend": { @@ -825,14 +1405,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -869,20 +1443,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 30 + "y": 35 }, "id": 28, "legend": { @@ -912,7 +1490,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), valueOne, valueTwo FROM metric_values ORDER BY 1", "refId": "A" } ], @@ -950,20 +1528,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 43 }, "id": 19, "legend": { @@ -995,14 +1577,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -1039,20 +1615,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 43 }, "id": 18, "legend": { @@ -1082,7 +1662,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" } ], @@ -1120,20 +1700,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 51 }, "id": 17, "legend": { @@ -1165,14 +1749,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -1209,20 +1787,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 2, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 51 }, "id": 20, "legend": { @@ -1252,7 +1834,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" } ], @@ -1290,186 +1872,24 @@ "min": null, "show": true } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_MSSQL_TEST}", - "fill": 2, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 54 - }, - "id": 29, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 3, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "", - "format": "time_series", - "rawSql": "DECLARE \n @from int = $__unixEpochFrom(),\n @to int = $__unixEpochTo()\n \nEXEC dbo.sp_test_epoch @from, @to", - "refId": "A" - } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Stored procedure support using epoch", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_MSSQL_TEST}", - "fill": 2, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 54 - }, - "id": 30, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 3, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "", - "format": "time_series", - "rawSql": "DECLARE \n @from datetime = $__timeFrom(),\n @to datetime = $__timeTo()\n \nEXEC dbo.sp_test_datetime @from, @to", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Stored procedure support using datetime", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 59 }, "id": 14, "legend": { @@ -1499,14 +1919,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -1546,20 +1960,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 62 + "y": 59 }, "id": 15, "legend": { @@ -1589,7 +2007,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" } ], @@ -1629,20 +2047,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 70 + "y": 67 }, "id": 25, "legend": { @@ -1672,14 +2094,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -1719,20 +2135,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 70 + "y": 67 }, "id": 22, "legend": { @@ -1762,7 +2182,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" } ], @@ -1802,20 +2222,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 78 + "y": 75 }, "id": 21, "legend": { @@ -1845,14 +2269,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -1892,20 +2310,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 78 + "y": 75 }, "id": 26, "legend": { @@ -1935,7 +2357,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" } ], @@ -1975,20 +2397,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 86 + "y": 83 }, "id": 23, "legend": { @@ -2018,14 +2444,8 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values WHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" - }, - { - "alias": "", - "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", - "refId": "B" } ], "thresholds": [], @@ -2065,20 +2485,24 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_MSSQL_TEST}", + "datasource": "gdev-mysql-ds-tests", "fill": 1, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 86 + "y": 83 }, "id": 24, "legend": { @@ -2108,7 +2532,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND measurement IN($metric) ORDER BY 1", "refId": "A" } ], @@ -2148,15 +2572,62 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": false, "schemaVersion": 16, "style": "dark", - "tags": [], + "tags": [ + "gdev", + "mysql" + ], "templating": { "list": [ + { + "allValue": "", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "gdev-mysql-ds-tests", + "hide": 0, + "includeAll": true, + "label": "Metric", + "multi": true, + "name": "metric", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "Metric A", + "value": "Metric A" + }, + { + "selected": false, + "text": "Metric B", + "value": "Metric B" + } + ], + "query": "SELECT DISTINCT measurement FROM metric_values", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "auto": false, "auto_count": 30, @@ -2202,13 +2673,14 @@ ], "query": "1s,10s,30s,1m,5m,10m", "refresh": 2, + "skipUrlSync": false, "type": "interval" } ] }, "time": { "from": "2018-03-15T12:30:00.000Z", - "to": "2018-03-15T13:55:00.000Z" + "to": "2018-03-15T13:55:01.000Z" }, "timepicker": { "refresh_intervals": [ @@ -2236,7 +2708,7 @@ ] }, "timezone": "", - "title": "Microsoft SQL Server Data Source Test", - "uid": "GlAqcPgmz", - "version": 37 + "title": "Datasource tests - MySQL (unittest)", + "uid": "Hmf8FDkmz", + "version": 2 } \ No newline at end of file diff --git a/docker/blocks/postgres/dashboard.json b/devenv/dev-dashboards/datasource_tests_postgres_fakedata.json similarity index 91% rename from docker/blocks/postgres/dashboard.json rename to devenv/dev-dashboards/datasource_tests_postgres_fakedata.json index 77b0ceac624..508cae86bc3 100644 --- a/docker/blocks/postgres/dashboard.json +++ b/devenv/dev-dashboards/datasource_tests_postgres_fakedata.json @@ -1,40 +1,4 @@ { - "__inputs": [ - { - "name": "DS_POSTGRESQL", - "label": "PostgreSQL", - "description": "", - "type": "datasource", - "pluginId": "postgres", - "pluginName": "PostgreSQL" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "postgres", - "name": "PostgreSQL", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], "annotations": { "list": [ { @@ -52,8 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": null, - "iteration": 1518601837383, + "iteration": 1532620601931, "links": [], "panels": [ { @@ -63,7 +26,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_POSTGRESQL}", + "datasource": "gdev-postgres", "fill": 2, "gridPos": { "h": 9, @@ -150,14 +113,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_POSTGRESQL}", + "datasource": "gdev-postgres", "fill": 2, "gridPos": { "h": 18, @@ -236,14 +203,18 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_POSTGRESQL}", + "datasource": "gdev-postgres", "fill": 2, "gridPos": { "h": 9, @@ -316,11 +287,15 @@ "min": null, "show": true } - ] + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "columns": [], - "datasource": "${DS_POSTGRESQL}", + "datasource": "gdev-postgres", "fontSize": "100%", "gridPos": { "h": 9, @@ -377,6 +352,7 @@ "schemaVersion": 16, "style": "dark", "tags": [ + "gdev", "fake-data-gen", "postgres" ], @@ -384,8 +360,11 @@ "list": [ { "allValue": null, - "current": {}, - "datasource": "${DS_POSTGRESQL}", + "current": { + "text": "America", + "value": "America" + }, + "datasource": "gdev-postgres", "hide": 0, "includeAll": false, "label": "Datacenter", @@ -395,6 +374,7 @@ "query": "SELECT DISTINCT datacenter FROM grafana_metric", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], @@ -404,8 +384,11 @@ }, { "allValue": null, - "current": {}, - "datasource": "${DS_POSTGRESQL}", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "gdev-postgres", "hide": 0, "includeAll": true, "label": "Hostname", @@ -415,6 +398,7 @@ "query": "SELECT DISTINCT hostname FROM grafana_metric WHERE datacenter='$datacenter'", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], @@ -507,6 +491,7 @@ ], "query": "1s,10s,30s,1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "refresh": 2, + "skipUrlSync": false, "type": "interval" } ] @@ -541,7 +526,7 @@ ] }, "timezone": "", - "title": "Grafana Fake Data Gen - PostgreSQL", + "title": "Datasource tests - Postgres", "uid": "JYola5qzz", - "version": 1 + "version": 4 } \ No newline at end of file diff --git a/devenv/dev-dashboards/datasource_tests_postgres_unittest.json b/devenv/dev-dashboards/datasource_tests_postgres_unittest.json new file mode 100644 index 00000000000..3c56868e9ff --- /dev/null +++ b/devenv/dev-dashboards/datasource_tests_postgres_unittest.json @@ -0,0 +1,2694 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "gdev-postgres-ds-tests", + "enable": false, + "hide": false, + "iconColor": "#6ed0e0", + "limit": 100, + "name": "Deploys", + "rawQuery": "SELECT \"time_sec\" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "gdev-postgres-ds-tests", + "enable": false, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Tickets", + "rawQuery": "SELECT \"time_sec\" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "gdev-postgres-ds-tests", + "enable": false, + "hide": false, + "iconColor": "#7eb26d", + "limit": 100, + "name": "Metric Values timeEpoch macro", + "rawQuery": "SELECT \n $__timeEpoch(time), \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "gdev-postgres-ds-tests", + "enable": false, + "hide": false, + "iconColor": "#1f78c1", + "limit": 100, + "name": "Metric Values native time", + "rawQuery": "SELECT \n time, \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", + "showIn": 0, + "tags": [], + "type": "tags" + } + ] + }, + "description": "Run the postgres unit tests to generate the data backing this dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534507993194, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "gdev-postgres-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT * FROM postgres_types", + "refId": "A" + } + ], + "title": "Data types", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-postgres-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 32, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as bigint) as time", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as bigint) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-postgres-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 33, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as timestamp) as time", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as datetime) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-postgres-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 34, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT localtimestamp as time", + "refId": "A", + "target": "" + } + ], + "title": "localtimestamp as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-postgres-ds-tests", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 35, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT NOW() as time", + "refId": "A", + "target": "" + } + ], + "title": "NOW() as time", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m'), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m without fill", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', NULL), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m with fill(NULL) and null as zero", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', 10.0), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m with fill(10.0)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '5m', previous), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "timeGroup macro 5m with fill(previous)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize'), avg(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize without fill", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', NULL), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize with fill(NULL)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', 100.0), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize with fill(100.0)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeGroupAlias(time, '$summarize', previous), sum(value) as value FROM metric WHERE $__timeFilter(time) GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metrics - timeGroup macro $summarize with fill(previous)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__timeGroupAlias(time, '$summarize'), \n measurement, \n avg(\"valueOne\") as \"valueOne\",\n avg(\"valueTwo\") as \"valueTwo\"\nFROM\n metric_values \nWHERE\n $__timeFilter(time) AND\n measurement in($metric)\nGROUP BY 1, 2\nORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column using timeGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__timeGroupAlias(time, '$summarize'), \n avg(\"valueOne\") as \"valueOne\", \n avg(\"valueTwo\") as \"valueTwo\" \nFROM\n metric_values \nWHERE\n $__timeFilter(time) AND\n measurement in($metric)\nGROUP BY 1\nORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column using timeGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__unixEpochGroupAlias(\"timeInt32\", '$summarize'), \n measurement, \n avg(\"valueOne\") as \"valueOne\",\n avg(\"valueTwo\") as \"valueTwo\"\nFROM\n metric_values \nWHERE\n $__unixEpochFilter(\"timeInt32\") AND\n measurement in($metric)\nGROUP BY 1, 2\nORDER BY 1, 2", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column using unixEpochGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n $__unixEpochGroupAlias(\"timeInt32\", '$summarize'), \n avg(\"valueOne\") as \"valueOne\",\n avg(\"valueTwo\") as \"valueTwo\"\nFROM\n metric_values \nWHERE\n $__unixEpochFilter(\"timeInt32\") AND\n measurement in($metric)\nGROUP BY 1\nORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column using timeGroup macro ($summarize)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement, \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement, \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement, \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement, \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - series mode", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - series mode", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement, \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - histogram", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 50, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values\nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - histogram", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 100, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement, \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - histogram stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - histogram stacked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), measurement, \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series with metric column - histogram stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-postgres-ds-tests", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values \nWHERE $__timeFilter(time) AND measurement in($metric) ORDER BY 1", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Multiple series without metric column - histogram stacked percent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": 20, + "mode": "histogram", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "gdev", + "postgres" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "gdev-postgres-ds-tests", + "hide": 0, + "includeAll": true, + "label": "Metric", + "multi": true, + "name": "metric", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "Metric A", + "value": "Metric A" + }, + { + "selected": false, + "text": "Metric B", + "value": "Metric B" + } + ], + "query": "SELECT DISTINCT measurement FROM metric_values", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": "Interval", + "name": "summarize", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + } + ], + "query": "1s,10s,30s,1m,5m,10m", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "2018-03-15T12:30:00.000Z", + "to": "2018-03-15T13:55:01.000Z" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Datasource tests - Postgres (unittest)", + "uid": "vHQdlVziz", + "version": 1 +} \ No newline at end of file diff --git a/devenv/dev-dashboards/panel_tests_graph.json b/devenv/dev-dashboards/panel_tests_graph.json new file mode 100644 index 00000000000..ba677764a43 --- /dev/null +++ b/devenv/dev-dashboards/panel_tests_graph.json @@ -0,0 +1,1675 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "no_data_points", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "No Data Points Warning", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "datapoints_outside_range", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Datapoints Outside Range Warning", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "random_walk", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Random walk series", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 7 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenario": "random_walk", + "scenarioId": "random_walk", + "target": "" + } + ], + "thresholds": [], + "timeFrom": "2s", + "timeShift": null, + "title": "Millisecond res x-axis and tooltip", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Just verify that the tooltip time has millisecond resolution ", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 6, + "links": [], + "mode": "markdown", + "title": "", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 14 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "B-series", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "2000,3000,4000,1000,3000,10000", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "2 yaxis and axis labels", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "Perecent", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "Pressure", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Verify that axis labels look ok", + "editable": true, + "error": false, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 7, + "links": [], + "mode": "markdown", + "title": "", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,null,null,null,null,null,null,100,10,10,20,30,40,10", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "null value connected", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,null,null,null,null,null,null,100,10,10,20,30,40,10", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "null value null as zero", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Should be a long line connecting the null region in the `connected` mode, and in zero it should just be a line with zero value at the null points. ", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 13, + "links": [], + "mode": "markdown", + "title": "", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 30 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "B-series", + "zindex": -3 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "hide": false, + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,null,null,null,null,null,null,100,10,10,20,30,40,10", + "target": "" + }, + { + "alias": "", + "hide": false, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,10,20,30,40,40,40,100,10,20,20", + "target": "" + }, + { + "alias": "", + "hide": false, + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,10,20,30,40,40,40,100,10,20,20", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Stacking value ontop of nulls", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Stacking values on top of nulls, should treat the null values as zero. ", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 30 + }, + "id": 14, + "links": [], + "mode": "markdown", + "title": "", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 37 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "B-series", + "zindex": -3 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "", + "hide": false, + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,40,null,null,null,null,null,null,100,10,10,20,30,40,10", + "target": "" + }, + { + "alias": "", + "hide": false, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,40,null,null,null,null,null,null,100,10,10,20,30,40,10", + "target": "" + }, + { + "alias": "", + "hide": false, + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,40,null,null,null,null,null,null,100,10,10,20,30,40,10", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Stacking all series null segment", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Stacking when all values are null should leave a gap in the graph", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 37 + }, + "id": 15, + "links": [], + "mode": "markdown", + "title": "", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 44 + }, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "C-series", + "steppedLine": true + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "hide": false, + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,null,40,null,90,null,null,100,null,null,100,null,null,80,null", + "target": "" + }, + { + "alias": "", + "hide": false, + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "20,null40,null,null,50,null,70,null,100,null,10,null,30,null", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Null between points", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Left is showing null between values for a normal line graph and staircase graph. Orphaned data points should be rendered as points", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 44 + }, + "id": 22, + "links": [], + "mode": "markdown", + "title": "", + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "decimals": 3, + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Legend Table Single Series Should Take Minimum Height", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "decimals": 3, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "D", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Legend Table No Scroll Visible", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "decimals": 3, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "D", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "E", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "F", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "G", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "H", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "I", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "J", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Legend Table Should Scroll", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "decimals": 3, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 65 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "D", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Legend Table No Scroll Visible", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "decimals": 3, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 65 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "D", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "E", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "F", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "G", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "H", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "I", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "J", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "K", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + }, + { + "refId": "L", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Legend Table No Scroll Visible", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "revision": 8, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "gdev", + "panel-tests" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Panel Tests - Graph", + "uid": "5SdHCadmz", + "version": 1 +} diff --git a/devenv/dev-dashboards/panel_tests_graph_time_regions.json b/devenv/dev-dashboards/panel_tests_graph_time_regions.json new file mode 100644 index 00000000000..8d0bae1221c --- /dev/null +++ b/devenv/dev-dashboards/panel_tests_graph_time_regions.json @@ -0,0 +1,511 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 2, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "random_walk", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [ + { + "colorMode": "gray", + "fill": true, + "fillColor": "rgba(255, 255, 255, 0.03)", + "from": "08:30", + "fromDayOfWeek": 1, + "line": false, + "lineColor": "rgba(255, 255, 255, 0.2)", + "op": "time", + "to": "16:45", + "toDayOfWeek": 5 + } + ], + "timeShift": null, + "title": "Business Hours", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 2, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "random_walk", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [ + { + "colorMode": "red", + "fill": true, + "fillColor": "rgba(255, 255, 255, 0.03)", + "from": "20:00", + "fromDayOfWeek": 7, + "line": false, + "lineColor": "rgba(255, 255, 255, 0.2)", + "op": "time", + "to": "23:00", + "toDayOfWeek": 7 + } + ], + "timeShift": null, + "title": "Sunday's 20-23", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "A-series": "#d683ce" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 2, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "scenarioId": "random_walk", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [ + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(255, 0, 0, 0.22)", + "from": "", + "fromDayOfWeek": 1, + "line": true, + "lineColor": "rgba(255, 0, 0, 0.32)", + "op": "time", + "to": "", + "toDayOfWeek": 1 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(255, 127, 0, 0.22)", + "fromDayOfWeek": 2, + "line": true, + "lineColor": "rgba(255, 127, 0, 0.32)", + "op": "time", + "toDayOfWeek": 2 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(255, 255, 0, 0.22)", + "fromDayOfWeek": 3, + "line": true, + "lineColor": "rgba(255, 255, 0, 0.22)", + "op": "time", + "toDayOfWeek": 3 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(0, 255, 0, 0.22)", + "fromDayOfWeek": 4, + "line": true, + "lineColor": "rgba(0, 255, 0, 0.32)", + "op": "time", + "toDayOfWeek": 4 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(0, 0, 255, 0.22)", + "fromDayOfWeek": 5, + "line": true, + "lineColor": "rgba(0, 0, 255, 0.32)", + "op": "time", + "toDayOfWeek": 5 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(75, 0, 130, 0.22)", + "fromDayOfWeek": 6, + "line": true, + "lineColor": "rgba(75, 0, 130, 0.32)", + "op": "time", + "toDayOfWeek": 6 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(148, 0, 211, 0.22)", + "fromDayOfWeek": 7, + "line": true, + "lineColor": "rgba(148, 0, 211, 0.32)", + "op": "time", + "toDayOfWeek": 7 + } + ], + "timeShift": null, + "title": "Each day of week", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 2, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "random_walk", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [ + { + "colorMode": "red", + "fill": false, + "from": "05:00", + "line": true, + "op": "time" + } + ], + "timeShift": null, + "title": "05:00", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "gdev", + "panel-tests" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Panel Tests - Graph (Time Regions)", + "uid": "XMjIZPmik", + "version": 1 +} \ No newline at end of file diff --git a/devenv/dev-dashboards/panel_tests_polystat.json b/devenv/dev-dashboards/panel_tests_polystat.json new file mode 100644 index 00000000000..51d3085c438 --- /dev/null +++ b/devenv/dev-dashboards/panel_tests_polystat.json @@ -0,0 +1,3343 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "animationModes": [ + { + "text": "Show All", + "value": "all" + }, + { + "text": "Show Triggered", + "value": "triggered" + } + ], + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "d3DivId": "d3_svg_4", + "datasource": "gdev-testdata", + "decimals": 2, + "displayModes": [ + { + "text": "Show All", + "value": "all" + }, + { + "text": "Show Triggered", + "value": "triggered" + } + ], + "fontSizes": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70 + ], + "fontTypes": [ + "Open Sans", + "Arial", + "Avant Garde", + "Bookman", + "Consolas", + "Courier", + "Courier New", + "Futura", + "Garamond", + "Helvetica", + "Palatino", + "Times", + "Times New Roman", + "Verdana" + ], + "format": "none", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "links": [], + "notcolors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "operatorName": "avg", + "operatorOptions": [ + { + "text": "Average", + "value": "avg" + }, + { + "text": "Count", + "value": "count" + }, + { + "text": "Current", + "value": "current" + }, + { + "text": "Delta", + "value": "delta" + }, + { + "text": "Difference", + "value": "diff" + }, + { + "text": "First", + "value": "first" + }, + { + "text": "Log Min", + "value": "logmin" + }, + { + "text": "Max", + "value": "max" + }, + { + "text": "Min", + "value": "min" + }, + { + "text": "Name", + "value": "name" + }, + { + "text": "Time of Last Point", + "value": "last_time" + }, + { + "text": "Time Step", + "value": "time_step" + }, + { + "text": "Total", + "value": "total" + } + ], + "polystat": { + "animationSpeed": 2500, + "columnAutoSize": true, + "columns": "", + "defaultClickThrough": "", + "defaultClickThroughSanitize": true, + "displayLimit": 100, + "fontAutoScale": true, + "fontSize": 12, + "globalDisplayMode": "all", + "globalOperatorName": "avg", + "gradientEnabled": true, + "hexagonSortByDirection": "asc", + "hexagonSortByField": "name", + "maxMetrics": 0, + "polygonBorderColor": "black", + "polygonBorderSize": 2, + "radius": "", + "radiusAutoSize": true, + "rowAutoSize": true, + "rows": "", + "shape": "hexagon_pointed_top", + "tooltipDisplayMode": "all", + "tooltipDisplayTextTriggeredEmpty": "OK", + "tooltipFontSize": 12, + "tooltipFontType": "Open Sans", + "tooltipPrimarySortDirection": "desc", + "tooltipPrimarySortField": "thresholdLevel", + "tooltipSecondarySortDirection": "desc", + "tooltipSecondarySortField": "value", + "tooltipTimestampEnabled": true + }, + "savedComposites": [], + "savedOverrides": [], + "shapes": [ + { + "text": "Hexagon Pointed Top", + "value": "hexagon_pointed_top" + }, + { + "text": "Hexagon Flat Top", + "value": "hexagon_flat_top" + }, + { + "text": "Circle", + "value": "circle" + }, + { + "text": "Cross", + "value": "cross" + }, + { + "text": "Diamond", + "value": "diamond" + }, + { + "text": "Square", + "value": "square" + }, + { + "text": "Star", + "value": "star" + }, + { + "text": "Triangle", + "value": "triangle" + }, + { + "text": "Wye", + "value": "wye" + } + ], + "sortDirections": [ + { + "text": "Ascending", + "value": "asc" + }, + { + "text": "Descending", + "value": "desc" + } + ], + "sortFields": [ + { + "text": "Name", + "value": "name" + }, + { + "text": "Threshold Level", + "value": "thresholdLevel" + }, + { + "text": "Value", + "value": "value" + } + ], + "svgContainer": {}, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "C", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "D", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "E", + "scenarioId": "random_walk" + } + ], + "thresholdStates": [ + { + "text": "ok", + "value": 0 + }, + { + "text": "warning", + "value": 1 + }, + { + "text": "critical", + "value": 2 + }, + { + "text": "custom", + "value": 3 + } + ], + "title": "Poor use of space", + "type": "grafana-polystat-panel", + "unitFormats": [ + { + "submenu": [ + { + "text": "none", + "value": "none" + }, + { + "text": "short", + "value": "short" + }, + { + "text": "percent (0-100)", + "value": "percent" + }, + { + "text": "percent (0.0-1.0)", + "value": "percentunit" + }, + { + "text": "Humidity (%H)", + "value": "humidity" + }, + { + "text": "decibel", + "value": "dB" + }, + { + "text": "hexadecimal (0x)", + "value": "hex0x" + }, + { + "text": "hexadecimal", + "value": "hex" + }, + { + "text": "scientific notation", + "value": "sci" + }, + { + "text": "locale format", + "value": "locale" + } + ], + "text": "none" + }, + { + "submenu": [ + { + "text": "Dollars ($)", + "value": "currencyUSD" + }, + { + "text": "Pounds (£)", + "value": "currencyGBP" + }, + { + "text": "Euro (€)", + "value": "currencyEUR" + }, + { + "text": "Yen (¥)", + "value": "currencyJPY" + }, + { + "text": "Rubles (₽)", + "value": "currencyRUB" + }, + { + "text": "Hryvnias (₴)", + "value": "currencyUAH" + }, + { + "text": "Real (R$)", + "value": "currencyBRL" + }, + { + "text": "Danish Krone (kr)", + "value": "currencyDKK" + }, + { + "text": "Icelandic Króna (kr)", + "value": "currencyISK" + }, + { + "text": "Norwegian Krone (kr)", + "value": "currencyNOK" + }, + { + "text": "Swedish Krona (kr)", + "value": "currencySEK" + }, + { + "text": "Czech koruna (czk)", + "value": "currencyCZK" + }, + { + "text": "Swiss franc (CHF)", + "value": "currencyCHF" + }, + { + "text": "Polish Złoty (PLN)", + "value": "currencyPLN" + }, + { + "text": "Bitcoin (฿)", + "value": "currencyBTC" + } + ], + "text": "currency" + }, + { + "submenu": [ + { + "text": "Hertz (1/s)", + "value": "hertz" + }, + { + "text": "nanoseconds (ns)", + "value": "ns" + }, + { + "text": "microseconds (µs)", + "value": "µs" + }, + { + "text": "milliseconds (ms)", + "value": "ms" + }, + { + "text": "seconds (s)", + "value": "s" + }, + { + "text": "minutes (m)", + "value": "m" + }, + { + "text": "hours (h)", + "value": "h" + }, + { + "text": "days (d)", + "value": "d" + }, + { + "text": "duration (ms)", + "value": "dtdurationms" + }, + { + "text": "duration (s)", + "value": "dtdurations" + }, + { + "text": "duration (hh:mm:ss)", + "value": "dthms" + }, + { + "text": "Timeticks (s/100)", + "value": "timeticks" + } + ], + "text": "time" + }, + { + "submenu": [ + { + "text": "YYYY-MM-DD HH:mm:ss", + "value": "dateTimeAsIso" + }, + { + "text": "DD/MM/YYYY h:mm:ss a", + "value": "dateTimeAsUS" + }, + { + "text": "From Now", + "value": "dateTimeFromNow" + } + ], + "text": "date & time" + }, + { + "submenu": [ + { + "text": "bits", + "value": "bits" + }, + { + "text": "bytes", + "value": "bytes" + }, + { + "text": "kibibytes", + "value": "kbytes" + }, + { + "text": "mebibytes", + "value": "mbytes" + }, + { + "text": "gibibytes", + "value": "gbytes" + } + ], + "text": "data (IEC)" + }, + { + "submenu": [ + { + "text": "bits", + "value": "decbits" + }, + { + "text": "bytes", + "value": "decbytes" + }, + { + "text": "kilobytes", + "value": "deckbytes" + }, + { + "text": "megabytes", + "value": "decmbytes" + }, + { + "text": "gigabytes", + "value": "decgbytes" + } + ], + "text": "data (Metric)" + }, + { + "submenu": [ + { + "text": "packets/sec", + "value": "pps" + }, + { + "text": "bits/sec", + "value": "bps" + }, + { + "text": "bytes/sec", + "value": "Bps" + }, + { + "text": "kilobits/sec", + "value": "Kbits" + }, + { + "text": "kilobytes/sec", + "value": "KBs" + }, + { + "text": "megabits/sec", + "value": "Mbits" + }, + { + "text": "megabytes/sec", + "value": "MBs" + }, + { + "text": "gigabytes/sec", + "value": "GBs" + }, + { + "text": "gigabits/sec", + "value": "Gbits" + } + ], + "text": "data rate" + }, + { + "submenu": [ + { + "text": "hashes/sec", + "value": "Hs" + }, + { + "text": "kilohashes/sec", + "value": "KHs" + }, + { + "text": "megahashes/sec", + "value": "MHs" + }, + { + "text": "gigahashes/sec", + "value": "GHs" + }, + { + "text": "terahashes/sec", + "value": "THs" + }, + { + "text": "petahashes/sec", + "value": "PHs" + }, + { + "text": "exahashes/sec", + "value": "EHs" + } + ], + "text": "hash rate" + }, + { + "submenu": [ + { + "text": "ops/sec (ops)", + "value": "ops" + }, + { + "text": "requests/sec (rps)", + "value": "reqps" + }, + { + "text": "reads/sec (rps)", + "value": "rps" + }, + { + "text": "writes/sec (wps)", + "value": "wps" + }, + { + "text": "I/O ops/sec (iops)", + "value": "iops" + }, + { + "text": "ops/min (opm)", + "value": "opm" + }, + { + "text": "reads/min (rpm)", + "value": "rpm" + }, + { + "text": "writes/min (wpm)", + "value": "wpm" + } + ], + "text": "throughput" + }, + { + "submenu": [ + { + "text": "millimetre (mm)", + "value": "lengthmm" + }, + { + "text": "meter (m)", + "value": "lengthm" + }, + { + "text": "feet (ft)", + "value": "lengthft" + }, + { + "text": "kilometer (km)", + "value": "lengthkm" + }, + { + "text": "mile (mi)", + "value": "lengthmi" + } + ], + "text": "length" + }, + { + "submenu": [ + { + "text": "Square Meters (m²)", + "value": "areaM2" + }, + { + "text": "Square Feet (ft²)", + "value": "areaF2" + }, + { + "text": "Square Miles (mi²)", + "value": "areaMI2" + } + ], + "text": "area" + }, + { + "submenu": [ + { + "text": "milligram (mg)", + "value": "massmg" + }, + { + "text": "gram (g)", + "value": "massg" + }, + { + "text": "kilogram (kg)", + "value": "masskg" + }, + { + "text": "metric ton (t)", + "value": "masst" + } + ], + "text": "mass" + }, + { + "submenu": [ + { + "text": "metres/second (m/s)", + "value": "velocityms" + }, + { + "text": "kilometers/hour (km/h)", + "value": "velocitykmh" + }, + { + "text": "miles/hour (mph)", + "value": "velocitymph" + }, + { + "text": "knot (kn)", + "value": "velocityknot" + } + ], + "text": "velocity" + }, + { + "submenu": [ + { + "text": "millilitre (mL)", + "value": "mlitre" + }, + { + "text": "litre (L)", + "value": "litre" + }, + { + "text": "cubic metre", + "value": "m3" + }, + { + "text": "Normal cubic metre", + "value": "Nm3" + }, + { + "text": "cubic decimetre", + "value": "dm3" + }, + { + "text": "gallons", + "value": "gallons" + } + ], + "text": "volume" + }, + { + "submenu": [ + { + "text": "Watt (W)", + "value": "watt" + }, + { + "text": "Kilowatt (kW)", + "value": "kwatt" + }, + { + "text": "Milliwatt (mW)", + "value": "mwatt" + }, + { + "text": "Watt per square metre (W/m²)", + "value": "Wm2" + }, + { + "text": "Volt-ampere (VA)", + "value": "voltamp" + }, + { + "text": "Kilovolt-ampere (kVA)", + "value": "kvoltamp" + }, + { + "text": "Volt-ampere reactive (var)", + "value": "voltampreact" + }, + { + "text": "Kilovolt-ampere reactive (kvar)", + "value": "kvoltampreact" + }, + { + "text": "Watt-hour (Wh)", + "value": "watth" + }, + { + "text": "Kilowatt-hour (kWh)", + "value": "kwatth" + }, + { + "text": "Kilowatt-min (kWm)", + "value": "kwattm" + }, + { + "text": "Joule (J)", + "value": "joule" + }, + { + "text": "Electron volt (eV)", + "value": "ev" + }, + { + "text": "Ampere (A)", + "value": "amp" + }, + { + "text": "Kiloampere (kA)", + "value": "kamp" + }, + { + "text": "Milliampere (mA)", + "value": "mamp" + }, + { + "text": "Volt (V)", + "value": "volt" + }, + { + "text": "Kilovolt (kV)", + "value": "kvolt" + }, + { + "text": "Millivolt (mV)", + "value": "mvolt" + }, + { + "text": "Decibel-milliwatt (dBm)", + "value": "dBm" + }, + { + "text": "Ohm (Ω)", + "value": "ohm" + }, + { + "text": "Lumens (Lm)", + "value": "lumens" + } + ], + "text": "energy" + }, + { + "submenu": [ + { + "text": "Celsius (°C)", + "value": "celsius" + }, + { + "text": "Farenheit (°F)", + "value": "farenheit" + }, + { + "text": "Kelvin (K)", + "value": "kelvin" + } + ], + "text": "temperature" + }, + { + "submenu": [ + { + "text": "Millibars", + "value": "pressurembar" + }, + { + "text": "Bars", + "value": "pressurebar" + }, + { + "text": "Kilobars", + "value": "pressurekbar" + }, + { + "text": "Hectopascals", + "value": "pressurehpa" + }, + { + "text": "Kilopascals", + "value": "pressurekpa" + }, + { + "text": "Inches of mercury", + "value": "pressurehg" + }, + { + "text": "PSI", + "value": "pressurepsi" + } + ], + "text": "pressure" + }, + { + "submenu": [ + { + "text": "Newton-meters (Nm)", + "value": "forceNm" + }, + { + "text": "Kilonewton-meters (kNm)", + "value": "forcekNm" + }, + { + "text": "Newtons (N)", + "value": "forceN" + }, + { + "text": "Kilonewtons (kN)", + "value": "forcekN" + } + ], + "text": "force" + }, + { + "submenu": [ + { + "text": "Gallons/min (gpm)", + "value": "flowgpm" + }, + { + "text": "Cubic meters/sec (cms)", + "value": "flowcms" + }, + { + "text": "Cubic feet/sec (cfs)", + "value": "flowcfs" + }, + { + "text": "Cubic feet/min (cfm)", + "value": "flowcfm" + }, + { + "text": "Litre/hour", + "value": "litreh" + }, + { + "text": "Litre/min (l/min)", + "value": "flowlpm" + }, + { + "text": "milliLitre/min (mL/min)", + "value": "flowmlpm" + } + ], + "text": "flow" + }, + { + "submenu": [ + { + "text": "Degrees (°)", + "value": "degree" + }, + { + "text": "Radians", + "value": "radian" + }, + { + "text": "Gradian", + "value": "grad" + } + ], + "text": "angle" + }, + { + "submenu": [ + { + "text": "Meters/sec²", + "value": "accMS2" + }, + { + "text": "Feet/sec²", + "value": "accFS2" + }, + { + "text": "G unit", + "value": "accG" + } + ], + "text": "acceleration" + }, + { + "submenu": [ + { + "text": "Becquerel (Bq)", + "value": "radbq" + }, + { + "text": "curie (Ci)", + "value": "radci" + }, + { + "text": "Gray (Gy)", + "value": "radgy" + }, + { + "text": "rad", + "value": "radrad" + }, + { + "text": "Sievert (Sv)", + "value": "radsv" + }, + { + "text": "rem", + "value": "radrem" + }, + { + "text": "Exposure (C/kg)", + "value": "radexpckg" + }, + { + "text": "roentgen (R)", + "value": "radr" + }, + { + "text": "Sievert/hour (Sv/h)", + "value": "radsvh" + } + ], + "text": "radiation" + }, + { + "submenu": [ + { + "text": "parts-per-million (ppm)", + "value": "ppm" + }, + { + "text": "parts-per-billion (ppb)", + "value": "conppb" + }, + { + "text": "nanogram per cubic metre (ng/m³)", + "value": "conngm3" + }, + { + "text": "nanogram per normal cubic metre (ng/Nm³)", + "value": "conngNm3" + }, + { + "text": "microgram per cubic metre (μg/m³)", + "value": "conμgm3" + }, + { + "text": "microgram per normal cubic metre (μg/Nm³)", + "value": "conμgNm3" + }, + { + "text": "milligram per cubic metre (mg/m³)", + "value": "conmgm3" + }, + { + "text": "milligram per normal cubic metre (mg/Nm³)", + "value": "conmgNm3" + }, + { + "text": "gram per cubic metre (g/m³)", + "value": "congm3" + }, + { + "text": "gram per normal cubic metre (g/Nm³)", + "value": "congNm3" + } + ], + "text": "concentration" + } + ] + }, + { + "animationModes": [ + { + "text": "Show All", + "value": "all" + }, + { + "text": "Show Triggered", + "value": "triggered" + } + ], + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "d3DivId": "d3_svg_5", + "datasource": "gdev-testdata", + "decimals": 2, + "displayModes": [ + { + "text": "Show All", + "value": "all" + }, + { + "text": "Show Triggered", + "value": "triggered" + } + ], + "fontSizes": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70 + ], + "fontTypes": [ + "Open Sans", + "Arial", + "Avant Garde", + "Bookman", + "Consolas", + "Courier", + "Courier New", + "Futura", + "Garamond", + "Helvetica", + "Palatino", + "Times", + "Times New Roman", + "Verdana" + ], + "format": "none", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [], + "notcolors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "operatorName": "avg", + "operatorOptions": [ + { + "text": "Average", + "value": "avg" + }, + { + "text": "Count", + "value": "count" + }, + { + "text": "Current", + "value": "current" + }, + { + "text": "Delta", + "value": "delta" + }, + { + "text": "Difference", + "value": "diff" + }, + { + "text": "First", + "value": "first" + }, + { + "text": "Log Min", + "value": "logmin" + }, + { + "text": "Max", + "value": "max" + }, + { + "text": "Min", + "value": "min" + }, + { + "text": "Name", + "value": "name" + }, + { + "text": "Time of Last Point", + "value": "last_time" + }, + { + "text": "Time Step", + "value": "time_step" + }, + { + "text": "Total", + "value": "total" + } + ], + "polystat": { + "animationSpeed": 2500, + "columnAutoSize": true, + "columns": "", + "defaultClickThrough": "", + "defaultClickThroughSanitize": true, + "displayLimit": 100, + "fontAutoScale": true, + "fontSize": 12, + "globalDisplayMode": "all", + "globalOperatorName": "avg", + "gradientEnabled": true, + "hexagonSortByDirection": "asc", + "hexagonSortByField": "name", + "maxMetrics": 0, + "polygonBorderColor": "black", + "polygonBorderSize": 2, + "radius": "", + "radiusAutoSize": true, + "rowAutoSize": true, + "rows": "", + "shape": "hexagon_pointed_top", + "tooltipDisplayMode": "all", + "tooltipDisplayTextTriggeredEmpty": "OK", + "tooltipFontSize": 12, + "tooltipFontType": "Open Sans", + "tooltipPrimarySortDirection": "desc", + "tooltipPrimarySortField": "thresholdLevel", + "tooltipSecondarySortDirection": "desc", + "tooltipSecondarySortField": "value", + "tooltipTimestampEnabled": true + }, + "savedComposites": [ + { + "compositeName": "comp", + "members": [ + { + "seriesName": "A-series" + }, + { + "seriesName": "B-series" + } + ], + "enabled": true, + "clickThrough": "", + "hideMembers": true, + "showName": true, + "showValue": true, + "animateMode": "all", + "thresholdLevel": 0, + "sanitizeURLEnabled": true, + "sanitizedURL": "" + } + ], + "savedOverrides": [], + "shapes": [ + { + "text": "Hexagon Pointed Top", + "value": "hexagon_pointed_top" + }, + { + "text": "Hexagon Flat Top", + "value": "hexagon_flat_top" + }, + { + "text": "Circle", + "value": "circle" + }, + { + "text": "Cross", + "value": "cross" + }, + { + "text": "Diamond", + "value": "diamond" + }, + { + "text": "Square", + "value": "square" + }, + { + "text": "Star", + "value": "star" + }, + { + "text": "Triangle", + "value": "triangle" + }, + { + "text": "Wye", + "value": "wye" + } + ], + "sortDirections": [ + { + "text": "Ascending", + "value": "asc" + }, + { + "text": "Descending", + "value": "desc" + } + ], + "sortFields": [ + { + "text": "Name", + "value": "name" + }, + { + "text": "Threshold Level", + "value": "thresholdLevel" + }, + { + "text": "Value", + "value": "value" + } + ], + "svgContainer": {}, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "C", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "D", + "scenarioId": "random_walk" + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "E", + "scenarioId": "random_walk" + } + ], + "thresholdStates": [ + { + "text": "ok", + "value": 0 + }, + { + "text": "warning", + "value": 1 + }, + { + "text": "critical", + "value": 2 + }, + { + "text": "custom", + "value": 3 + } + ], + "title": "Composite crash", + "type": "grafana-polystat-panel", + "unitFormats": [ + { + "submenu": [ + { + "text": "none", + "value": "none" + }, + { + "text": "short", + "value": "short" + }, + { + "text": "percent (0-100)", + "value": "percent" + }, + { + "text": "percent (0.0-1.0)", + "value": "percentunit" + }, + { + "text": "Humidity (%H)", + "value": "humidity" + }, + { + "text": "decibel", + "value": "dB" + }, + { + "text": "hexadecimal (0x)", + "value": "hex0x" + }, + { + "text": "hexadecimal", + "value": "hex" + }, + { + "text": "scientific notation", + "value": "sci" + }, + { + "text": "locale format", + "value": "locale" + } + ], + "text": "none" + }, + { + "submenu": [ + { + "text": "Dollars ($)", + "value": "currencyUSD" + }, + { + "text": "Pounds (£)", + "value": "currencyGBP" + }, + { + "text": "Euro (€)", + "value": "currencyEUR" + }, + { + "text": "Yen (¥)", + "value": "currencyJPY" + }, + { + "text": "Rubles (₽)", + "value": "currencyRUB" + }, + { + "text": "Hryvnias (₴)", + "value": "currencyUAH" + }, + { + "text": "Real (R$)", + "value": "currencyBRL" + }, + { + "text": "Danish Krone (kr)", + "value": "currencyDKK" + }, + { + "text": "Icelandic Króna (kr)", + "value": "currencyISK" + }, + { + "text": "Norwegian Krone (kr)", + "value": "currencyNOK" + }, + { + "text": "Swedish Krona (kr)", + "value": "currencySEK" + }, + { + "text": "Czech koruna (czk)", + "value": "currencyCZK" + }, + { + "text": "Swiss franc (CHF)", + "value": "currencyCHF" + }, + { + "text": "Polish Złoty (PLN)", + "value": "currencyPLN" + }, + { + "text": "Bitcoin (฿)", + "value": "currencyBTC" + } + ], + "text": "currency" + }, + { + "submenu": [ + { + "text": "Hertz (1/s)", + "value": "hertz" + }, + { + "text": "nanoseconds (ns)", + "value": "ns" + }, + { + "text": "microseconds (µs)", + "value": "µs" + }, + { + "text": "milliseconds (ms)", + "value": "ms" + }, + { + "text": "seconds (s)", + "value": "s" + }, + { + "text": "minutes (m)", + "value": "m" + }, + { + "text": "hours (h)", + "value": "h" + }, + { + "text": "days (d)", + "value": "d" + }, + { + "text": "duration (ms)", + "value": "dtdurationms" + }, + { + "text": "duration (s)", + "value": "dtdurations" + }, + { + "text": "duration (hh:mm:ss)", + "value": "dthms" + }, + { + "text": "Timeticks (s/100)", + "value": "timeticks" + } + ], + "text": "time" + }, + { + "submenu": [ + { + "text": "YYYY-MM-DD HH:mm:ss", + "value": "dateTimeAsIso" + }, + { + "text": "DD/MM/YYYY h:mm:ss a", + "value": "dateTimeAsUS" + }, + { + "text": "From Now", + "value": "dateTimeFromNow" + } + ], + "text": "date & time" + }, + { + "submenu": [ + { + "text": "bits", + "value": "bits" + }, + { + "text": "bytes", + "value": "bytes" + }, + { + "text": "kibibytes", + "value": "kbytes" + }, + { + "text": "mebibytes", + "value": "mbytes" + }, + { + "text": "gibibytes", + "value": "gbytes" + } + ], + "text": "data (IEC)" + }, + { + "submenu": [ + { + "text": "bits", + "value": "decbits" + }, + { + "text": "bytes", + "value": "decbytes" + }, + { + "text": "kilobytes", + "value": "deckbytes" + }, + { + "text": "megabytes", + "value": "decmbytes" + }, + { + "text": "gigabytes", + "value": "decgbytes" + } + ], + "text": "data (Metric)" + }, + { + "submenu": [ + { + "text": "packets/sec", + "value": "pps" + }, + { + "text": "bits/sec", + "value": "bps" + }, + { + "text": "bytes/sec", + "value": "Bps" + }, + { + "text": "kilobits/sec", + "value": "Kbits" + }, + { + "text": "kilobytes/sec", + "value": "KBs" + }, + { + "text": "megabits/sec", + "value": "Mbits" + }, + { + "text": "megabytes/sec", + "value": "MBs" + }, + { + "text": "gigabytes/sec", + "value": "GBs" + }, + { + "text": "gigabits/sec", + "value": "Gbits" + } + ], + "text": "data rate" + }, + { + "submenu": [ + { + "text": "hashes/sec", + "value": "Hs" + }, + { + "text": "kilohashes/sec", + "value": "KHs" + }, + { + "text": "megahashes/sec", + "value": "MHs" + }, + { + "text": "gigahashes/sec", + "value": "GHs" + }, + { + "text": "terahashes/sec", + "value": "THs" + }, + { + "text": "petahashes/sec", + "value": "PHs" + }, + { + "text": "exahashes/sec", + "value": "EHs" + } + ], + "text": "hash rate" + }, + { + "submenu": [ + { + "text": "ops/sec (ops)", + "value": "ops" + }, + { + "text": "requests/sec (rps)", + "value": "reqps" + }, + { + "text": "reads/sec (rps)", + "value": "rps" + }, + { + "text": "writes/sec (wps)", + "value": "wps" + }, + { + "text": "I/O ops/sec (iops)", + "value": "iops" + }, + { + "text": "ops/min (opm)", + "value": "opm" + }, + { + "text": "reads/min (rpm)", + "value": "rpm" + }, + { + "text": "writes/min (wpm)", + "value": "wpm" + } + ], + "text": "throughput" + }, + { + "submenu": [ + { + "text": "millimetre (mm)", + "value": "lengthmm" + }, + { + "text": "meter (m)", + "value": "lengthm" + }, + { + "text": "feet (ft)", + "value": "lengthft" + }, + { + "text": "kilometer (km)", + "value": "lengthkm" + }, + { + "text": "mile (mi)", + "value": "lengthmi" + } + ], + "text": "length" + }, + { + "submenu": [ + { + "text": "Square Meters (m²)", + "value": "areaM2" + }, + { + "text": "Square Feet (ft²)", + "value": "areaF2" + }, + { + "text": "Square Miles (mi²)", + "value": "areaMI2" + } + ], + "text": "area" + }, + { + "submenu": [ + { + "text": "milligram (mg)", + "value": "massmg" + }, + { + "text": "gram (g)", + "value": "massg" + }, + { + "text": "kilogram (kg)", + "value": "masskg" + }, + { + "text": "metric ton (t)", + "value": "masst" + } + ], + "text": "mass" + }, + { + "submenu": [ + { + "text": "metres/second (m/s)", + "value": "velocityms" + }, + { + "text": "kilometers/hour (km/h)", + "value": "velocitykmh" + }, + { + "text": "miles/hour (mph)", + "value": "velocitymph" + }, + { + "text": "knot (kn)", + "value": "velocityknot" + } + ], + "text": "velocity" + }, + { + "submenu": [ + { + "text": "millilitre (mL)", + "value": "mlitre" + }, + { + "text": "litre (L)", + "value": "litre" + }, + { + "text": "cubic metre", + "value": "m3" + }, + { + "text": "Normal cubic metre", + "value": "Nm3" + }, + { + "text": "cubic decimetre", + "value": "dm3" + }, + { + "text": "gallons", + "value": "gallons" + } + ], + "text": "volume" + }, + { + "submenu": [ + { + "text": "Watt (W)", + "value": "watt" + }, + { + "text": "Kilowatt (kW)", + "value": "kwatt" + }, + { + "text": "Milliwatt (mW)", + "value": "mwatt" + }, + { + "text": "Watt per square metre (W/m²)", + "value": "Wm2" + }, + { + "text": "Volt-ampere (VA)", + "value": "voltamp" + }, + { + "text": "Kilovolt-ampere (kVA)", + "value": "kvoltamp" + }, + { + "text": "Volt-ampere reactive (var)", + "value": "voltampreact" + }, + { + "text": "Kilovolt-ampere reactive (kvar)", + "value": "kvoltampreact" + }, + { + "text": "Watt-hour (Wh)", + "value": "watth" + }, + { + "text": "Kilowatt-hour (kWh)", + "value": "kwatth" + }, + { + "text": "Kilowatt-min (kWm)", + "value": "kwattm" + }, + { + "text": "Joule (J)", + "value": "joule" + }, + { + "text": "Electron volt (eV)", + "value": "ev" + }, + { + "text": "Ampere (A)", + "value": "amp" + }, + { + "text": "Kiloampere (kA)", + "value": "kamp" + }, + { + "text": "Milliampere (mA)", + "value": "mamp" + }, + { + "text": "Volt (V)", + "value": "volt" + }, + { + "text": "Kilovolt (kV)", + "value": "kvolt" + }, + { + "text": "Millivolt (mV)", + "value": "mvolt" + }, + { + "text": "Decibel-milliwatt (dBm)", + "value": "dBm" + }, + { + "text": "Ohm (Ω)", + "value": "ohm" + }, + { + "text": "Lumens (Lm)", + "value": "lumens" + } + ], + "text": "energy" + }, + { + "submenu": [ + { + "text": "Celsius (°C)", + "value": "celsius" + }, + { + "text": "Farenheit (°F)", + "value": "farenheit" + }, + { + "text": "Kelvin (K)", + "value": "kelvin" + } + ], + "text": "temperature" + }, + { + "submenu": [ + { + "text": "Millibars", + "value": "pressurembar" + }, + { + "text": "Bars", + "value": "pressurebar" + }, + { + "text": "Kilobars", + "value": "pressurekbar" + }, + { + "text": "Hectopascals", + "value": "pressurehpa" + }, + { + "text": "Kilopascals", + "value": "pressurekpa" + }, + { + "text": "Inches of mercury", + "value": "pressurehg" + }, + { + "text": "PSI", + "value": "pressurepsi" + } + ], + "text": "pressure" + }, + { + "submenu": [ + { + "text": "Newton-meters (Nm)", + "value": "forceNm" + }, + { + "text": "Kilonewton-meters (kNm)", + "value": "forcekNm" + }, + { + "text": "Newtons (N)", + "value": "forceN" + }, + { + "text": "Kilonewtons (kN)", + "value": "forcekN" + } + ], + "text": "force" + }, + { + "submenu": [ + { + "text": "Gallons/min (gpm)", + "value": "flowgpm" + }, + { + "text": "Cubic meters/sec (cms)", + "value": "flowcms" + }, + { + "text": "Cubic feet/sec (cfs)", + "value": "flowcfs" + }, + { + "text": "Cubic feet/min (cfm)", + "value": "flowcfm" + }, + { + "text": "Litre/hour", + "value": "litreh" + }, + { + "text": "Litre/min (l/min)", + "value": "flowlpm" + }, + { + "text": "milliLitre/min (mL/min)", + "value": "flowmlpm" + } + ], + "text": "flow" + }, + { + "submenu": [ + { + "text": "Degrees (°)", + "value": "degree" + }, + { + "text": "Radians", + "value": "radian" + }, + { + "text": "Gradian", + "value": "grad" + } + ], + "text": "angle" + }, + { + "submenu": [ + { + "text": "Meters/sec²", + "value": "accMS2" + }, + { + "text": "Feet/sec²", + "value": "accFS2" + }, + { + "text": "G unit", + "value": "accG" + } + ], + "text": "acceleration" + }, + { + "submenu": [ + { + "text": "Becquerel (Bq)", + "value": "radbq" + }, + { + "text": "curie (Ci)", + "value": "radci" + }, + { + "text": "Gray (Gy)", + "value": "radgy" + }, + { + "text": "rad", + "value": "radrad" + }, + { + "text": "Sievert (Sv)", + "value": "radsv" + }, + { + "text": "rem", + "value": "radrem" + }, + { + "text": "Exposure (C/kg)", + "value": "radexpckg" + }, + { + "text": "roentgen (R)", + "value": "radr" + }, + { + "text": "Sievert/hour (Sv/h)", + "value": "radsvh" + } + ], + "text": "radiation" + }, + { + "submenu": [ + { + "text": "parts-per-million (ppm)", + "value": "ppm" + }, + { + "text": "parts-per-billion (ppb)", + "value": "conppb" + }, + { + "text": "nanogram per cubic metre (ng/m³)", + "value": "conngm3" + }, + { + "text": "nanogram per normal cubic metre (ng/Nm³)", + "value": "conngNm3" + }, + { + "text": "microgram per cubic metre (μg/m³)", + "value": "conμgm3" + }, + { + "text": "microgram per normal cubic metre (μg/Nm³)", + "value": "conμgNm3" + }, + { + "text": "milligram per cubic metre (mg/m³)", + "value": "conmgm3" + }, + { + "text": "milligram per normal cubic metre (mg/Nm³)", + "value": "conmgNm3" + }, + { + "text": "gram per cubic metre (g/m³)", + "value": "congm3" + }, + { + "text": "gram per normal cubic metre (g/Nm³)", + "value": "congNm3" + } + ], + "text": "concentration" + } + ] + }, + { + "animationModes": [ + { + "text": "Show All", + "value": "all" + }, + { + "text": "Show Triggered", + "value": "triggered" + } + ], + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "d3DivId": "d3_svg_2", + "datasource": "gdev-testdata", + "decimals": 2, + "displayModes": [ + { + "text": "Show All", + "value": "all" + }, + { + "text": "Show Triggered", + "value": "triggered" + } + ], + "fontSizes": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70 + ], + "fontTypes": [ + "Open Sans", + "Arial", + "Avant Garde", + "Bookman", + "Consolas", + "Courier", + "Courier New", + "Futura", + "Garamond", + "Helvetica", + "Palatino", + "Times", + "Times New Roman", + "Verdana" + ], + "format": "none", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "links": [], + "notcolors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "operatorName": "avg", + "operatorOptions": [ + { + "text": "Average", + "value": "avg" + }, + { + "text": "Count", + "value": "count" + }, + { + "text": "Current", + "value": "current" + }, + { + "text": "Delta", + "value": "delta" + }, + { + "text": "Difference", + "value": "diff" + }, + { + "text": "First", + "value": "first" + }, + { + "text": "Log Min", + "value": "logmin" + }, + { + "text": "Max", + "value": "max" + }, + { + "text": "Min", + "value": "min" + }, + { + "text": "Name", + "value": "name" + }, + { + "text": "Time of Last Point", + "value": "last_time" + }, + { + "text": "Time Step", + "value": "time_step" + }, + { + "text": "Total", + "value": "total" + } + ], + "polystat": { + "animationSpeed": 2500, + "columnAutoSize": true, + "columns": 1, + "defaultClickThrough": "", + "defaultClickThroughSanitize": true, + "displayLimit": 100, + "fontAutoScale": true, + "fontSize": 12, + "globalDisplayMode": "all", + "globalOperatorName": "avg", + "gradientEnabled": true, + "hexagonSortByDirection": "asc", + "hexagonSortByField": "name", + "maxMetrics": 0, + "polygonBorderColor": "black", + "polygonBorderSize": 2, + "radius": "", + "radiusAutoSize": true, + "rowAutoSize": true, + "rows": 1, + "shape": "hexagon_pointed_top", + "tooltipDisplayMode": "all", + "tooltipDisplayTextTriggeredEmpty": "OK", + "tooltipFontSize": 12, + "tooltipFontType": "Open Sans", + "tooltipPrimarySortDirection": "desc", + "tooltipPrimarySortField": "thresholdLevel", + "tooltipSecondarySortDirection": "desc", + "tooltipSecondarySortField": "value", + "tooltipTimestampEnabled": true + }, + "savedComposites": [], + "savedOverrides": [], + "shapes": [ + { + "text": "Hexagon Pointed Top", + "value": "hexagon_pointed_top" + }, + { + "text": "Hexagon Flat Top", + "value": "hexagon_flat_top" + }, + { + "text": "Circle", + "value": "circle" + }, + { + "text": "Cross", + "value": "cross" + }, + { + "text": "Diamond", + "value": "diamond" + }, + { + "text": "Square", + "value": "square" + }, + { + "text": "Star", + "value": "star" + }, + { + "text": "Triangle", + "value": "triangle" + }, + { + "text": "Wye", + "value": "wye" + } + ], + "sortDirections": [ + { + "text": "Ascending", + "value": "asc" + }, + { + "text": "Descending", + "value": "desc" + } + ], + "sortFields": [ + { + "text": "Name", + "value": "name" + }, + { + "text": "Threshold Level", + "value": "thresholdLevel" + }, + { + "text": "Value", + "value": "value" + } + ], + "svgContainer": {}, + "targets": [ + { + "alias": "Sensor-A", + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0" + }, + { + "alias": "Sensor-B", + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "3433,23432,55" + }, + { + "alias": "Sensor-C", + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "C", + "scenarioId": "csv_metric_values", + "stringInput": "1,2,3,4,5,6" + }, + { + "alias": "Sensor-E", + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "D", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0" + } + ], + "thresholdStates": [ + { + "text": "ok", + "value": 0 + }, + { + "text": "warning", + "value": 1 + }, + { + "text": "critical", + "value": 2 + }, + { + "text": "custom", + "value": 3 + } + ], + "title": "No Value in Sensor-C Bug", + "type": "grafana-polystat-panel", + "unitFormats": [ + { + "submenu": [ + { + "text": "none", + "value": "none" + }, + { + "text": "short", + "value": "short" + }, + { + "text": "percent (0-100)", + "value": "percent" + }, + { + "text": "percent (0.0-1.0)", + "value": "percentunit" + }, + { + "text": "Humidity (%H)", + "value": "humidity" + }, + { + "text": "decibel", + "value": "dB" + }, + { + "text": "hexadecimal (0x)", + "value": "hex0x" + }, + { + "text": "hexadecimal", + "value": "hex" + }, + { + "text": "scientific notation", + "value": "sci" + }, + { + "text": "locale format", + "value": "locale" + } + ], + "text": "none" + }, + { + "submenu": [ + { + "text": "Dollars ($)", + "value": "currencyUSD" + }, + { + "text": "Pounds (£)", + "value": "currencyGBP" + }, + { + "text": "Euro (€)", + "value": "currencyEUR" + }, + { + "text": "Yen (¥)", + "value": "currencyJPY" + }, + { + "text": "Rubles (₽)", + "value": "currencyRUB" + }, + { + "text": "Hryvnias (₴)", + "value": "currencyUAH" + }, + { + "text": "Real (R$)", + "value": "currencyBRL" + }, + { + "text": "Danish Krone (kr)", + "value": "currencyDKK" + }, + { + "text": "Icelandic Króna (kr)", + "value": "currencyISK" + }, + { + "text": "Norwegian Krone (kr)", + "value": "currencyNOK" + }, + { + "text": "Swedish Krona (kr)", + "value": "currencySEK" + }, + { + "text": "Czech koruna (czk)", + "value": "currencyCZK" + }, + { + "text": "Swiss franc (CHF)", + "value": "currencyCHF" + }, + { + "text": "Polish Złoty (PLN)", + "value": "currencyPLN" + }, + { + "text": "Bitcoin (฿)", + "value": "currencyBTC" + } + ], + "text": "currency" + }, + { + "submenu": [ + { + "text": "Hertz (1/s)", + "value": "hertz" + }, + { + "text": "nanoseconds (ns)", + "value": "ns" + }, + { + "text": "microseconds (µs)", + "value": "µs" + }, + { + "text": "milliseconds (ms)", + "value": "ms" + }, + { + "text": "seconds (s)", + "value": "s" + }, + { + "text": "minutes (m)", + "value": "m" + }, + { + "text": "hours (h)", + "value": "h" + }, + { + "text": "days (d)", + "value": "d" + }, + { + "text": "duration (ms)", + "value": "dtdurationms" + }, + { + "text": "duration (s)", + "value": "dtdurations" + }, + { + "text": "duration (hh:mm:ss)", + "value": "dthms" + }, + { + "text": "Timeticks (s/100)", + "value": "timeticks" + } + ], + "text": "time" + }, + { + "submenu": [ + { + "text": "YYYY-MM-DD HH:mm:ss", + "value": "dateTimeAsIso" + }, + { + "text": "DD/MM/YYYY h:mm:ss a", + "value": "dateTimeAsUS" + }, + { + "text": "From Now", + "value": "dateTimeFromNow" + } + ], + "text": "date & time" + }, + { + "submenu": [ + { + "text": "bits", + "value": "bits" + }, + { + "text": "bytes", + "value": "bytes" + }, + { + "text": "kibibytes", + "value": "kbytes" + }, + { + "text": "mebibytes", + "value": "mbytes" + }, + { + "text": "gibibytes", + "value": "gbytes" + } + ], + "text": "data (IEC)" + }, + { + "submenu": [ + { + "text": "bits", + "value": "decbits" + }, + { + "text": "bytes", + "value": "decbytes" + }, + { + "text": "kilobytes", + "value": "deckbytes" + }, + { + "text": "megabytes", + "value": "decmbytes" + }, + { + "text": "gigabytes", + "value": "decgbytes" + } + ], + "text": "data (Metric)" + }, + { + "submenu": [ + { + "text": "packets/sec", + "value": "pps" + }, + { + "text": "bits/sec", + "value": "bps" + }, + { + "text": "bytes/sec", + "value": "Bps" + }, + { + "text": "kilobits/sec", + "value": "Kbits" + }, + { + "text": "kilobytes/sec", + "value": "KBs" + }, + { + "text": "megabits/sec", + "value": "Mbits" + }, + { + "text": "megabytes/sec", + "value": "MBs" + }, + { + "text": "gigabytes/sec", + "value": "GBs" + }, + { + "text": "gigabits/sec", + "value": "Gbits" + } + ], + "text": "data rate" + }, + { + "submenu": [ + { + "text": "hashes/sec", + "value": "Hs" + }, + { + "text": "kilohashes/sec", + "value": "KHs" + }, + { + "text": "megahashes/sec", + "value": "MHs" + }, + { + "text": "gigahashes/sec", + "value": "GHs" + }, + { + "text": "terahashes/sec", + "value": "THs" + }, + { + "text": "petahashes/sec", + "value": "PHs" + }, + { + "text": "exahashes/sec", + "value": "EHs" + } + ], + "text": "hash rate" + }, + { + "submenu": [ + { + "text": "ops/sec (ops)", + "value": "ops" + }, + { + "text": "requests/sec (rps)", + "value": "reqps" + }, + { + "text": "reads/sec (rps)", + "value": "rps" + }, + { + "text": "writes/sec (wps)", + "value": "wps" + }, + { + "text": "I/O ops/sec (iops)", + "value": "iops" + }, + { + "text": "ops/min (opm)", + "value": "opm" + }, + { + "text": "reads/min (rpm)", + "value": "rpm" + }, + { + "text": "writes/min (wpm)", + "value": "wpm" + } + ], + "text": "throughput" + }, + { + "submenu": [ + { + "text": "millimetre (mm)", + "value": "lengthmm" + }, + { + "text": "meter (m)", + "value": "lengthm" + }, + { + "text": "feet (ft)", + "value": "lengthft" + }, + { + "text": "kilometer (km)", + "value": "lengthkm" + }, + { + "text": "mile (mi)", + "value": "lengthmi" + } + ], + "text": "length" + }, + { + "submenu": [ + { + "text": "Square Meters (m²)", + "value": "areaM2" + }, + { + "text": "Square Feet (ft²)", + "value": "areaF2" + }, + { + "text": "Square Miles (mi²)", + "value": "areaMI2" + } + ], + "text": "area" + }, + { + "submenu": [ + { + "text": "milligram (mg)", + "value": "massmg" + }, + { + "text": "gram (g)", + "value": "massg" + }, + { + "text": "kilogram (kg)", + "value": "masskg" + }, + { + "text": "metric ton (t)", + "value": "masst" + } + ], + "text": "mass" + }, + { + "submenu": [ + { + "text": "metres/second (m/s)", + "value": "velocityms" + }, + { + "text": "kilometers/hour (km/h)", + "value": "velocitykmh" + }, + { + "text": "miles/hour (mph)", + "value": "velocitymph" + }, + { + "text": "knot (kn)", + "value": "velocityknot" + } + ], + "text": "velocity" + }, + { + "submenu": [ + { + "text": "millilitre (mL)", + "value": "mlitre" + }, + { + "text": "litre (L)", + "value": "litre" + }, + { + "text": "cubic metre", + "value": "m3" + }, + { + "text": "Normal cubic metre", + "value": "Nm3" + }, + { + "text": "cubic decimetre", + "value": "dm3" + }, + { + "text": "gallons", + "value": "gallons" + } + ], + "text": "volume" + }, + { + "submenu": [ + { + "text": "Watt (W)", + "value": "watt" + }, + { + "text": "Kilowatt (kW)", + "value": "kwatt" + }, + { + "text": "Milliwatt (mW)", + "value": "mwatt" + }, + { + "text": "Watt per square metre (W/m²)", + "value": "Wm2" + }, + { + "text": "Volt-ampere (VA)", + "value": "voltamp" + }, + { + "text": "Kilovolt-ampere (kVA)", + "value": "kvoltamp" + }, + { + "text": "Volt-ampere reactive (var)", + "value": "voltampreact" + }, + { + "text": "Kilovolt-ampere reactive (kvar)", + "value": "kvoltampreact" + }, + { + "text": "Watt-hour (Wh)", + "value": "watth" + }, + { + "text": "Kilowatt-hour (kWh)", + "value": "kwatth" + }, + { + "text": "Kilowatt-min (kWm)", + "value": "kwattm" + }, + { + "text": "Joule (J)", + "value": "joule" + }, + { + "text": "Electron volt (eV)", + "value": "ev" + }, + { + "text": "Ampere (A)", + "value": "amp" + }, + { + "text": "Kiloampere (kA)", + "value": "kamp" + }, + { + "text": "Milliampere (mA)", + "value": "mamp" + }, + { + "text": "Volt (V)", + "value": "volt" + }, + { + "text": "Kilovolt (kV)", + "value": "kvolt" + }, + { + "text": "Millivolt (mV)", + "value": "mvolt" + }, + { + "text": "Decibel-milliwatt (dBm)", + "value": "dBm" + }, + { + "text": "Ohm (Ω)", + "value": "ohm" + }, + { + "text": "Lumens (Lm)", + "value": "lumens" + } + ], + "text": "energy" + }, + { + "submenu": [ + { + "text": "Celsius (°C)", + "value": "celsius" + }, + { + "text": "Farenheit (°F)", + "value": "farenheit" + }, + { + "text": "Kelvin (K)", + "value": "kelvin" + } + ], + "text": "temperature" + }, + { + "submenu": [ + { + "text": "Millibars", + "value": "pressurembar" + }, + { + "text": "Bars", + "value": "pressurebar" + }, + { + "text": "Kilobars", + "value": "pressurekbar" + }, + { + "text": "Hectopascals", + "value": "pressurehpa" + }, + { + "text": "Kilopascals", + "value": "pressurekpa" + }, + { + "text": "Inches of mercury", + "value": "pressurehg" + }, + { + "text": "PSI", + "value": "pressurepsi" + } + ], + "text": "pressure" + }, + { + "submenu": [ + { + "text": "Newton-meters (Nm)", + "value": "forceNm" + }, + { + "text": "Kilonewton-meters (kNm)", + "value": "forcekNm" + }, + { + "text": "Newtons (N)", + "value": "forceN" + }, + { + "text": "Kilonewtons (kN)", + "value": "forcekN" + } + ], + "text": "force" + }, + { + "submenu": [ + { + "text": "Gallons/min (gpm)", + "value": "flowgpm" + }, + { + "text": "Cubic meters/sec (cms)", + "value": "flowcms" + }, + { + "text": "Cubic feet/sec (cfs)", + "value": "flowcfs" + }, + { + "text": "Cubic feet/min (cfm)", + "value": "flowcfm" + }, + { + "text": "Litre/hour", + "value": "litreh" + }, + { + "text": "Litre/min (l/min)", + "value": "flowlpm" + }, + { + "text": "milliLitre/min (mL/min)", + "value": "flowmlpm" + } + ], + "text": "flow" + }, + { + "submenu": [ + { + "text": "Degrees (°)", + "value": "degree" + }, + { + "text": "Radians", + "value": "radian" + }, + { + "text": "Gradian", + "value": "grad" + } + ], + "text": "angle" + }, + { + "submenu": [ + { + "text": "Meters/sec²", + "value": "accMS2" + }, + { + "text": "Feet/sec²", + "value": "accFS2" + }, + { + "text": "G unit", + "value": "accG" + } + ], + "text": "acceleration" + }, + { + "submenu": [ + { + "text": "Becquerel (Bq)", + "value": "radbq" + }, + { + "text": "curie (Ci)", + "value": "radci" + }, + { + "text": "Gray (Gy)", + "value": "radgy" + }, + { + "text": "rad", + "value": "radrad" + }, + { + "text": "Sievert (Sv)", + "value": "radsv" + }, + { + "text": "rem", + "value": "radrem" + }, + { + "text": "Exposure (C/kg)", + "value": "radexpckg" + }, + { + "text": "roentgen (R)", + "value": "radr" + }, + { + "text": "Sievert/hour (Sv/h)", + "value": "radsvh" + } + ], + "text": "radiation" + }, + { + "submenu": [ + { + "text": "parts-per-million (ppm)", + "value": "ppm" + }, + { + "text": "parts-per-billion (ppb)", + "value": "conppb" + }, + { + "text": "nanogram per cubic metre (ng/m³)", + "value": "conngm3" + }, + { + "text": "nanogram per normal cubic metre (ng/Nm³)", + "value": "conngNm3" + }, + { + "text": "microgram per cubic metre (μg/m³)", + "value": "conμgm3" + }, + { + "text": "microgram per normal cubic metre (μg/Nm³)", + "value": "conμgNm3" + }, + { + "text": "milligram per cubic metre (mg/m³)", + "value": "conmgm3" + }, + { + "text": "milligram per normal cubic metre (mg/Nm³)", + "value": "conmgNm3" + }, + { + "text": "gram per cubic metre (g/m³)", + "value": "congm3" + }, + { + "text": "gram per normal cubic metre (g/Nm³)", + "value": "congNm3" + } + ], + "text": "concentration" + } + ] + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [ + "panel-test", + "gdev" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Panel Tests - Polystat", + "uid": "Kp9Z0hTik", + "version": 5 +} diff --git a/devenv/dev-dashboards/panel_tests_singlestat.json b/devenv/dev-dashboards/panel_tests_singlestat.json new file mode 100644 index 00000000000..2d69f27bcb6 --- /dev/null +++ b/devenv/dev-dashboards/panel_tests_singlestat.json @@ -0,0 +1,574 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "gdev-testdata", + "decimals": null, + "description": "", + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "postfix", + "postfixFontSize": "50%", + "prefix": "prefix", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,2,3,4,5" + } + ], + "thresholds": "5,10", + "title": "prefix 3 ms (green) postfixt + sparkline", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPrefix": false, + "colorValue": true, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "gdev-testdata", + "decimals": null, + "description": "", + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,2,3,4,5" + } + ], + "thresholds": "5,10", + "title": "3 ms (red) + full height sparkline", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "gdev-testdata", + "decimals": null, + "description": "", + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,2,3,4,5" + } + ], + "thresholds": "5,10", + "title": "3 ms + red background", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPrefix": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "gdev-testdata", + "decimals": null, + "description": "", + "format": "ms", + "gauge": { + "maxValue": 150, + "minValue": 0, + "show": true, + "thresholdLabels": true, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "10,20,80" + } + ], + "thresholds": "81,90", + "title": "80 ms green gauge, thresholds 81, 90", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPrefix": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "gdev-testdata", + "decimals": null, + "description": "", + "format": "ms", + "gauge": { + "maxValue": 150, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "10,20,80" + } + ], + "thresholds": "81,90", + "title": "80 ms green gauge, thresholds 81, 90, no labels", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPrefix": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "gdev-testdata", + "decimals": null, + "description": "", + "format": "ms", + "gauge": { + "maxValue": 150, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "10,20,80" + } + ], + "thresholds": "81,90", + "title": "80 ms green gauge, thresholds 81, 90, no markers or labels", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "refresh": false, + "revision": 8, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "gdev", + "panel-tests" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Panel Tests - Singlestat", + "uid": "singlestat", + "version": 14 +} diff --git a/devenv/dev-dashboards/panel_tests_slow_queries_and_annotations.json b/devenv/dev-dashboards/panel_tests_slow_queries_and_annotations.json new file mode 100644 index 00000000000..08bf6dce9d0 --- /dev/null +++ b/devenv/dev-dashboards/panel_tests_slow_queries_and_annotations.json @@ -0,0 +1,1166 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "-- Grafana --", + "enable": true, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "matchAny": false, + "name": "annotations", + "showIn": 0, + "tags": [ + "asd" + ], + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 7, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "30s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 0 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "30s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "30s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "30s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "30s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 6, + "w": 16, + "x": 0, + "y": 24 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "gdev-testdata", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "slow_query", + "stringInput": "5s" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Panel tests - Slow Queries & Annotations", + "uid": "xtY_uCAiz", + "version": 11 +} diff --git a/devenv/dev-dashboards/panel_tests_table.json b/devenv/dev-dashboards/panel_tests_table.json new file mode 100644 index 00000000000..ff0288c340a --- /dev/null +++ b/devenv/dev-dashboards/panel_tests_table.json @@ -0,0 +1,559 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "gdev-testdata", + "fontSize": "100%", + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "links": [], + "pageSize": 10, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "ColorCell", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "ColorValue", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "server1", + "expr": "", + "format": "table", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0,20,10" + }, + { + "alias": "server2", + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0" + } + ], + "title": "Time series to rows (2 pages)", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + }, + { + "text": "Max", + "value": "max" + }, + { + "text": "Current", + "value": "current" + } + ], + "datasource": "gdev-testdata", + "fontSize": "100%", + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "links": [], + "pageSize": 10, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "ColorCell", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "ColorValue", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "server1", + "expr": "", + "format": "table", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0,20,10" + }, + { + "alias": "server2", + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0" + } + ], + "title": "Time series aggregations", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-testdata", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 5, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": "row", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "/Color/", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "ColorValue", + "expr": "", + "format": "table", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0,20,10" + } + ], + "title": "color row by threshold", + "transform": "timeseries_to_columns", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-testdata", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "ColorCell", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "ColorValue", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "ColorValue", + "expr": "", + "format": "table", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "1,20,90,30,5,0,20,10" + }, + { + "alias": "ColorCell", + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "5,1,2,3,4,5,10,20" + } + ], + "title": "Column style thresholds & units", + "transform": "timeseries_to_columns", + "type": "table" + }, + { + "columns": [], + "datasource": "gdev-testdata", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 6, + "links": [], + "pageSize": 20, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.5)", + "rgba(237, 129, 40, 0.5)", + "rgba(50, 172, 45, 0.5)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "http://www.grafana.com", + "mappingType": 1, + "pattern": "ColorCell", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.5)", + "rgba(237, 129, 40, 0.5)", + "rgba(50, 172, 45, 0.5)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkUrl": "http://www.grafana.com", + "mappingType": 1, + "pattern": "ColorValue", + "thresholds": [ + "5", + "10" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "ColorValue", + "expr": "", + "format": "table", + "intervalFactor": 1, + "refId": "A", + "scenarioId": "csv_metric_values", + "stringInput": "null,1,20,90,30,5,0,20,10" + }, + { + "alias": "ColorCell", + "refId": "B", + "scenarioId": "csv_metric_values", + "stringInput": "null,5,1,2,3,4,5,10,20" + } + ], + "title": "Column style thresholds and links", + "transform": "timeseries_to_columns", + "type": "table" + } + ], + "refresh": false, + "revision": 8, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "gdev", + "panel-tests" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Panel Tests - Table", + "uid": "pttable", + "version": 2 +} \ No newline at end of file diff --git a/public/app/plugins/app/testdata/dashboards/alerts.json b/devenv/dev-dashboards/testdata_alerts.json similarity index 98% rename from public/app/plugins/app/testdata/dashboards/alerts.json rename to devenv/dev-dashboards/testdata_alerts.json index 159df0f458b..8c2edebf155 100644 --- a/public/app/plugins/app/testdata/dashboards/alerts.json +++ b/devenv/dev-dashboards/testdata_alerts.json @@ -1,6 +1,6 @@ { "revision": 2, - "title": "TestData - Alerts", + "title": "Alerting with TestData", "tags": [ "grafana-test" ], @@ -48,7 +48,7 @@ }, "aliasColors": {}, "bars": false, - "datasource": "Grafana TestData", + "datasource": "gdev-testdata", "editable": true, "error": false, "fill": 1, @@ -161,7 +161,7 @@ }, "aliasColors": {}, "bars": false, - "datasource": "Grafana TestData", + "datasource": "gdev-testdata", "editable": true, "error": false, "fill": 1, diff --git a/docker/blocks/apache_proxy/Dockerfile b/devenv/docker/blocks/apache_proxy/Dockerfile similarity index 100% rename from docker/blocks/apache_proxy/Dockerfile rename to devenv/docker/blocks/apache_proxy/Dockerfile diff --git a/docker/blocks/apache_proxy/docker-compose.yaml b/devenv/docker/blocks/apache_proxy/docker-compose.yaml similarity index 72% rename from docker/blocks/apache_proxy/docker-compose.yaml rename to devenv/docker/blocks/apache_proxy/docker-compose.yaml index 2aec3d4bc4f..3791213f05a 100644 --- a/docker/blocks/apache_proxy/docker-compose.yaml +++ b/devenv/docker/blocks/apache_proxy/docker-compose.yaml @@ -2,8 +2,8 @@ # http://localhost:3000 (Grafana running locally) # # Please note that you'll need to change the root_url in the Grafana configuration: -# root_url = %(protocol)s://%(domain)s:/grafana/ +# root_url = %(protocol)s://%(domain)s:10081/grafana/ apacheproxy: - build: blocks/apache_proxy + build: docker/blocks/apache_proxy network_mode: host diff --git a/docker/blocks/apache_proxy/ports.conf b/devenv/docker/blocks/apache_proxy/ports.conf similarity index 100% rename from docker/blocks/apache_proxy/ports.conf rename to devenv/docker/blocks/apache_proxy/ports.conf diff --git a/docker/blocks/apache_proxy/proxy.conf b/devenv/docker/blocks/apache_proxy/proxy.conf similarity index 100% rename from docker/blocks/apache_proxy/proxy.conf rename to devenv/docker/blocks/apache_proxy/proxy.conf diff --git a/docker/blocks/collectd/Dockerfile b/devenv/docker/blocks/collectd/Dockerfile similarity index 100% rename from docker/blocks/collectd/Dockerfile rename to devenv/docker/blocks/collectd/Dockerfile diff --git a/docker/blocks/collectd/README.md b/devenv/docker/blocks/collectd/README.md similarity index 100% rename from docker/blocks/collectd/README.md rename to devenv/docker/blocks/collectd/README.md diff --git a/docker/blocks/collectd/collectd.conf.tpl b/devenv/docker/blocks/collectd/collectd.conf.tpl similarity index 100% rename from docker/blocks/collectd/collectd.conf.tpl rename to devenv/docker/blocks/collectd/collectd.conf.tpl diff --git a/docker/blocks/collectd/docker-compose.yaml b/devenv/docker/blocks/collectd/docker-compose.yaml similarity index 87% rename from docker/blocks/collectd/docker-compose.yaml rename to devenv/docker/blocks/collectd/docker-compose.yaml index c95827f7928..c5e189b58d8 100644 --- a/docker/blocks/collectd/docker-compose.yaml +++ b/devenv/docker/blocks/collectd/docker-compose.yaml @@ -1,5 +1,5 @@ collectd: - build: blocks/collectd + build: docker/blocks/collectd environment: HOST_NAME: myserver GRAPHITE_HOST: graphite diff --git a/docker/blocks/collectd/etc_mtab b/devenv/docker/blocks/collectd/etc_mtab similarity index 100% rename from docker/blocks/collectd/etc_mtab rename to devenv/docker/blocks/collectd/etc_mtab diff --git a/docker/blocks/collectd/start_container b/devenv/docker/blocks/collectd/start_container similarity index 100% rename from docker/blocks/collectd/start_container rename to devenv/docker/blocks/collectd/start_container diff --git a/docker/blocks/elastic/docker-compose.yaml b/devenv/docker/blocks/elastic/docker-compose.yaml similarity index 100% rename from docker/blocks/elastic/docker-compose.yaml rename to devenv/docker/blocks/elastic/docker-compose.yaml diff --git a/docker/blocks/elastic/elasticsearch.yml b/devenv/docker/blocks/elastic/elasticsearch.yml similarity index 100% rename from docker/blocks/elastic/elasticsearch.yml rename to devenv/docker/blocks/elastic/elasticsearch.yml diff --git a/docker/blocks/elastic1/docker-compose.yaml b/devenv/docker/blocks/elastic1/docker-compose.yaml similarity index 100% rename from docker/blocks/elastic1/docker-compose.yaml rename to devenv/docker/blocks/elastic1/docker-compose.yaml diff --git a/docker/blocks/elastic1/elasticsearch.yml b/devenv/docker/blocks/elastic1/elasticsearch.yml similarity index 100% rename from docker/blocks/elastic1/elasticsearch.yml rename to devenv/docker/blocks/elastic1/elasticsearch.yml diff --git a/docker/blocks/elastic5/docker-compose.yaml b/devenv/docker/blocks/elastic5/docker-compose.yaml similarity index 100% rename from docker/blocks/elastic5/docker-compose.yaml rename to devenv/docker/blocks/elastic5/docker-compose.yaml diff --git a/docker/blocks/elastic5/elasticsearch.yml b/devenv/docker/blocks/elastic5/elasticsearch.yml similarity index 100% rename from docker/blocks/elastic5/elasticsearch.yml rename to devenv/docker/blocks/elastic5/elasticsearch.yml diff --git a/devenv/docker/blocks/elastic6/docker-compose.yaml b/devenv/docker/blocks/elastic6/docker-compose.yaml new file mode 100644 index 00000000000..dd2439f88e4 --- /dev/null +++ b/devenv/docker/blocks/elastic6/docker-compose.yaml @@ -0,0 +1,15 @@ +# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine + + elasticsearch6: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 + command: elasticsearch + ports: + - "11200:9200" + - "11300:9300" + + fake-elastic6-data: + image: grafana/fake-data-gen + network_mode: bridge + environment: + FD_DATASOURCE: elasticsearch6 + FD_PORT: 11200 diff --git a/devenv/docker/blocks/elastic6/elasticsearch.yml b/devenv/docker/blocks/elastic6/elasticsearch.yml new file mode 100644 index 00000000000..c57b2c12908 --- /dev/null +++ b/devenv/docker/blocks/elastic6/elasticsearch.yml @@ -0,0 +1,2 @@ +script.inline: on +script.indexed: on diff --git a/docker/blocks/graphite/Dockerfile b/devenv/docker/blocks/graphite/Dockerfile similarity index 100% rename from docker/blocks/graphite/Dockerfile rename to devenv/docker/blocks/graphite/Dockerfile diff --git a/docker/blocks/graphite/docker-compose.yaml b/devenv/docker/blocks/graphite/docker-compose.yaml similarity index 89% rename from docker/blocks/graphite/docker-compose.yaml rename to devenv/docker/blocks/graphite/docker-compose.yaml index 606e28638f7..acebd2bd9c0 100644 --- a/docker/blocks/graphite/docker-compose.yaml +++ b/devenv/docker/blocks/graphite/docker-compose.yaml @@ -1,5 +1,5 @@ graphite09: - build: blocks/graphite + build: docker/blocks/graphite ports: - "8080:80" - "2003:2003" diff --git a/docker/blocks/graphite/files/carbon.conf b/devenv/docker/blocks/graphite/files/carbon.conf similarity index 97% rename from docker/blocks/graphite/files/carbon.conf rename to devenv/docker/blocks/graphite/files/carbon.conf index 50762b3fff5..fc03aba6398 100644 --- a/docker/blocks/graphite/files/carbon.conf +++ b/devenv/docker/blocks/graphite/files/carbon.conf @@ -38,7 +38,7 @@ CACHE_QUERY_PORT = 7002 LOG_UPDATES = False -# Enable AMQP if you want to receve metrics using an amqp broker +# Enable AMQP if you want to receive metrics using an amqp broker # ENABLE_AMQP = False # Verbose means a line will be logged for every metric received diff --git a/docker/blocks/graphite/files/events_views.py b/devenv/docker/blocks/graphite/files/events_views.py similarity index 100% rename from docker/blocks/graphite/files/events_views.py rename to devenv/docker/blocks/graphite/files/events_views.py diff --git a/docker/blocks/graphite/files/initial_data.json b/devenv/docker/blocks/graphite/files/initial_data.json similarity index 100% rename from docker/blocks/graphite/files/initial_data.json rename to devenv/docker/blocks/graphite/files/initial_data.json diff --git a/docker/blocks/graphite/files/local_settings.py b/devenv/docker/blocks/graphite/files/local_settings.py similarity index 100% rename from docker/blocks/graphite/files/local_settings.py rename to devenv/docker/blocks/graphite/files/local_settings.py diff --git a/docker/blocks/graphite/files/my_htpasswd b/devenv/docker/blocks/graphite/files/my_htpasswd similarity index 100% rename from docker/blocks/graphite/files/my_htpasswd rename to devenv/docker/blocks/graphite/files/my_htpasswd diff --git a/docker/blocks/graphite/files/nginx.conf b/devenv/docker/blocks/graphite/files/nginx.conf similarity index 100% rename from docker/blocks/graphite/files/nginx.conf rename to devenv/docker/blocks/graphite/files/nginx.conf diff --git a/docker/blocks/graphite/files/statsd_config.js b/devenv/docker/blocks/graphite/files/statsd_config.js similarity index 100% rename from docker/blocks/graphite/files/statsd_config.js rename to devenv/docker/blocks/graphite/files/statsd_config.js diff --git a/docker/blocks/graphite/files/storage-aggregation.conf b/devenv/docker/blocks/graphite/files/storage-aggregation.conf similarity index 100% rename from docker/blocks/graphite/files/storage-aggregation.conf rename to devenv/docker/blocks/graphite/files/storage-aggregation.conf diff --git a/docker/blocks/graphite/files/storage-schemas.conf b/devenv/docker/blocks/graphite/files/storage-schemas.conf similarity index 100% rename from docker/blocks/graphite/files/storage-schemas.conf rename to devenv/docker/blocks/graphite/files/storage-schemas.conf diff --git a/docker/blocks/graphite/files/supervisord.conf b/devenv/docker/blocks/graphite/files/supervisord.conf similarity index 100% rename from docker/blocks/graphite/files/supervisord.conf rename to devenv/docker/blocks/graphite/files/supervisord.conf diff --git a/docker/blocks/graphite1/Dockerfile b/devenv/docker/blocks/graphite1/Dockerfile similarity index 100% rename from docker/blocks/graphite1/Dockerfile rename to devenv/docker/blocks/graphite1/Dockerfile diff --git a/docker/blocks/graphite1/big-dashboard.json b/devenv/docker/blocks/graphite1/big-dashboard.json similarity index 100% rename from docker/blocks/graphite1/big-dashboard.json rename to devenv/docker/blocks/graphite1/big-dashboard.json diff --git a/docker/blocks/graphite1/conf/etc/logrotate.d/graphite-statsd b/devenv/docker/blocks/graphite1/conf/etc/logrotate.d/graphite-statsd similarity index 100% rename from docker/blocks/graphite1/conf/etc/logrotate.d/graphite-statsd rename to devenv/docker/blocks/graphite1/conf/etc/logrotate.d/graphite-statsd diff --git a/docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh b/devenv/docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh similarity index 100% rename from docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh rename to devenv/docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh diff --git a/docker/blocks/graphite1/conf/etc/nginx/nginx.conf b/devenv/docker/blocks/graphite1/conf/etc/nginx/nginx.conf similarity index 100% rename from docker/blocks/graphite1/conf/etc/nginx/nginx.conf rename to devenv/docker/blocks/graphite1/conf/etc/nginx/nginx.conf diff --git a/docker/blocks/graphite1/conf/etc/nginx/sites-enabled/graphite-statsd.conf b/devenv/docker/blocks/graphite1/conf/etc/nginx/sites-enabled/graphite-statsd.conf similarity index 100% rename from docker/blocks/graphite1/conf/etc/nginx/sites-enabled/graphite-statsd.conf rename to devenv/docker/blocks/graphite1/conf/etc/nginx/sites-enabled/graphite-statsd.conf diff --git a/docker/blocks/graphite1/conf/etc/service/carbon-aggregator/run b/devenv/docker/blocks/graphite1/conf/etc/service/carbon-aggregator/run similarity index 100% rename from docker/blocks/graphite1/conf/etc/service/carbon-aggregator/run rename to devenv/docker/blocks/graphite1/conf/etc/service/carbon-aggregator/run diff --git a/docker/blocks/graphite1/conf/etc/service/carbon/run b/devenv/docker/blocks/graphite1/conf/etc/service/carbon/run similarity index 100% rename from docker/blocks/graphite1/conf/etc/service/carbon/run rename to devenv/docker/blocks/graphite1/conf/etc/service/carbon/run diff --git a/docker/blocks/graphite1/conf/etc/service/graphite/run b/devenv/docker/blocks/graphite1/conf/etc/service/graphite/run similarity index 100% rename from docker/blocks/graphite1/conf/etc/service/graphite/run rename to devenv/docker/blocks/graphite1/conf/etc/service/graphite/run diff --git a/docker/blocks/graphite1/conf/etc/service/nginx/run b/devenv/docker/blocks/graphite1/conf/etc/service/nginx/run similarity index 100% rename from docker/blocks/graphite1/conf/etc/service/nginx/run rename to devenv/docker/blocks/graphite1/conf/etc/service/nginx/run diff --git a/docker/blocks/graphite1/conf/etc/service/statsd/run b/devenv/docker/blocks/graphite1/conf/etc/service/statsd/run similarity index 100% rename from docker/blocks/graphite1/conf/etc/service/statsd/run rename to devenv/docker/blocks/graphite1/conf/etc/service/statsd/run diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/aggregation-rules.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/aggregation-rules.conf similarity index 96% rename from docker/blocks/graphite1/conf/opt/graphite/conf/aggregation-rules.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/aggregation-rules.conf index c9520124a2a..792bbfd6857 100644 --- a/docker/blocks/graphite1/conf/opt/graphite/conf/aggregation-rules.conf +++ b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/aggregation-rules.conf @@ -8,7 +8,7 @@ # 'avg'. The name of the aggregate metric will be derived from # 'output_template' filling in any captured fields from 'input_pattern'. # -# For example, if you're metric naming scheme is: +# For example, if your metric naming scheme is: # # .applications... # diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/blacklist.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/blacklist.conf similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/conf/blacklist.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/blacklist.conf diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.amqp.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.amqp.conf similarity index 97% rename from docker/blocks/graphite1/conf/opt/graphite/conf/carbon.amqp.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.amqp.conf index fc36328b25f..f8a53a61115 100644 --- a/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.amqp.conf +++ b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.amqp.conf @@ -41,7 +41,7 @@ PICKLE_RECEIVER_PORT = 2004 CACHE_QUERY_INTERFACE = 0.0.0.0 CACHE_QUERY_PORT = 7002 -# Enable AMQP if you want to receve metrics using you amqp broker +# Enable AMQP if you want to receive metrics using you amqp broker ENABLE_AMQP = True # Verbose means a line will be logged for every metric received diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf similarity index 99% rename from docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf index 3e10dcec9cf..6741932da37 100644 --- a/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf +++ b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf @@ -265,7 +265,7 @@ WHISPER_FALLOCATE_CREATE = True # CARBON_METRIC_PREFIX = carbon # CARBON_METRIC_INTERVAL = 60 -# Enable AMQP if you want to receve metrics using an amqp broker +# Enable AMQP if you want to receive metrics using an amqp broker # ENABLE_AMQP = False # Verbose means a line will be logged for every metric received diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/dashboard.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/dashboard.conf similarity index 96% rename from docker/blocks/graphite1/conf/opt/graphite/conf/dashboard.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/dashboard.conf index 2e1b0bc4db3..f558b273f57 100644 --- a/docker/blocks/graphite1/conf/opt/graphite/conf/dashboard.conf +++ b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/dashboard.conf @@ -30,7 +30,7 @@ give_completer_focus = shift-space # pertain only to specific metric types. # # The dashboard presents only metrics that fall into specified naming schemes -# defined in this file. This creates a simpler, more targetted view of the +# defined in this file. This creates a simpler, more targeted view of the # data. The general form for defining a naming scheme is as follows: # #[Metric Type] diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/graphTemplates.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/graphTemplates.conf similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/conf/graphTemplates.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/graphTemplates.conf diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/relay-rules.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/relay-rules.conf similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/conf/relay-rules.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/relay-rules.conf diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/rewrite-rules.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/rewrite-rules.conf similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/conf/rewrite-rules.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/rewrite-rules.conf diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/whitelist.conf b/devenv/docker/blocks/graphite1/conf/opt/graphite/conf/whitelist.conf similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/conf/whitelist.conf rename to devenv/docker/blocks/graphite1/conf/opt/graphite/conf/whitelist.conf diff --git a/docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/app_settings.py b/devenv/docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/app_settings.py similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/app_settings.py rename to devenv/docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/app_settings.py diff --git a/docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/local_settings.py b/devenv/docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/local_settings.py similarity index 100% rename from docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/local_settings.py rename to devenv/docker/blocks/graphite1/conf/opt/graphite/webapp/graphite/local_settings.py diff --git a/docker/blocks/graphite1/conf/opt/statsd/config.js b/devenv/docker/blocks/graphite1/conf/opt/statsd/config.js similarity index 100% rename from docker/blocks/graphite1/conf/opt/statsd/config.js rename to devenv/docker/blocks/graphite1/conf/opt/statsd/config.js diff --git a/docker/blocks/graphite1/conf/usr/local/bin/django_admin_init.exp b/devenv/docker/blocks/graphite1/conf/usr/local/bin/django_admin_init.exp similarity index 100% rename from docker/blocks/graphite1/conf/usr/local/bin/django_admin_init.exp rename to devenv/docker/blocks/graphite1/conf/usr/local/bin/django_admin_init.exp diff --git a/docker/blocks/graphite1/conf/usr/local/bin/manage.sh b/devenv/docker/blocks/graphite1/conf/usr/local/bin/manage.sh similarity index 100% rename from docker/blocks/graphite1/conf/usr/local/bin/manage.sh rename to devenv/docker/blocks/graphite1/conf/usr/local/bin/manage.sh diff --git a/docker/blocks/graphite1/docker-compose.yaml b/devenv/docker/blocks/graphite1/docker-compose.yaml similarity index 90% rename from docker/blocks/graphite1/docker-compose.yaml rename to devenv/docker/blocks/graphite1/docker-compose.yaml index cd10593f423..1fa3e738ba8 100644 --- a/docker/blocks/graphite1/docker-compose.yaml +++ b/devenv/docker/blocks/graphite1/docker-compose.yaml @@ -1,6 +1,6 @@ graphite: build: - context: blocks/graphite1 + context: docker/blocks/graphite1 args: version: master ports: diff --git a/docker/blocks/graphite11/big-dashboard.json b/devenv/docker/blocks/graphite11/big-dashboard.json similarity index 100% rename from docker/blocks/graphite11/big-dashboard.json rename to devenv/docker/blocks/graphite11/big-dashboard.json diff --git a/docker/blocks/graphite11/docker-compose.yaml b/devenv/docker/blocks/graphite11/docker-compose.yaml similarity index 100% rename from docker/blocks/graphite11/docker-compose.yaml rename to devenv/docker/blocks/graphite11/docker-compose.yaml diff --git a/docker/blocks/influxdb/docker-compose.yaml b/devenv/docker/blocks/influxdb/docker-compose.yaml similarity index 80% rename from docker/blocks/influxdb/docker-compose.yaml rename to devenv/docker/blocks/influxdb/docker-compose.yaml index 3434f5d09b9..e1727807d41 100644 --- a/docker/blocks/influxdb/docker-compose.yaml +++ b/devenv/docker/blocks/influxdb/docker-compose.yaml @@ -6,7 +6,7 @@ - "8083:8083" - "8086:8086" volumes: - - ./blocks/influxdb/influxdb.conf:/etc/influxdb/influxdb.conf + - ./docker/blocks/influxdb/influxdb.conf:/etc/influxdb/influxdb.conf fake-influxdb-data: image: grafana/fake-data-gen diff --git a/docker/blocks/influxdb/influxdb.conf b/devenv/docker/blocks/influxdb/influxdb.conf similarity index 100% rename from docker/blocks/influxdb/influxdb.conf rename to devenv/docker/blocks/influxdb/influxdb.conf diff --git a/docker/blocks/jaeger/docker-compose.yaml b/devenv/docker/blocks/jaeger/docker-compose.yaml similarity index 100% rename from docker/blocks/jaeger/docker-compose.yaml rename to devenv/docker/blocks/jaeger/docker-compose.yaml diff --git a/docker/blocks/memcached/docker-compose.yaml b/devenv/docker/blocks/memcached/docker-compose.yaml similarity index 100% rename from docker/blocks/memcached/docker-compose.yaml rename to devenv/docker/blocks/memcached/docker-compose.yaml diff --git a/docker/blocks/mssql/build/Dockerfile b/devenv/docker/blocks/mssql/build/Dockerfile similarity index 100% rename from docker/blocks/mssql/build/Dockerfile rename to devenv/docker/blocks/mssql/build/Dockerfile diff --git a/docker/blocks/mssql/build/entrypoint.sh b/devenv/docker/blocks/mssql/build/entrypoint.sh similarity index 100% rename from docker/blocks/mssql/build/entrypoint.sh rename to devenv/docker/blocks/mssql/build/entrypoint.sh diff --git a/docker/blocks/mssql/build/setup.sh b/devenv/docker/blocks/mssql/build/setup.sh similarity index 100% rename from docker/blocks/mssql/build/setup.sh rename to devenv/docker/blocks/mssql/build/setup.sh diff --git a/devenv/docker/blocks/mssql/build/setup.sql.template b/devenv/docker/blocks/mssql/build/setup.sql.template new file mode 100644 index 00000000000..5ff4e194df3 --- /dev/null +++ b/devenv/docker/blocks/mssql/build/setup.sql.template @@ -0,0 +1,26 @@ +CREATE LOGIN %%USER%% WITH PASSWORD = '%%PWD%%' +GO + +CREATE DATABASE %%DB%% +ON +( NAME = %%DB%%, + FILENAME = '/var/opt/mssql/data/%%DB%%.mdf', + SIZE = 500MB, + MAXSIZE = 1000MB, + FILEGROWTH = 100MB ) +LOG ON +( NAME = %%DB%%_log, + FILENAME = '/var/opt/mssql/data/%%DB%%_log.ldf', + SIZE = 500MB, + MAXSIZE = 1000MB, + FILEGROWTH = 100MB ); +GO + +USE %%DB%%; +GO + +CREATE USER %%USER%% FOR LOGIN %%USER%%; +GO + +EXEC sp_addrolemember 'db_owner', '%%USER%%'; +GO diff --git a/docker/blocks/mssql/docker-compose.yaml b/devenv/docker/blocks/mssql/docker-compose.yaml similarity index 83% rename from docker/blocks/mssql/docker-compose.yaml rename to devenv/docker/blocks/mssql/docker-compose.yaml index 538908fec72..05a93629e73 100644 --- a/docker/blocks/mssql/docker-compose.yaml +++ b/devenv/docker/blocks/mssql/docker-compose.yaml @@ -1,10 +1,10 @@ mssql: build: - context: blocks/mssql/build + context: docker/blocks/mssql/build environment: ACCEPT_EULA: Y MSSQL_SA_PASSWORD: Password! - MSSQL_PID: Express + MSSQL_PID: Developer MSSQL_DATABASE: grafana MSSQL_USER: grafana MSSQL_PASSWORD: Password! diff --git a/docker/blocks/mssql_tests/docker-compose.yaml b/devenv/docker/blocks/mssql_tests/docker-compose.yaml similarity index 85% rename from docker/blocks/mssql_tests/docker-compose.yaml rename to devenv/docker/blocks/mssql_tests/docker-compose.yaml index 5da6aad82af..eea4d1e3561 100644 --- a/docker/blocks/mssql_tests/docker-compose.yaml +++ b/devenv/docker/blocks/mssql_tests/docker-compose.yaml @@ -1,6 +1,6 @@ mssqltests: build: - context: blocks/mssql/build + context: docker/blocks/mssql/build environment: ACCEPT_EULA: Y MSSQL_SA_PASSWORD: Password! diff --git a/docker/blocks/mysql/config b/devenv/docker/blocks/mysql/config similarity index 100% rename from docker/blocks/mysql/config rename to devenv/docker/blocks/mysql/config diff --git a/docker/blocks/mysql/docker-compose.yaml b/devenv/docker/blocks/mysql/docker-compose.yaml similarity index 79% rename from docker/blocks/mysql/docker-compose.yaml rename to devenv/docker/blocks/mysql/docker-compose.yaml index f7881e66539..381b04a53c8 100644 --- a/docker/blocks/mysql/docker-compose.yaml +++ b/devenv/docker/blocks/mysql/docker-compose.yaml @@ -1,5 +1,5 @@ mysql: - image: mysql:latest + image: mysql:5.6 environment: MYSQL_ROOT_PASSWORD: rootpass MYSQL_DATABASE: grafana @@ -7,9 +7,6 @@ MYSQL_PASSWORD: password ports: - "3306:3306" - volumes: - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --innodb_monitor_enable=all] fake-mysql-data: diff --git a/docker/blocks/mysql_opendata/Dockerfile b/devenv/docker/blocks/mysql_opendata/Dockerfile similarity index 100% rename from docker/blocks/mysql_opendata/Dockerfile rename to devenv/docker/blocks/mysql_opendata/Dockerfile diff --git a/docker/blocks/mysql_opendata/docker-compose.yaml b/devenv/docker/blocks/mysql_opendata/docker-compose.yaml similarity index 82% rename from docker/blocks/mysql_opendata/docker-compose.yaml rename to devenv/docker/blocks/mysql_opendata/docker-compose.yaml index 594eeed284a..4d478ee0860 100644 --- a/docker/blocks/mysql_opendata/docker-compose.yaml +++ b/devenv/docker/blocks/mysql_opendata/docker-compose.yaml @@ -1,5 +1,5 @@ mysql_opendata: - build: blocks/mysql_opendata + build: docker/blocks/mysql_opendata environment: MYSQL_ROOT_PASSWORD: rootpass MYSQL_DATABASE: testdata diff --git a/docker/blocks/mysql_opendata/import_csv.sql b/devenv/docker/blocks/mysql_opendata/import_csv.sql similarity index 100% rename from docker/blocks/mysql_opendata/import_csv.sql rename to devenv/docker/blocks/mysql_opendata/import_csv.sql diff --git a/devenv/docker/blocks/mysql_tests/Dockerfile b/devenv/docker/blocks/mysql_tests/Dockerfile new file mode 100644 index 00000000000..89e16bc2ed6 --- /dev/null +++ b/devenv/docker/blocks/mysql_tests/Dockerfile @@ -0,0 +1,3 @@ +FROM mysql:5.6 +ADD setup.sql /docker-entrypoint-initdb.d +CMD ["mysqld"] diff --git a/docker/blocks/mysql_tests/docker-compose.yaml b/devenv/docker/blocks/mysql_tests/docker-compose.yaml similarity index 65% rename from docker/blocks/mysql_tests/docker-compose.yaml rename to devenv/docker/blocks/mysql_tests/docker-compose.yaml index 3c59b66b5ac..a7509d47880 100644 --- a/docker/blocks/mysql_tests/docker-compose.yaml +++ b/devenv/docker/blocks/mysql_tests/docker-compose.yaml @@ -1,5 +1,6 @@ mysqltests: - image: mysql:latest + build: + context: docker/blocks/mysql_tests environment: MYSQL_ROOT_PASSWORD: rootpass MYSQL_DATABASE: grafana_tests @@ -7,7 +8,4 @@ MYSQL_PASSWORD: password ports: - "3306:3306" - volumes: - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro tmpfs: /var/lib/mysql:rw diff --git a/devenv/docker/blocks/mysql_tests/setup.sql b/devenv/docker/blocks/mysql_tests/setup.sql new file mode 100644 index 00000000000..be917a1c542 --- /dev/null +++ b/devenv/docker/blocks/mysql_tests/setup.sql @@ -0,0 +1,2 @@ +CREATE DATABASE grafana_ds_tests; +GRANT ALL PRIVILEGES ON grafana_ds_tests.* TO 'grafana'; diff --git a/devenv/docker/blocks/nginx_proxy/Dockerfile b/devenv/docker/blocks/nginx_proxy/Dockerfile new file mode 100644 index 00000000000..04de507499d --- /dev/null +++ b/devenv/docker/blocks/nginx_proxy/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx:alpine + +COPY nginx.conf /etc/nginx/nginx.conf +COPY htpasswd /etc/nginx/htpasswd diff --git a/docker/blocks/nginx_proxy/docker-compose.yaml b/devenv/docker/blocks/nginx_proxy/docker-compose.yaml similarity index 72% rename from docker/blocks/nginx_proxy/docker-compose.yaml rename to devenv/docker/blocks/nginx_proxy/docker-compose.yaml index 7c3447ade5c..aefd7226f36 100644 --- a/docker/blocks/nginx_proxy/docker-compose.yaml +++ b/devenv/docker/blocks/nginx_proxy/docker-compose.yaml @@ -2,8 +2,8 @@ # http://localhost:3000 (Grafana running locally) # # Please note that you'll need to change the root_url in the Grafana configuration: -# root_url = %(protocol)s://%(domain)s:/grafana/ +# root_url = %(protocol)s://%(domain)s:10080/grafana/ nginxproxy: - build: blocks/nginx_proxy + build: docker/blocks/nginx_proxy network_mode: host diff --git a/devenv/docker/blocks/nginx_proxy/htpasswd b/devenv/docker/blocks/nginx_proxy/htpasswd new file mode 100755 index 00000000000..e2c5eeeff7b --- /dev/null +++ b/devenv/docker/blocks/nginx_proxy/htpasswd @@ -0,0 +1,3 @@ +user1:$apr1$1odeeQb.$kwV8D/VAAGUDU7pnHuKoV0 +user2:$apr1$A2kf25r.$6S0kp3C7vIuixS5CL0XA9. +admin:$apr1$IWn4DoRR$E2ol7fS/dkI18eU4bXnBO1 diff --git a/devenv/docker/blocks/nginx_proxy/nginx.conf b/devenv/docker/blocks/nginx_proxy/nginx.conf new file mode 100644 index 00000000000..860d3d0b89f --- /dev/null +++ b/devenv/docker/blocks/nginx_proxy/nginx.conf @@ -0,0 +1,38 @@ +events { worker_connections 1024; } + +http { + sendfile on; + + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + + server { + listen 10080; + + location /grafana/ { + ################################################################ + # Enable these settings to test with basic auth and an auth proxy header + # the htpasswd file contains an admin user with password admin and + # user1: grafana and user2: grafana + ################################################################ + + # auth_basic "Restricted Content"; + # auth_basic_user_file /etc/nginx/htpasswd; + + ################################################################ + # To use the auth proxy header, set the following in custom.ini: + # [auth.proxy] + # enabled = true + # header_name = X-WEBAUTH-USER + # header_property = username + ################################################################ + + # proxy_set_header X-WEBAUTH-USER $remote_user; + + proxy_pass http://localhost:3000/; + } + } +} diff --git a/docker/blocks/openldap/Dockerfile b/devenv/docker/blocks/openldap/Dockerfile similarity index 72% rename from docker/blocks/openldap/Dockerfile rename to devenv/docker/blocks/openldap/Dockerfile index d073e274356..76172e133a4 100644 --- a/docker/blocks/openldap/Dockerfile +++ b/devenv/docker/blocks/openldap/Dockerfile @@ -1,3 +1,5 @@ +# Fork of https://github.com/dinkel/docker-openldap + FROM debian:jessie LABEL maintainer="Christian Luginbühl " @@ -6,7 +8,8 @@ ENV OPENLDAP_VERSION 2.4.40 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - slapd=${OPENLDAP_VERSION}* && \ + slapd=${OPENLDAP_VERSION}* \ + ldap-utils && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -17,8 +20,10 @@ EXPOSE 389 VOLUME ["/etc/ldap", "/var/lib/ldap"] COPY modules/ /etc/ldap.dist/modules +COPY prepopulate/ /etc/ldap.dist/prepopulate COPY entrypoint.sh /entrypoint.sh +COPY prepopulate.sh /prepopulate.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/blocks/openldap/docker-compose.yaml b/devenv/docker/blocks/openldap/docker-compose.yaml similarity index 82% rename from docker/blocks/openldap/docker-compose.yaml rename to devenv/docker/blocks/openldap/docker-compose.yaml index be06524a57d..d11858ccfb9 100644 --- a/docker/blocks/openldap/docker-compose.yaml +++ b/devenv/docker/blocks/openldap/docker-compose.yaml @@ -1,5 +1,5 @@ openldap: - build: blocks/openldap + build: docker/blocks/openldap environment: SLAPD_PASSWORD: grafana SLAPD_DOMAIN: grafana.org diff --git a/docker/blocks/openldap/entrypoint.sh b/devenv/docker/blocks/openldap/entrypoint.sh similarity index 84% rename from docker/blocks/openldap/entrypoint.sh rename to devenv/docker/blocks/openldap/entrypoint.sh index 39a8b892de8..d202ed14b31 100755 --- a/docker/blocks/openldap/entrypoint.sh +++ b/devenv/docker/blocks/openldap/entrypoint.sh @@ -65,7 +65,7 @@ EOF fi if [[ -n "$SLAPD_ADDITIONAL_SCHEMAS" ]]; then - IFS=","; declare -a schemas=($SLAPD_ADDITIONAL_SCHEMAS) + IFS=","; declare -a schemas=($SLAPD_ADDITIONAL_SCHEMAS); unset IFS for schema in "${schemas[@]}"; do slapadd -n0 -F /etc/ldap/slapd.d -l "/etc/ldap/schema/${schema}.ldif" >/dev/null 2>&1 @@ -73,14 +73,19 @@ EOF fi if [[ -n "$SLAPD_ADDITIONAL_MODULES" ]]; then - IFS=","; declare -a modules=($SLAPD_ADDITIONAL_MODULES) + IFS=","; declare -a modules=($SLAPD_ADDITIONAL_MODULES); unset IFS for module in "${modules[@]}"; do - slapadd -n0 -F /etc/ldap/slapd.d -l "/etc/ldap/modules/${module}.ldif" >/dev/null 2>&1 + echo "Adding module ${module}" + slapadd -n0 -F /etc/ldap/slapd.d -l "/etc/ldap/modules/${module}.ldif" >/dev/null 2>&1 done fi - chown -R openldap:openldap /etc/ldap/slapd.d/ + # This needs to run in background + # Will prepopulate entries after ldap daemon has started + ./prepopulate.sh & + + chown -R openldap:openldap /etc/ldap/slapd.d/ /var/lib/ldap/ /var/run/slapd/ else slapd_configs_in_env=`env | grep 'SLAPD_'` diff --git a/devenv/docker/blocks/openldap/ldap_dev.toml b/devenv/docker/blocks/openldap/ldap_dev.toml new file mode 100644 index 00000000000..8767ff3c64a --- /dev/null +++ b/devenv/docker/blocks/openldap/ldap_dev.toml @@ -0,0 +1,86 @@ +# To troubleshoot and get more log info enable ldap debug logging in grafana.ini +# [log] +# filters = ldap:debug + +[[servers]] +# Ldap server host (specify multiple hosts space separated) +host = "127.0.0.1" +# Default port is 389 or 636 if use_ssl = true +port = 389 +# Set to true if ldap server supports TLS +use_ssl = false +# Set to true if connect ldap server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS) +start_tls = false +# set to true if you want to skip ssl cert validation +ssl_skip_verify = false +# set to the path to your root CA certificate or leave unset to use system defaults +# root_ca_cert = "/path/to/certificate.crt" + +# Search user bind dn +bind_dn = "cn=admin,dc=grafana,dc=org" +# Search user bind password +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +bind_password = 'grafana' + +# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)" +search_filter = "(cn=%s)" + +# An array of base dns to search through +search_base_dns = ["dc=grafana,dc=org"] + +# In POSIX LDAP schemas, without memberOf attribute a secondary query must be made for groups. +# This is done by enabling group_search_filter below. You must also set member_of= "cn" +# in [servers.attributes] below. + +# Users with nested/recursive group membership and an LDAP server that supports LDAP_MATCHING_RULE_IN_CHAIN +# can set group_search_filter, group_search_filter_user_attribute, group_search_base_dns and member_of +# below in such a way that the user's recursive group membership is considered. +# +# Nested Groups + Active Directory (AD) Example: +# +# AD groups store the Distinguished Names (DNs) of members, so your filter must +# recursively search your groups for the authenticating user's DN. For example: +# +# group_search_filter = "(member:1.2.840.113556.1.4.1941:=%s)" +# group_search_filter_user_attribute = "distinguishedName" +# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] +# +# [servers.attributes] +# ... +# member_of = "distinguishedName" + +## Group search filter, to retrieve the groups of which the user is a member (only set if memberOf attribute is not available) +# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" +## Group search filter user attribute defines what user attribute gets substituted for %s in group_search_filter. +## Defaults to the value of username in [server.attributes] +## Valid options are any of your values in [servers.attributes] +## If you are using nested groups you probably want to set this and member_of in +## [servers.attributes] to "distinguishedName" +# group_search_filter_user_attribute = "distinguishedName" +## An array of the base DNs to search through for groups. Typically uses ou=groups +# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] + +# Specify names of the ldap attributes your ldap uses +[servers.attributes] +name = "givenName" +surname = "sn" +username = "cn" +member_of = "memberOf" +email = "email" + +# Map ldap groups to grafana org roles +[[servers.group_mappings]] +group_dn = "cn=admins,ou=groups,dc=grafana,dc=org" +org_role = "Admin" +grafana_admin = true +# The Grafana organization database id, optional, if left out the default org (id 1) will be used +# org_id = 1 + +[[servers.group_mappings]] +group_dn = "cn=editors,ou=groups,dc=grafana,dc=org" +org_role = "Editor" + +[[servers.group_mappings]] +# If you want to match all (or no ldap groups) then you can use wildcard +group_dn = "*" +org_role = "Viewer" diff --git a/docker/blocks/openldap/modules/memberof.ldif b/devenv/docker/blocks/openldap/modules/memberof.ldif similarity index 100% rename from docker/blocks/openldap/modules/memberof.ldif rename to devenv/docker/blocks/openldap/modules/memberof.ldif diff --git a/devenv/docker/blocks/openldap/notes.md b/devenv/docker/blocks/openldap/notes.md new file mode 100644 index 00000000000..65155423616 --- /dev/null +++ b/devenv/docker/blocks/openldap/notes.md @@ -0,0 +1,45 @@ +# Notes on OpenLdap Docker Block + +Any ldif files added to the prepopulate subdirectory will be automatically imported into the OpenLdap database. + +The ldif files add three users, `ldapviewer`, `ldapeditor` and `ldapadmin`. Two groups, `admins` and `users`, are added that correspond with the group mappings in the default conf/ldap.toml. `ldapadmin` is a member of `admins` and `ldapeditor` is a member of `users`. + +Note that users that are added here need to specify a `memberOf` attribute manually as well as the `member` attribute for the group. The `memberOf` module usually does this automatically (if you add a group in Apache Directory Studio for example) but this does not work in the entrypoint script as it uses the `slapadd` command to add entries before the server has started and before the `memberOf` module is loaded. + +After adding ldif files to `prepopulate`: + +1. Remove your current docker image: `docker rm docker_openldap_1` +2. Build: `docker-compose build` +3. `docker-compose up` + +## Enabling LDAP in Grafana + +Copy the ldap_dev.toml file in this folder into your `conf` folder (it is gitignored already). To enable it in the .ini file to get Grafana to use this block: + +```ini +[auth.ldap] +enabled = true +config_file = conf/ldap_dev.toml +; allow_sign_up = true +``` + +Test groups & users + +admins + ldap-admin + ldap-torkel + ldap-daniel +backend + ldap-carl + ldap-torkel + ldap-leo +frontend + ldap-torkel + ldap-tobias + ldap-daniel +editors + ldap-editors + + +no groups + ldap-viewer diff --git a/devenv/docker/blocks/openldap/prepopulate.sh b/devenv/docker/blocks/openldap/prepopulate.sh new file mode 100755 index 00000000000..aa11f8aba4f --- /dev/null +++ b/devenv/docker/blocks/openldap/prepopulate.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +echo "Pre-populating ldap entries, first waiting for ldap to start" + +sleep 3 + +adminUserDn="cn=admin,dc=grafana,dc=org" +adminPassword="grafana" + +for file in `ls /etc/ldap/prepopulate/*.ldif`; do + ldapadd -x -D $adminUserDn -w $adminPassword -f "$file" +done + + diff --git a/devenv/docker/blocks/openldap/prepopulate/1_units.ldif b/devenv/docker/blocks/openldap/prepopulate/1_units.ldif new file mode 100644 index 00000000000..22e06303688 --- /dev/null +++ b/devenv/docker/blocks/openldap/prepopulate/1_units.ldif @@ -0,0 +1,9 @@ +dn: ou=groups,dc=grafana,dc=org +ou: Groups +objectclass: top +objectclass: organizationalUnit + +dn: ou=users,dc=grafana,dc=org +ou: Users +objectclass: top +objectclass: organizationalUnit diff --git a/devenv/docker/blocks/openldap/prepopulate/2_users.ldif b/devenv/docker/blocks/openldap/prepopulate/2_users.ldif new file mode 100644 index 00000000000..52e74b1e4b1 --- /dev/null +++ b/devenv/docker/blocks/openldap/prepopulate/2_users.ldif @@ -0,0 +1,80 @@ +# ldap-admin +dn: cn=ldap-admin,ou=users,dc=grafana,dc=org +mail: ldap-admin@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-admin +cn: ldap-admin + +dn: cn=ldap-editor,ou=users,dc=grafana,dc=org +mail: ldap-editor@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-editor +cn: ldap-editor + +dn: cn=ldap-viewer,ou=users,dc=grafana,dc=org +mail: ldap-viewer@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-viewer +cn: ldap-viewer + +dn: cn=ldap-carl,ou=users,dc=grafana,dc=org +mail: ldap-carl@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-carl +cn: ldap-carl + +dn: cn=ldap-daniel,ou=users,dc=grafana,dc=org +mail: ldap-daniel@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-daniel +cn: ldap-daniel + +dn: cn=ldap-leo,ou=users,dc=grafana,dc=org +mail: ldap-leo@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-leo +cn: ldap-leo + +dn: cn=ldap-tobias,ou=users,dc=grafana,dc=org +mail: ldap-tobias@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-tobias +cn: ldap-tobias + +dn: cn=ldap-torkel,ou=users,dc=grafana,dc=org +mail: ldap-torkel@grafana.com +userPassword: grafana +objectClass: person +objectClass: top +objectClass: inetOrgPerson +objectClass: organizationalPerson +sn: ldap-torkel +cn: ldap-torkel diff --git a/devenv/docker/blocks/openldap/prepopulate/3_groups.ldif b/devenv/docker/blocks/openldap/prepopulate/3_groups.ldif new file mode 100644 index 00000000000..8638a089cc8 --- /dev/null +++ b/devenv/docker/blocks/openldap/prepopulate/3_groups.ldif @@ -0,0 +1,25 @@ +dn: cn=admins,ou=groups,dc=grafana,dc=org +cn: admins +objectClass: groupOfNames +objectClass: top +member: cn=ldap-admin,ou=users,dc=grafana,dc=org +member: cn=ldap-torkel,ou=users,dc=grafana,dc=org + +dn: cn=editors,ou=groups,dc=grafana,dc=org +cn: editors +objectClass: groupOfNames +member: cn=ldap-editor,ou=users,dc=grafana,dc=org + +dn: cn=backend,ou=groups,dc=grafana,dc=org +cn: backend +objectClass: groupOfNames +member: cn=ldap-carl,ou=users,dc=grafana,dc=org +member: cn=ldap-leo,ou=users,dc=grafana,dc=org +member: cn=ldap-torkel,ou=users,dc=grafana,dc=org + +dn: cn=frontend,ou=groups,dc=grafana,dc=org +cn: frontend +objectClass: groupOfNames +member: cn=ldap-torkel,ou=users,dc=grafana,dc=org +member: cn=ldap-daniel,ou=users,dc=grafana,dc=org +member: cn=ldap-leo,ou=users,dc=grafana,dc=org diff --git a/docker/blocks/opentsdb/docker-compose.yaml b/devenv/docker/blocks/opentsdb/docker-compose.yaml similarity index 100% rename from docker/blocks/opentsdb/docker-compose.yaml rename to devenv/docker/blocks/opentsdb/docker-compose.yaml diff --git a/docker/blocks/postgres/docker-compose.yaml b/devenv/docker/blocks/postgres/docker-compose.yaml similarity index 90% rename from docker/blocks/postgres/docker-compose.yaml rename to devenv/docker/blocks/postgres/docker-compose.yaml index 566df7b8877..27736042f7b 100644 --- a/docker/blocks/postgres/docker-compose.yaml +++ b/devenv/docker/blocks/postgres/docker-compose.yaml @@ -1,5 +1,5 @@ postgrestest: - image: postgres:latest + image: postgres:9.3 environment: POSTGRES_USER: grafana POSTGRES_PASSWORD: password @@ -13,4 +13,4 @@ network_mode: bridge environment: FD_DATASOURCE: postgres - FD_PORT: 5432 \ No newline at end of file + FD_PORT: 5432 diff --git a/devenv/docker/blocks/postgres_tests/Dockerfile b/devenv/docker/blocks/postgres_tests/Dockerfile new file mode 100644 index 00000000000..df188e1094d --- /dev/null +++ b/devenv/docker/blocks/postgres_tests/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:9.3 +ADD setup.sql /docker-entrypoint-initdb.d +CMD ["postgres"] diff --git a/docker/blocks/postgres_tests/docker-compose.yaml b/devenv/docker/blocks/postgres_tests/docker-compose.yaml similarity index 75% rename from docker/blocks/postgres_tests/docker-compose.yaml rename to devenv/docker/blocks/postgres_tests/docker-compose.yaml index 44b66e8e558..7e6da7d8517 100644 --- a/docker/blocks/postgres_tests/docker-compose.yaml +++ b/devenv/docker/blocks/postgres_tests/docker-compose.yaml @@ -1,5 +1,6 @@ postgrestest: - image: postgres:latest + build: + context: docker/blocks/postgres_tests environment: POSTGRES_USER: grafanatest POSTGRES_PASSWORD: grafanatest diff --git a/devenv/docker/blocks/postgres_tests/setup.sql b/devenv/docker/blocks/postgres_tests/setup.sql new file mode 100644 index 00000000000..3b8a48f938d --- /dev/null +++ b/devenv/docker/blocks/postgres_tests/setup.sql @@ -0,0 +1,3 @@ +CREATE DATABASE grafanadstest; +REVOKE CONNECT ON DATABASE grafanadstest FROM PUBLIC; +GRANT CONNECT ON DATABASE grafanadstest TO grafanatest; diff --git a/docker/blocks/prometheus/Dockerfile b/devenv/docker/blocks/prometheus/Dockerfile similarity index 100% rename from docker/blocks/prometheus/Dockerfile rename to devenv/docker/blocks/prometheus/Dockerfile diff --git a/docker/blocks/prometheus/alert.rules b/devenv/docker/blocks/prometheus/alert.rules similarity index 100% rename from docker/blocks/prometheus/alert.rules rename to devenv/docker/blocks/prometheus/alert.rules diff --git a/docker/blocks/prometheus/docker-compose.yaml b/devenv/docker/blocks/prometheus/docker-compose.yaml similarity index 86% rename from docker/blocks/prometheus/docker-compose.yaml rename to devenv/docker/blocks/prometheus/docker-compose.yaml index 3c304cc74ad..db778060dde 100644 --- a/docker/blocks/prometheus/docker-compose.yaml +++ b/devenv/docker/blocks/prometheus/docker-compose.yaml @@ -1,5 +1,5 @@ prometheus: - build: blocks/prometheus + build: docker/blocks/prometheus network_mode: host ports: - "9090:9090" @@ -25,7 +25,7 @@ - "9093:9093" prometheus-random-data: - build: blocks/prometheus_random_data + build: docker/blocks/prometheus_random_data network_mode: host ports: - "8081:8080" diff --git a/docker/blocks/prometheus/prometheus.yml b/devenv/docker/blocks/prometheus/prometheus.yml similarity index 100% rename from docker/blocks/prometheus/prometheus.yml rename to devenv/docker/blocks/prometheus/prometheus.yml diff --git a/docker/blocks/prometheus2/Dockerfile b/devenv/docker/blocks/prometheus2/Dockerfile similarity index 100% rename from docker/blocks/prometheus2/Dockerfile rename to devenv/docker/blocks/prometheus2/Dockerfile diff --git a/docker/blocks/prometheus2/alert.rules b/devenv/docker/blocks/prometheus2/alert.rules similarity index 100% rename from docker/blocks/prometheus2/alert.rules rename to devenv/docker/blocks/prometheus2/alert.rules diff --git a/docker/blocks/prometheus2/docker-compose.yaml b/devenv/docker/blocks/prometheus2/docker-compose.yaml similarity index 85% rename from docker/blocks/prometheus2/docker-compose.yaml rename to devenv/docker/blocks/prometheus2/docker-compose.yaml index 589df868084..d586b4b5742 100644 --- a/docker/blocks/prometheus2/docker-compose.yaml +++ b/devenv/docker/blocks/prometheus2/docker-compose.yaml @@ -1,5 +1,5 @@ prometheus: - build: blocks/prometheus2 + build: docker/blocks/prometheus2 network_mode: host ports: - "9090:9090" @@ -25,7 +25,7 @@ - "9093:9093" prometheus-random-data: - build: blocks/prometheus_random_data + build: docker/blocks/prometheus_random_data network_mode: host ports: - "8081:8080" diff --git a/docker/blocks/prometheus2/prometheus.yml b/devenv/docker/blocks/prometheus2/prometheus.yml similarity index 100% rename from docker/blocks/prometheus2/prometheus.yml rename to devenv/docker/blocks/prometheus2/prometheus.yml diff --git a/devenv/docker/blocks/prometheus_mac/Dockerfile b/devenv/docker/blocks/prometheus_mac/Dockerfile new file mode 100644 index 00000000000..2098e6527d3 --- /dev/null +++ b/devenv/docker/blocks/prometheus_mac/Dockerfile @@ -0,0 +1,3 @@ +FROM prom/prometheus:v1.8.2 +ADD prometheus.yml /etc/prometheus/ +ADD alert.rules /etc/prometheus/ diff --git a/devenv/docker/blocks/prometheus_mac/alert.rules b/devenv/docker/blocks/prometheus_mac/alert.rules new file mode 100644 index 00000000000..563d1e89994 --- /dev/null +++ b/devenv/docker/blocks/prometheus_mac/alert.rules @@ -0,0 +1,10 @@ +# Alert Rules + +ALERT AppCrash + IF process_open_fds > 0 + FOR 15s + LABELS { severity="critical" } + ANNOTATIONS { + summary = "Number of open fds > 0", + description = "Just testing" + } diff --git a/devenv/docker/blocks/prometheus_mac/docker-compose.yaml b/devenv/docker/blocks/prometheus_mac/docker-compose.yaml new file mode 100644 index 00000000000..b73d278fae2 --- /dev/null +++ b/devenv/docker/blocks/prometheus_mac/docker-compose.yaml @@ -0,0 +1,26 @@ + prometheus: + build: docker/blocks/prometheus_mac + ports: + - "9090:9090" + + node_exporter: + image: prom/node-exporter + ports: + - "9100:9100" + + fake-prometheus-data: + image: grafana/fake-data-gen + ports: + - "9091:9091" + environment: + FD_DATASOURCE: prom + + alertmanager: + image: quay.io/prometheus/alertmanager + ports: + - "9093:9093" + + prometheus-random-data: + build: docker/blocks/prometheus_random_data + ports: + - "8081:8080" diff --git a/devenv/docker/blocks/prometheus_mac/prometheus.yml b/devenv/docker/blocks/prometheus_mac/prometheus.yml new file mode 100644 index 00000000000..299447ffb25 --- /dev/null +++ b/devenv/docker/blocks/prometheus_mac/prometheus.yml @@ -0,0 +1,39 @@ +# my global config +global: + scrape_interval: 10s # By default, scrape targets every 15 seconds. + evaluation_interval: 10s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + +# Load and evaluate rules in this file every 'evaluation_interval' seconds. +rule_files: + - "alert.rules" + # - "first.rules" + # - "second.rules" + +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager:9093" + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'node_exporter' + static_configs: + - targets: ['node_exporter:9100'] + + - job_name: 'fake-data-gen' + static_configs: + - targets: ['fake-prometheus-data:9091'] + + - job_name: 'grafana' + static_configs: + - targets: ['host.docker.internal:3000'] + + - job_name: 'prometheus-random-data' + static_configs: + - targets: ['prometheus-random-data:8080'] diff --git a/docker/blocks/prometheus_random_data/Dockerfile b/devenv/docker/blocks/prometheus_random_data/Dockerfile similarity index 100% rename from docker/blocks/prometheus_random_data/Dockerfile rename to devenv/docker/blocks/prometheus_random_data/Dockerfile diff --git a/devenv/docker/blocks/redis/docker-compose.yaml b/devenv/docker/blocks/redis/docker-compose.yaml new file mode 100644 index 00000000000..65071d4966b --- /dev/null +++ b/devenv/docker/blocks/redis/docker-compose.yaml @@ -0,0 +1,5 @@ + memcached: + image: redis:latest + ports: + - "6379:6379" + diff --git a/docker/blocks/smtp/Dockerfile b/devenv/docker/blocks/smtp/Dockerfile similarity index 100% rename from docker/blocks/smtp/Dockerfile rename to devenv/docker/blocks/smtp/Dockerfile diff --git a/docker/blocks/smtp/bootstrap.sh b/devenv/docker/blocks/smtp/bootstrap.sh similarity index 86% rename from docker/blocks/smtp/bootstrap.sh rename to devenv/docker/blocks/smtp/bootstrap.sh index a78f9d6dc16..27f6a2c3ef8 100755 --- a/docker/blocks/smtp/bootstrap.sh +++ b/devenv/docker/blocks/smtp/bootstrap.sh @@ -22,6 +22,6 @@ log() { log $RUN_CMD $RUN_CMD -# Exit immidiately in case of any errors or when we have interactive terminal +# Exit immediately in case of any errors or when we have interactive terminal if [[ $? != 0 ]] || test -t 0; then exit $?; fi log diff --git a/docker/blocks/smtp/docker-compose.yaml b/devenv/docker/blocks/smtp/docker-compose.yaml similarity index 100% rename from docker/blocks/smtp/docker-compose.yaml rename to devenv/docker/blocks/smtp/docker-compose.yaml diff --git a/docker/buildcontainer/Dockerfile b/devenv/docker/buildcontainer/Dockerfile similarity index 100% rename from docker/buildcontainer/Dockerfile rename to devenv/docker/buildcontainer/Dockerfile diff --git a/docker/buildcontainer/build.sh b/devenv/docker/buildcontainer/build.sh similarity index 100% rename from docker/buildcontainer/build.sh rename to devenv/docker/buildcontainer/build.sh diff --git a/docker/buildcontainer/build_circle.sh b/devenv/docker/buildcontainer/build_circle.sh similarity index 100% rename from docker/buildcontainer/build_circle.sh rename to devenv/docker/buildcontainer/build_circle.sh diff --git a/docker/buildcontainer/run_circle.sh b/devenv/docker/buildcontainer/run_circle.sh similarity index 100% rename from docker/buildcontainer/run_circle.sh rename to devenv/docker/buildcontainer/run_circle.sh diff --git a/docker/compose_header.yml b/devenv/docker/compose_header.yml similarity index 100% rename from docker/compose_header.yml rename to devenv/docker/compose_header.yml diff --git a/docker/debtest/Dockerfile b/devenv/docker/debtest/Dockerfile similarity index 100% rename from docker/debtest/Dockerfile rename to devenv/docker/debtest/Dockerfile diff --git a/docker/debtest/build.sh b/devenv/docker/debtest/build.sh similarity index 100% rename from docker/debtest/build.sh rename to devenv/docker/debtest/build.sh diff --git a/devenv/docker/ha_test/.gitignore b/devenv/docker/ha_test/.gitignore new file mode 100644 index 00000000000..0f4e139e204 --- /dev/null +++ b/devenv/docker/ha_test/.gitignore @@ -0,0 +1 @@ +grafana/provisioning/dashboards/alerts/alert-* \ No newline at end of file diff --git a/devenv/docker/ha_test/README.md b/devenv/docker/ha_test/README.md new file mode 100644 index 00000000000..bc93727ceae --- /dev/null +++ b/devenv/docker/ha_test/README.md @@ -0,0 +1,137 @@ +# Grafana High Availability (HA) test setup + +A set of docker compose services which together creates a Grafana HA test setup with capability of easily +scaling up/down number of Grafana instances. + +Included services + +* Grafana +* Mysql - Grafana configuration database and session storage +* Prometheus - Monitoring of Grafana and used as datasource of provisioned alert rules +* Nginx - Reverse proxy for Grafana and Prometheus. Enables browsing Grafana/Prometheus UI using a hostname + +## Prerequisites + +### Build grafana docker container + +Build a Grafana docker container from current branch and commit and tag it as grafana/grafana:dev. + +```bash +$ cd +$ make build-docker-full +``` + +### Virtual host names + +#### Alternative 1 - Use dnsmasq + +```bash +$ sudo apt-get install dnsmasq +$ echo 'address=/loc/127.0.0.1' | sudo tee /etc/dnsmasq.d/dnsmasq-loc.conf > /dev/null +$ sudo /etc/init.d/dnsmasq restart +$ ping whatever.loc +PING whatever.loc (127.0.0.1) 56(84) bytes of data. +64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.076 ms +--- whatever.loc ping statistics --- +1 packet transmitted, 1 received, 0% packet loss, time 1998ms +``` + +#### Alternative 2 - Manually update /etc/hosts + +Update your `/etc/hosts` to be able to access Grafana and/or Prometheus UI using a hostname. + +```bash +$ cat /etc/hosts +127.0.0.1 grafana.loc +127.0.0.1 prometheus.loc +``` + +## Start services + +```bash +$ docker-compose up -d +``` + +Browse +* http://grafana.loc/ +* http://prometheus.loc/ + +Check for any errors + +```bash +$ docker-compose logs | grep error +``` + +### Scale Grafana instances up/down + +Scale number of Grafana instances to `` + +```bash +$ docker-compose up --scale grafana= -d +# for example 3 instances +$ docker-compose up --scale grafana=3 -d +``` + +## Test alerting + +### Create notification channels + +Creates default notification channels, if not already exists + +```bash +$ ./alerts.sh setup +``` + +### Slack notifications + +Disable + +```bash +$ ./alerts.sh slack -d +``` + +Enable and configure url + +```bash +$ ./alerts.sh slack -u https://hooks.slack.com/services/... +``` + +Enable, configure url and enable reminders + +```bash +$ ./alerts.sh slack -u https://hooks.slack.com/services/... -r -e 10m +``` + +### Provision alert dashboards with alert rules + +Provision 1 dashboard/alert rule (default) + +```bash +$ ./alerts.sh provision +``` + +Provision 10 dashboards/alert rules + +```bash +$ ./alerts.sh provision -a 10 +``` + +Provision 10 dashboards/alert rules and change condition to `gt > 100` + +```bash +$ ./alerts.sh provision -a 10 -c 100 +``` + +### Pause/unpause all alert rules + +Pause + +```bash +$ ./alerts.sh pause +``` + +Unpause + +```bash +$ ./alerts.sh unpause +``` diff --git a/devenv/docker/ha_test/alerts.sh b/devenv/docker/ha_test/alerts.sh new file mode 100755 index 00000000000..a05a4581739 --- /dev/null +++ b/devenv/docker/ha_test/alerts.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +requiresJsonnet() { + if ! type "jsonnet" > /dev/null; then + echo "you need you install jsonnet to run this script" + echo "follow the instructions on https://github.com/google/jsonnet" + exit 1 + fi +} + +setup() { + STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://admin:admin@grafana.loc/api/alert-notifications/1) + if [ $STATUS -eq 200 ]; then + echo "Email already exists, skipping..." + else + curl -H "Content-Type: application/json" \ + -d '{ + "name": "Email", + "type": "email", + "isDefault": false, + "sendReminder": false, + "uploadImage": true, + "settings": { + "addresses": "user@test.com" + } + }' \ + http://admin:admin@grafana.loc/api/alert-notifications + fi + + STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://admin:admin@grafana.loc/api/alert-notifications/2) + if [ $STATUS -eq 200 ]; then + echo "Slack already exists, skipping..." + else + curl -H "Content-Type: application/json" \ + -d '{ + "name": "Slack", + "type": "slack", + "isDefault": false, + "sendReminder": false, + "uploadImage": true + }' \ + http://admin:admin@grafana.loc/api/alert-notifications + fi +} + +slack() { + enabled=true + url='' + remind=false + remindEvery='10m' + + while getopts ":e:u:dr" o; do + case "${o}" in + e) + remindEvery=${OPTARG} + ;; + u) + url=${OPTARG} + ;; + d) + enabled=false + ;; + r) + remind=true + ;; + esac + done + shift $((OPTIND-1)) + + curl -X PUT \ + -H "Content-Type: application/json" \ + -d '{ + "id": 2, + "name": "Slack", + "type": "slack", + "isDefault": '$enabled', + "sendReminder": '$remind', + "frequency": "'$remindEvery'", + "uploadImage": true, + "settings": { + "url": "'$url'" + } + }' \ + http://admin:admin@grafana.loc/api/alert-notifications/2 +} + +provision() { + alerts=1 + condition=65 + while getopts ":a:c:" o; do + case "${o}" in + a) + alerts=${OPTARG} + ;; + c) + condition=${OPTARG} + ;; + esac + done + shift $((OPTIND-1)) + + requiresJsonnet + + rm -rf grafana/provisioning/dashboards/alerts/alert-*.json + jsonnet -m grafana/provisioning/dashboards/alerts grafana/provisioning/alerts.jsonnet --ext-code alerts=$alerts --ext-code condition=$condition +} + +pause() { + curl -H "Content-Type: application/json" \ + -d '{"paused":true}' \ + http://admin:admin@grafana.loc/api/admin/pause-all-alerts +} + +unpause() { + curl -H "Content-Type: application/json" \ + -d '{"paused":false}' \ + http://admin:admin@grafana.loc/api/admin/pause-all-alerts +} + +usage() { + echo -e "Usage: ./alerts.sh COMMAND [OPTIONS]\n" + echo -e "Commands" + echo -e " setup\t\t creates default alert notification channels" + echo -e " slack\t\t configure slack notification channel" + echo -e " [-d]\t\t\t disable notifier, default enabled" + echo -e " [-u]\t\t\t url" + echo -e " [-r]\t\t\t send reminders" + echo -e " [-e ]\t\t default 10m\n" + echo -e " provision\t provision alerts" + echo -e " [-a ]\t default 1" + echo -e " [-c ]\t default 65\n" + echo -e " pause\t\t pause all alerts" + echo -e " unpause\t unpause all alerts" +} + +main() { + local cmd=$1 + + if [[ $cmd == "setup" ]]; then + setup + elif [[ $cmd == "slack" ]]; then + slack "${@:2}" + elif [[ $cmd == "provision" ]]; then + provision "${@:2}" + elif [[ $cmd == "pause" ]]; then + pause + elif [[ $cmd == "unpause" ]]; then + unpause + fi + + if [[ -z "$cmd" ]]; then + usage + fi +} + +main "$@" diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml new file mode 100644 index 00000000000..ce8630d88a4 --- /dev/null +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -0,0 +1,78 @@ +version: "2.1" + +services: + nginx-proxy: + image: jwilder/nginx-proxy + ports: + - "80:80" + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + + db: + image: mysql + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: grafana + MYSQL_USER: grafana + MYSQL_PASSWORD: password + ports: + - 3306 + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + timeout: 10s + retries: 10 + + # db: + # image: postgres:9.3 + # environment: + # POSTGRES_DATABASE: grafana + # POSTGRES_USER: grafana + # POSTGRES_PASSWORD: password + # ports: + # - 5432 + # healthcheck: + # test: ["CMD-SHELL", "pg_isready -d grafana -U grafana"] + # timeout: 10s + # retries: 10 + + grafana: + image: grafana/grafana:dev + volumes: + - ./grafana/provisioning/:/etc/grafana/provisioning/ + environment: + - VIRTUAL_HOST=grafana.loc + - GF_SERVER_ROOT_URL=http://grafana.loc + - GF_DATABASE_NAME=grafana + - GF_DATABASE_USER=grafana + - GF_DATABASE_PASSWORD=password + - GF_DATABASE_TYPE=mysql + - GF_DATABASE_HOST=db:3306 + - GF_SESSION_PROVIDER=mysql + - GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(db:3306)/grafana?allowNativePasswords=true + # - GF_DATABASE_TYPE=postgres + # - GF_DATABASE_HOST=db:5432 + # - GF_DATABASE_SSL_MODE=disable + # - GF_SESSION_PROVIDER=postgres + # - GF_SESSION_PROVIDER_CONFIG=user=grafana password=password host=db port=5432 dbname=grafana sslmode=disable + - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug + ports: + - 3000 + depends_on: + db: + condition: service_healthy + + prometheus: + image: prom/prometheus:v2.4.2 + volumes: + - ./prometheus/:/etc/prometheus/ + environment: + - VIRTUAL_HOST=prometheus.loc + ports: + - 9090 + + # mysqld-exporter: + # image: prom/mysqld-exporter + # environment: + # - DATA_SOURCE_NAME=grafana:password@(mysql:3306)/ + # ports: + # - 9104 diff --git a/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet b/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet new file mode 100644 index 00000000000..86ded7e79d6 --- /dev/null +++ b/devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet @@ -0,0 +1,202 @@ +local numAlerts = std.extVar('alerts'); +local condition = std.extVar('condition'); +local arr = std.range(1, numAlerts); + +local alertDashboardTemplate = { + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 65 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "10s", + "handler": 1, + "name": "bulk alerting", + "noDataState": "no_data", + "notifications": [ + { + "id": 2 + } + ] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "$$hashKey": "object:117", + "expr": "go_goroutines", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 50 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "New dashboard", + "uid": null, + "version": 0 +}; + + +{ + ['alert-' + std.toString(x) + '.json']: + alertDashboardTemplate + { + panels: [ + alertDashboardTemplate.panels[0] + + { + alert+: { + name: 'Alert rule ' + x, + conditions: [ + alertDashboardTemplate.panels[0].alert.conditions[0] + + { + evaluator+: { + params: [condition] + } + }, + ], + }, + }, + ], + uid: 'alert-' + x, + title: 'Alert ' + x + }, + for x in arr +} \ No newline at end of file diff --git a/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts.yaml b/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts.yaml new file mode 100644 index 00000000000..60b6cd4bb04 --- /dev/null +++ b/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 + +providers: + - name: 'Alerts' + folder: 'Alerts' + type: file + options: + path: /etc/grafana/provisioning/dashboards/alerts diff --git a/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts/overview.json b/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts/overview.json new file mode 100644 index 00000000000..53e33c37b1f --- /dev/null +++ b/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts/overview.json @@ -0,0 +1,172 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": { + "Active alerts": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Active grafana instances", + "dashes": true, + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(grafana_alerting_notification_sent_total[1m])) by(job)", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "Notifications sent", + "refId": "A" + }, + { + "expr": "min(grafana_alerting_active_alerts) without(instance)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "Active alerts", + "refId": "B" + }, + { + "expr": "count(up{job=\"grafana\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active grafana instances", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Notifications sent vs active alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 3 + } + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Overview", + "uid": "xHy7-hAik", + "version": 6 +} \ No newline at end of file diff --git a/devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml b/devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 00000000000..8d59793be16 --- /dev/null +++ b/devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + jsonData: + timeInterval: 10s + queryTimeout: 30s + httpMethod: POST \ No newline at end of file diff --git a/devenv/docker/ha_test/prometheus/prometheus.yml b/devenv/docker/ha_test/prometheus/prometheus.yml new file mode 100644 index 00000000000..ea97ba8ba05 --- /dev/null +++ b/devenv/docker/ha_test/prometheus/prometheus.yml @@ -0,0 +1,39 @@ +# my global config +global: + scrape_interval: 10s # By default, scrape targets every 15 seconds. + evaluation_interval: 10s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + +# Load and evaluate rules in this file every 'evaluation_interval' seconds. +#rule_files: +# - "alert.rules" +# - "first.rules" +# - "second.rules" + +# alerting: +# alertmanagers: +# - scheme: http +# static_configs: +# - targets: +# - "127.0.0.1:9093" + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'grafana' + dns_sd_configs: + - names: + - 'grafana' + type: 'A' + port: 3000 + refresh_interval: 10s + + # - job_name: 'mysql' + # dns_sd_configs: + # - names: + # - 'mysqld-exporter' + # type: 'A' + # port: 9104 + # refresh_interval: 10s \ No newline at end of file diff --git a/docker/rpmtest/build.sh b/devenv/docker/rpmtest/build.sh similarity index 100% rename from docker/rpmtest/build.sh rename to devenv/docker/rpmtest/build.sh diff --git a/tests/api/clearState.test.ts b/devenv/e2e-api-tests/clearState.test.ts similarity index 100% rename from tests/api/clearState.test.ts rename to devenv/e2e-api-tests/clearState.test.ts diff --git a/tests/api/client.ts b/devenv/e2e-api-tests/client.ts similarity index 100% rename from tests/api/client.ts rename to devenv/e2e-api-tests/client.ts diff --git a/tests/api/dashboard.test.ts b/devenv/e2e-api-tests/dashboard.test.ts similarity index 100% rename from tests/api/dashboard.test.ts rename to devenv/e2e-api-tests/dashboard.test.ts diff --git a/tests/api/folder.test.ts b/devenv/e2e-api-tests/folder.test.ts similarity index 100% rename from tests/api/folder.test.ts rename to devenv/e2e-api-tests/folder.test.ts diff --git a/tests/api/jest.js b/devenv/e2e-api-tests/jest.js similarity index 100% rename from tests/api/jest.js rename to devenv/e2e-api-tests/jest.js diff --git a/tests/api/search.test.ts b/devenv/e2e-api-tests/search.test.ts similarity index 100% rename from tests/api/search.test.ts rename to devenv/e2e-api-tests/search.test.ts diff --git a/tests/api/setup.ts b/devenv/e2e-api-tests/setup.ts similarity index 100% rename from tests/api/setup.ts rename to devenv/e2e-api-tests/setup.ts diff --git a/tests/api/tsconfig.json b/devenv/e2e-api-tests/tsconfig.json similarity index 100% rename from tests/api/tsconfig.json rename to devenv/e2e-api-tests/tsconfig.json diff --git a/tests/api/user.test.ts b/devenv/e2e-api-tests/user.test.ts similarity index 100% rename from tests/api/user.test.ts rename to devenv/e2e-api-tests/user.test.ts diff --git a/devenv/setup.sh b/devenv/setup.sh new file mode 100755 index 00000000000..c9cc0d47a6f --- /dev/null +++ b/devenv/setup.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +bulkDashboard() { + + requiresJsonnet + + COUNTER=0 + MAX=400 + while [ $COUNTER -lt $MAX ]; do + jsonnet -o "bulk-dashboards/dashboard${COUNTER}.json" -e "local bulkDash = import 'bulk-dashboards/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" + let COUNTER=COUNTER+1 + done + + ln -s -f ../../../devenv/bulk-dashboards/bulk-dashboards.yaml ../conf/provisioning/dashboards/custom.yaml +} + +bulkAlertingDashboard() { + + requiresJsonnet + + COUNTER=0 + MAX=100 + while [ $COUNTER -lt $MAX ]; do + jsonnet -o "bulk_alerting_dashboards/alerting_dashboard${COUNTER}.json" -e "local bulkDash = import 'bulk_alerting_dashboards/bulkdash_alerting.jsonnet'; bulkDash + { uid: 'bd-${COUNTER}', title: 'alerting-title-${COUNTER}' }" + let COUNTER=COUNTER+1 + done + + ln -s -f ../../../devenv/bulk_alerting_dashboards/bulk_alerting_dashboards.yaml ../conf/provisioning/dashboards/custom.yaml +} + +requiresJsonnet() { + if ! type "jsonnet" > /dev/null; then + echo "you need you install jsonnet to run this script" + echo "follow the instructions on https://github.com/google/jsonnet" + exit 1 + fi +} + +devDashboards() { + echo -e "\xE2\x9C\x94 Setting up all dev dashboards using provisioning" + ln -s -f ../../../devenv/dashboards.yaml ../conf/provisioning/dashboards/dev.yaml +} + +devDatasources() { + echo -e "\xE2\x9C\x94 Setting up all dev datasources using provisioning" + + ln -s -f ../../../devenv/datasources.yaml ../conf/provisioning/datasources/dev.yaml +} + +usage() { + echo -e "\n" + echo "Usage:" + echo " bulk-dashboards - create and provisioning 400 dashboards" + echo " bulk-alerting-dashboards - create and provisioning 400 dashboards with alerts" + echo " no args - provisiong core datasources and dev dashboards" +} + +main() { + echo -e "------------------------------------------------------------------" + echo -e "This script setups provisioning for dev datasources and dashboards" + echo -e "------------------------------------------------------------------" + echo -e "\n" + + local cmd=$1 + + if [[ $cmd == "bulk-alerting-dashboards" ]]; then + bulkAlertingDashboard + elif [[ $cmd == "bulk-dashboards" ]]; then + bulkDashboard + else + devDashboards + devDatasources + fi + + if [[ -z "$cmd" ]]; then + usage + fi + +} + +main "$@" diff --git a/docker/blocks/mssql/build/setup.sql.template b/docker/blocks/mssql/build/setup.sql.template deleted file mode 100644 index 1746a18d241..00000000000 --- a/docker/blocks/mssql/build/setup.sql.template +++ /dev/null @@ -1,14 +0,0 @@ -CREATE LOGIN %%USER%% WITH PASSWORD = '%%PWD%%' -GO - -CREATE DATABASE %%DB%%; -GO - -USE %%DB%%; -GO - -CREATE USER %%USER%% FOR LOGIN %%USER%%; -GO - -EXEC sp_addrolemember 'db_owner', '%%USER%%'; -GO diff --git a/docker/blocks/nginx_proxy/Dockerfile b/docker/blocks/nginx_proxy/Dockerfile deleted file mode 100644 index 9ded20dfdda..00000000000 --- a/docker/blocks/nginx_proxy/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM nginx:alpine - -COPY nginx.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/docker/blocks/nginx_proxy/nginx.conf b/docker/blocks/nginx_proxy/nginx.conf deleted file mode 100644 index 18e27b3fb01..00000000000 --- a/docker/blocks/nginx_proxy/nginx.conf +++ /dev/null @@ -1,19 +0,0 @@ -events { worker_connections 1024; } - -http { - sendfile on; - - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; - - server { - listen 10080; - - location /grafana/ { - proxy_pass http://localhost:3000/; - } - } -} \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index ff5ef6a4131..7310f184a60 100644 --- a/docs/README.md +++ b/docs/README.md @@ -65,7 +65,7 @@ make docs-build This will rebuild the docs docker container. -To be able to use the image your have to quit (CTRL-C) the `make watch` command (that you run in the same directory as this README). Then simply rerun `make watch`, it will restart the docs server but now with access to your image. +To be able to use the image you have to quit (CTRL-C) the `make watch` command (that you run in the same directory as this README). Then simply rerun `make watch`, it will restart the docs server but now with access to your image. ### Editing content diff --git a/docs/sources/administration/permissions.md b/docs/sources/administration/permissions.md deleted file mode 100644 index e7b84a417c0..00000000000 --- a/docs/sources/administration/permissions.md +++ /dev/null @@ -1,118 +0,0 @@ -+++ -title = "Permissions" -description = "Grafana user permissions" -keywords = ["grafana", "configuration", "documentation", "admin", "users", "permissions"] -type = "docs" -aliases = ["/reference/admin"] -[menu.docs] -name = "Permissions" -parent = "admin" -weight = 3 -+++ - -# Permissions - -Grafana users have permissions that are determined by their: - -- **Organization Role** (Admin, Editor, Viewer) -- Via **Team** memberships where the **Team** has been assigned specific permissions. -- Via permissions assigned directly to user (on folders or dashboards) -- The Grafana Admin (i.e. Super Admin) user flag. - -## Organization Roles - -Users can be belong to one or more organizations. A user's organization membership is tied to a role that defines what the user is allowed to do -in that organization. - -### Admin Role - -Can do everything scoped to the organization. For example: - -- Add & Edit data sources. -- Add & Edit organization users & teams. -- Configure App plugins & set org settings. - -### Editor Role - -- Can create and modify dashboards & alert rules. This can be disabled on specific folders and dashboards. -- **Cannot** create or edit data sources nor invite new users. - -### Viewer Role - -- View any dashboard. This can be disabled on specific folders and dashboards. -- **Cannot** create or edit dashboards nor data sources. - -This role can be tweaked via Grafana server setting [viewers_can_edit]({{< relref "installation/configuration.md#viewers-can-edit" >}}). If you set this to true users -with **Viewer** can also make transient dashboard edits, meaning they can modify panels & queries but not save the changes (nor create new dashboards). -Useful for public Grafana installations where you want anonymous users to be able to edit panels & queries but not save or create new dashboards. - -## Grafana Admin - -This admin flag makes a user a `Super Admin`. This means they can access the `Server Admin` views where all users and organizations can be administrated. - -### Dashboard & Folder Permissions - -> Introduced in Grafana v5.0 - -{{< docs-imagebox img="/img/docs/v50/folder_permissions.png" max-width="500px" class="docs-image--right" >}} - -For dashboards and dashboard folders there is a **Permissions** page that make it possible to -remove the default role based permssions for Editors and Viewers. It's here you can add and assign permissions to specific **Users** and **Teams**. - -You can assign & remove permissions for **Organization Roles**, **Users** and **Teams**. - -Permission levels: - -- **Admin**: Can edit & create dashboards and edit permissions. -- **Edit**: Can edit & create dashboards. **Cannot** edit folder/dashboard permissions. -- **View**: Can only view existing dashboards/folders. - -#### Restricting Access - -The highest permission always wins so if you for example want to hide a folder or dashboard from others you need to remove the **Organization Role** based permission from the Access Control List (ACL). - -- You cannot override permissions for users with the **Org Admin Role**. Admins always have access to everything. -- A more specific permission with a lower permission level will not have any effect if a more general rule exists with higher permission level. You need to remove or lower the permission level of the more general rule. - -#### How Grafana Resolves Multiple Permissions - Examples - -##### Example 1 (`user1` has the Editor Role) - -Permissions for a dashboard: - -- `Everyone with Editor Role Can Edit` -- `user1 Can View` - -Result: `user1` has Edit permission as the highest permission always wins. - -##### Example 2 (`user1` has the Viewer Role and is a member of `team1`) - -Permissions for a dashboard: - -- `Everyone with Viewer Role Can View` -- `user1 Can Edit` -- `team1 Can Admin` - -Result: `user1` has Admin permission as the highest permission always wins. - -##### Example 3 - -Permissions for a dashboard: - -- `user1 Can Admin (inherited from parent folder)` -- `user1 Can Edit` - -Result: You cannot override to a lower permission. `user1` has Admin permission as the highest permission always wins. - -- **View**: Can only view existing dashboars/folders. -- You cannot override permissions for users with **Org Admin Role** -- A more specific permission with lower permission level will not have any effect if a more general rule exists with higher permission level. For example if "Everyone with Editor Role Can Edit" exists in the ACL list then **John Doe** will still have Edit permission even after you have specifically added a permission for this user with the permission set to **View**. You need to remove or lower the permission level of the more general rule. - -### Data source permissions - -Permissions on dashboards and folders **do not** include permissions on data sources. A user with `Viewer` role -can still issue any possible query to a data source, not just those queries that exist on dashboards he/she has access to. -We hope to add permissions on data sources in a future release. Until then **do not** view dashboard permissions as a secure -way to restrict user data access. Dashboard permissions only limits what dashboards & folders a user can view & edit not which -data sources a user can access nor what queries a user can issue. - diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index 135973df52a..60e89b486a5 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -11,11 +11,13 @@ weight = 8 # Provisioning Grafana -## Config file +In previous versions of Grafana, you could only use the API for provisioning data sources and dashboards. But that required the service to be running before you started creating dashboards and you also needed to set up credentials for the HTTP API. In v5.0 we decided to improve this experience by adding a new active provisioning system that uses config files. This will make GitOps more natural as data sources and dashboards can be defined via files that can be version controlled. We hope to extend this system to later add support for users, orgs and alerts as well. + +## Config File Checkout the [configuration](/installation/configuration) page for more information on what you can configure in `grafana.ini` -### Config file locations +### Config File Locations - Default configuration from `$WORKING_DIR/conf/defaults.ini` - Custom configuration from `$WORKING_DIR/conf/custom.ini` @@ -26,7 +28,7 @@ Checkout the [configuration](/installation/configuration) page for more informat > `/etc/grafana/grafana.ini`. This path is specified in the Grafana > init.d script using `--config` file parameter. -### Using environment variables +### Using Environment Variables All options in the configuration file (listed below) can be overridden using environment variables using the syntax: @@ -59,7 +61,7 @@ export GF_AUTH_GOOGLE_CLIENT_SECRET=newS3cretKey
-## Configuration management tools +## Configuration Management Tools Currently we do not provide any scripts/manifests for configuring Grafana. Rather than spending time learning and creating scripts/manifests for each tool, we think our time is better spent making Grafana easier to provision. Therefore, we heavily relay on the expertise of the community. @@ -69,17 +71,20 @@ Puppet | [https://forge.puppet.com/puppet/grafana](https://forge.puppet.com/pupp Ansible | [https://github.com/cloudalchemy/ansible-grafana](https://github.com/cloudalchemy/ansible-grafana) Chef | [https://github.com/JonathanTron/chef-grafana](https://github.com/JonathanTron/chef-grafana) Saltstack | [https://github.com/salt-formulas/salt-formula-grafana](https://github.com/salt-formulas/salt-formula-grafana) +Jsonnet | [https://github.com/grafana/grafonnet-lib/](https://github.com/grafana/grafonnet-lib/) ## Datasources > This feature is available from v5.0 -It's possible to manage datasources in Grafana by adding one or more yaml config files in the [`provisioning/datasources`](/installation/configuration/#provisioning) directory. Each config file can contain a list of `datasources` that will be added or updated during start up. If the datasource already exists, Grafana will update it to match the configuration file. The config file can also contain a list of datasources that should be deleted. That list is called `delete_datasources`. Grafana will delete datasources listed in `delete_datasources` before inserting/updating those in the `datasource` list. +It's possible to manage datasources in Grafana by adding one or more yaml config files in the [`provisioning/datasources`](/installation/configuration/#provisioning) directory. Each config file can contain a list of `datasources` that will be added or updated during start up. If the datasource already exists, Grafana will update it to match the configuration file. The config file can also contain a list of datasources that should be deleted. That list is called `deleteDatasources`. Grafana will delete datasources listed in `deleteDatasources` before inserting/updating those in the `datasource` list. + +### Running Multiple Grafana Instances -### Running multiple Grafana instances. If you are running multiple instances of Grafana you might run into problems if they have different versions of the `datasource.yaml` configuration file. The best way to solve this problem is to add a version number to each datasource in the configuration and increase it when you update the config. Grafana will only update datasources with the same or lower version number than specified in the config. That way, old configs cannot overwrite newer configs if they restart at the same time. -### Example datasource config file +### Example Datasource Config File + ```yaml # config file version apiVersion: 1 @@ -90,13 +95,13 @@ deleteDatasources: orgId: 1 # list of datasources to insert/update depending -# whats available in the database +# what's available in the database datasources: # name of the datasource. Required - name: Graphite # datasource type. Required type: graphite - # access mode. direct or proxy. Required + # access mode. proxy or direct (Server or Browser in the UI). Required access: proxy # org id. will default to orgId 1 if not specified orgId: 1 @@ -118,7 +123,7 @@ datasources: withCredentials: # mark as default datasource. Max one per org isDefault: - # fields that will be converted to json and stored in json_data + # fields that will be converted to json and stored in jsonData jsonData: graphiteVersion: "1.1" tlsAuth: true @@ -133,30 +138,42 @@ datasources: editable: false ``` -#### Json data +#### Custom Settings per Datasource +Please refer to each datasource documentation for specific provisioning examples. -Since not all datasources have the same configuration settings we only have the most common ones as fields. The rest should be stored as a json blob in the `json_data` field. Here are the most common settings that the core datasources use. +| Datasource | Misc | +| ---- | ---- | +| Elasticsearch | Elasticsearch uses the `database` property to configure the index for a datasource | -| Name | Type | Datasource |Description | -| ----| ---- | ---- | --- | +#### Json Data + +Since not all datasources have the same configuration settings we only have the most common ones as fields. The rest should be stored as a json blob in the `jsonData` field. Here are the most common settings that the core datasources use. + +| Name | Type | Datasource | Description | +| ---- | ---- | ---- | ---- | | tlsAuth | boolean | *All* | Enable TLS authentication using client cert configured in secure json data | -| tlsAuthWithCACert | boolean | *All* | Enable TLS authtication using CA cert | +| tlsAuthWithCACert | boolean | *All* | Enable TLS authentication using CA cert | | tlsSkipVerify | boolean | *All* | Controls whether a client verifies the server's certificate chain and host name. | | graphiteVersion | string | Graphite | Graphite version | -| timeInterval | string | Elastic, Influxdb & Prometheus | Lowest interval/step value that should be used for this data source | -| esVersion | string | Elastic | Elasticsearch version as an number (2/5/56) | -| timeField | string | Elastic | Which field that should be used as timestamp | -| interval | string | Elastic | Index date time format | +| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL & MSSQL | Lowest interval/step value that should be used for this data source | +| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60) | +| timeField | string | Elasticsearch | Which field that should be used as timestamp | +| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' | | authType | string | Cloudwatch | Auth provider. keys/credentials/arn | | assumeRoleArn | string | Cloudwatch | ARN of Assume Role | | defaultRegion | string | Cloudwatch | AWS region | | customMetricsNamespaces | string | Cloudwatch | Namespaces of Custom Metrics | -| tsdbVersion | string | OpenTsdb | Version | -| tsdbResolution | string | OpenTsdb | Resolution | -| sslmode | string | Postgre | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | +| tsdbVersion | string | OpenTSDB | Version | +| tsdbResolution | string | OpenTSDB | Resolution | +| sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | +| encrypt | string | MSSQL | Connection SSL encryption handling. 'disable', 'false' or 'true' | +| postgresVersion | number | PostgreSQL | Postgres version as a number (903/904/905/906/1000) meaning v9.3, v9.4, ..., v10 | +| timescaledb | boolean | PostgreSQL | Enable usage of TimescaleDB extension | +| maxOpenConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of open connections to the database (Grafana v5.4+) | +| maxIdleConns | number | MySQL, PostgreSQL & MSSQL | Maximum number of connections in the idle connection pool (Grafana v5.4+) | +| connMaxLifetime | number | MySQL, PostgreSQL & MSSQL | Maximum amount of time in seconds a connection may be reused (Grafana v5.4+) | - -#### Secure Json data +#### Secure Json Data `{"authType":"keys","defaultRegion":"us-west-2","timeField":"@timestamp"}` @@ -167,8 +184,8 @@ Secure json data is a map of settings that will be encrypted with [secret key](/ | tlsCACert | string | *All* |CA cert for out going requests | | tlsClientCert | string | *All* |TLS Client cert for outgoing requests | | tlsClientKey | string | *All* |TLS Client key for outgoing requests | -| password | string | Postgre | password | -| user | string | Postgre | user | +| password | string | PostgreSQL | password | +| user | string | PostgreSQL | user | | accessKey | string | Cloudwatch | Access key for connecting to Cloudwatch | | secretKey | string | Cloudwatch | Secret key for connecting to Cloudwatch | @@ -187,16 +204,26 @@ providers: folder: '' type: file disableDeletion: false - editable: false + updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards options: path: /var/lib/grafana/dashboards ``` When Grafana starts, it will update/insert all dashboards available in the configured path. Then later on poll that path and look for updated json files and insert those update/insert those into the database. -### Reuseable dashboard urls +#### Making changes to a provisioned dashboard -If the dashboard in the json file contains an [uid](/reference/dashboard/#json-fields), Grafana will force insert/update on that uid. This allows you to migrate dashboards betweens Grafana instances and provisioning Grafana from configuration without breaking the urls given since the new dashboard url uses the uid as identifer. +It's possible to make changes to a provisioned dashboard in Grafana UI, but there's currently no possibility to automatically save the changes back to the provisioning source. +However, if you make changes to a provisioned dashboard you can `Save` the dashboard which will bring up a *Cannot save provisioned dashboard* dialog like seen in the screenshot below. +Here available options will let you `Copy JSON to Clipboard` and/or `Save JSON to file` which can help you synchronize your dashboard changes back to the provisioning source. + +Note: The JSON shown in input field and when using `Copy JSON to Clipboard` and/or `Save JSON to file` will have the `id` field automatically removed to aid the provisioning workflow. + +{{< docs-imagebox img="/img/docs/v51/provisioning_cannot_save_dashboard.png" max-width="500px" class="docs-image--no-shadow" >}} + +### Reusable Dashboard Urls + +If the dashboard in the json file contains an [uid](/reference/dashboard/#json-fields), Grafana will force insert/update on that uid. This allows you to migrate dashboards betweens Grafana instances and provisioning Grafana from configuration without breaking the urls given since the new dashboard url uses the uid as identifier. When Grafana starts, it will update/insert all dashboards available in the configured folders. If you modify the file, the dashboard will also be updated. By default Grafana will delete dashboards in the database if the file is removed. You can disable this behavior using the `disableDeletion` setting. diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index 40ef0a818f1..670ef9595f8 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -16,12 +16,11 @@ weight = 2 When an alert changes state, it sends out notifications. Each alert rule can have multiple notifications. In order to add a notification to an alert rule you first need -to add and configure a `notification` channel (can be email, PagerDuty or other integration). This is done from the Notification Channels page. +to add and configure a `notification` channel (can be email, PagerDuty or other integration). +This is done from the Notification Channels page. ## Notification Channel Setup -{{< imgbox max-width="30%" img="/img/docs/v50/alerts_notifications_menu.png" caption="Alerting Notification Channels" >}} - On the Notification Channels page hit the `New Channel` button to go the page where you can configure and setup a new Notification Channel. @@ -30,7 +29,31 @@ sure it's setup correctly. ### Send on all alerts -When checked, this option will nofity for all alert rules - existing and new. +When checked, this option will notify for all alert rules - existing and new. + +### Send reminders + +> Only available in Grafana v5.3 and above. + +{{< docs-imagebox max-width="600px" img="/img/docs/v53/alerting_notification_reminders.png" class="docs-image--right" caption="Alerting notification reminders setup" >}} + +When this option is checked additional notifications (reminders) will be sent for triggered alerts. You can specify how often reminders +should be sent using number of seconds (s), minutes (m) or hours (h), for example `30s`, `3m`, `5m` or `1h` etc. + +**Important:** Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent more frequently than a configured [alert rule evaluation interval](/alerting/rules/#name-evaluation-interval). + +These examples show how often and when reminders are sent for a triggered alert. + +Alert rule evaluation interval | Send reminders every | Reminder sent every (after last alert notification) +---------- | ----------- | ----------- +`30s` | `15s` | ~30 seconds +`1m` | `5m` | ~5 minutes +`5m` | `15m` | ~15 minutes +`6m` | `20m` | ~24 minutes +`1h` | `15m` | ~1 hour +`1h` | `2h` | ~2 hours + +
## Supported Notification Types @@ -41,6 +64,8 @@ Grafana ships with the following set of notification types: To enable email notifications you have to setup [SMTP settings](/installation/configuration/#smtp) in the Grafana config. Email notifications will upload an image of the alert graph to an external image destination if available or fallback to attaching the image to the email. +Be aware that if you use the `local` image storage email servers and clients might not be +able to access the image. ### Slack @@ -103,7 +128,7 @@ Example json body: In DingTalk PC Client: -1. Click "more" icon on left bottom of the panel. +1. Click "more" icon on upper right of the panel. 2. Click "Robot Manage" item in the pop menu, there will be a new panel call "Robot Manage". @@ -115,7 +140,7 @@ In DingTalk PC Client: 6. There will be a Webhook URL in the panel, looks like this: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxx. Copy this URL to the grafana Dingtalk setting page and then click "finish". -Dingtalk supports the following "message type": `text`, `link` and `markdown`. Only the `text` message type is supported. +Dingtalk supports the following "message type": `text`, `link` and `markdown`. Only the `link` message type is supported. ### Kafka @@ -152,14 +177,12 @@ Telegram | `telegram` | no Line | `line` | no Prometheus Alertmanager | `prometheus-alertmanager` | no - - # Enable images in notifications {#external-image-store} -Grafana can render the panel associated with the alert rule and include that in the notification. Most Notification Channels require that this image be publicly accessable (Slack and PagerDuty for example). In order to include images in alert notifications, Grafana can upload the image to an image store. It currently supports +Grafana can render the panel associated with the alert rule and include that in the notification. Most Notification Channels require that this image be publicly accessible (Slack and PagerDuty for example). In order to include images in alert notifications, Grafana can upload the image to an image store. It currently supports Amazon S3, Webdav, Google Cloud Storage and Azure Blob Storage. So to set that up you need to configure the [external image uploader](/installation/configuration/#external-image-storage) in your grafana-server ini config file. -Be aware that some notifiers requires public access to the image to be able to include it in the notification. So make sure to enable public access to the images. If your using local image uploader, your Grafana instance need to be accessible by the internet. +Be aware that some notifiers requires public access to the image to be able to include it in the notification. So make sure to enable public access to the images. If you're using local image uploader, your Grafana instance need to be accessible by the internet. Currently only the Email Channels attaches images if no external image store is specified. To include images in alert notifications for other channels then you need to set up an external image store. diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md index 9bbbd70641d..488619055e2 100644 --- a/docs/sources/alerting/rules.md +++ b/docs/sources/alerting/rules.md @@ -27,7 +27,9 @@ and the conditions that need to be met for the alert to change state and trigger ## Execution The alert rules are evaluated in the Grafana backend in a scheduler and query execution engine that is part -of core Grafana. Only some data sources are supported right now. They include `Graphite`, `Prometheus`, `InfluxDB`, `OpenTSDB`, `MySQL`, `Postgres` and `Cloudwatch`. +of core Grafana. Only some data sources are supported right now. They include `Graphite`, `Prometheus`, `Elasticsearch`, `InfluxDB`, `OpenTSDB`, `MySQL`, `Postgres` and `Cloudwatch`. + +> Alerting support for Elasticsearch is only available in Grafana v5.2 and above. ### Clustering @@ -86,6 +88,11 @@ So as you can see from the above scenario Grafana will not send out notification to fire if the rule already is in state `Alerting`. To improve support for queries that return multiple series we plan to track state **per series** in a future release. +> Starting with Grafana v5.3 you can configure reminders to be sent for triggered alerts. This will send additional notifications +> when an alert continues to fire. If other series (like server2 in the example above) also cause the alert rule to fire they will +> be included in the reminder notification. Depending on what notification channel you're using you may be able to take advantage +> of this feature for identifying new/existing series causing alert to fire. [Read more about notification reminders here](/alerting/notifications/#send-reminders). + ### No Data / Null values Below your conditions you can configure how the rule evaluation engine should handle queries that return no data or only null values. @@ -110,7 +117,7 @@ to `Keep Last State` in order to basically ignore them. ## Notifications -In alert tab you can also specify alert rule notifications along with a detailed messsage about the alert rule. +In alert tab you can also specify alert rule notifications along with a detailed message about the alert rule. The message can contain anything, information about how you might solve the issue, link to runbook, etc. The actual notifications are configured and shared between multiple alerts. Read the @@ -152,6 +159,8 @@ filters = alerting.scheduler:debug \ tsdb.prometheus:debug \ tsdb.opentsdb:debug \ tsdb.influxdb:debug \ + tsdb.elasticsearch:debug \ + tsdb.elasticsearch.client:debug \ ``` If you want to log raw query sent to your TSDB and raw response in log you also have to set grafana.ini option `app_mode` to diff --git a/docs/sources/tutorials/authproxy.md b/docs/sources/auth/auth-proxy.md similarity index 64% rename from docs/sources/tutorials/authproxy.md rename to docs/sources/auth/auth-proxy.md index 8003be20644..e066eed9190 100644 --- a/docs/sources/tutorials/authproxy.md +++ b/docs/sources/auth/auth-proxy.md @@ -1,42 +1,43 @@ +++ -title = "Grafana Authproxy" +title = "Auth Proxy" +description = "Grafana Auth Proxy Guide " +keywords = ["grafana", "configuration", "documentation", "proxy"] type = "docs" -keywords = ["grafana", "tutorials", "authproxy"] +aliases = ["/tutorials/authproxy/"] [menu.docs] -parent = "tutorials" -weight = 10 +name = "Auth Proxy" +identifier = "auth-proxy" +parent = "authentication" +weight = 2 +++ -# Grafana Authproxy +# Auth Proxy Authentication -AuthProxy allows you to offload the authentication of users to a web server (there are many reasons why you’d want to run a web server in front of a production version of Grafana, especially if it’s exposed to the Internet). +You can configure Grafana to let a http reverse proxy handling authentication. Popular web servers have a very +extensive list of pluggable authentication modules, and any of them can be used with the AuthProxy feature. +Below we detail the configuration options for auth proxy. -Popular web servers have a very extensive list of pluggable authentication modules, and any of them can be used with the AuthProxy feature. - -The Grafana AuthProxy feature is very simple in design, but it is this simplicity that makes it so powerful. - -## Interacting with Grafana’s AuthProxy via curl - -The AuthProxy feature can be configured through the Grafana configuration file with the following options: - -```js +```bash [auth.proxy] +# Defaults to false, but set to true to enable this feature enabled = true +# HTTP Header name that will contain the username or email header_name = X-WEBAUTH-USER +# HTTP Header property, defaults to `username` but can also be `email` header_property = username +# Set to `true` to enable auto sign up of users who do not exist in Grafana DB. Defaults to `true`. auto_sign_up = true +# If combined with Grafana LDAP integration define sync interval ldap_sync_ttl = 60 +# Limit where auth proxy requests come from by configuring a list of IP addresses. +# This can be used to prevent users spoofing the X-WEBAUTH-USER header. whitelist = +# Optionally define more headers to sync other user attributes +# Example `headers = Name:X-WEBAUTH-NAME Email:X-WEBAUTH-EMAIL`` +headers = ``` -* **enabled**: this is to toggle the feature on or off -* **header_name**: this is the HTTP header name that passes the username or email address of the authenticated user to Grafana. Grafana will trust what ever username is contained in this header and automatically log the user in. -* **header_property**: this tells Grafana whether the value in the header_name is a username or an email address. (In Grafana you can log in using your account username or account email) -* **auto_sign_up**: If set to true, Grafana will automatically create user accounts in the Grafana DB if one does not exist. If set to false, users who do not exist in the GrafanaDB won’t be able to log in, even though their username and password are valid. -* **ldap_sync_ttl**: When both auth.proxy and auth.ldap are enabled, user's organisation and role are synchronised from ldap after the http proxy authentication. You can force ldap re-synchronisation after `ldap_sync_ttl` minutes. -* **whitelist**: Comma separated list of trusted authentication proxies IP. - -With a fresh install of Grafana, using the above configuration for the authProxy feature, we can send a simple API call to list all users. The only user that will be present is the default “Admin” user that is added the first time Grafana starts up. As you can see all we need to do to authenticate the request is to provide the “X-WEBAUTH-USER” header. +## Interacting with Grafana’s AuthProxy via curl ```bash curl -H "X-WEBAUTH-USER: admin" http://localhost:3000/api/users @@ -71,7 +72,8 @@ I’ll demonstrate how to use Apache for authenticating users. In this example w ### Apache BasicAuth -In this example we use Apache as a reverseProxy in front of Grafana. Apache handles the Authentication of users before forwarding requests to the Grafana backend service. +In this example we use Apache as a reverse proxy in front of Grafana. Apache handles the Authentication of users before forwarding requests to the Grafana backend service. + #### Apache configuration @@ -108,7 +110,7 @@ In this example we use Apache as a reverseProxy in front of Grafana. Apache hand * The next part of the configuration is the tricky part. We use Apache’s rewrite engine to create our **X-WEBAUTH-USER header**, populated with the authenticated user. - * **RewriteRule .* - [E=PROXY_USER:%{LA-U:REMOTE_USER}, NS]**: This line is a little bit of magic. What it does, is for every request use the rewriteEngines look-ahead (LA-U) feature to determine what the REMOTE_USER variable would be set to after processing the request. Then assign the result to the variable PROXY_USER. This is neccessary as the REMOTE_USER variable is not available to the RequestHeader function. + * **RewriteRule .* - [E=PROXY_USER:%{LA-U:REMOTE_USER}, NS]**: This line is a little bit of magic. What it does, is for every request use the rewriteEngines look-ahead (LA-U) feature to determine what the REMOTE_USER variable would be set to after processing the request. Then assign the result to the variable PROXY_USER. This is necessary as the REMOTE_USER variable is not available to the RequestHeader function. * **RequestHeader set X-WEBAUTH-USER “%{PROXY_USER}e”**: With the authenticated username now stored in the PROXY_USER variable, we create a new HTTP request header that will be sent to our backend Grafana containing the username. @@ -116,40 +118,9 @@ In this example we use Apache as a reverseProxy in front of Grafana. Apache hand * The last 3 lines are then just standard reverse proxy configuration to direct all authenticated requests to our Grafana server running on port 3000. -#### Grafana configuration +## Full walk through using Docker. -```bash -############# Users ################ -[users] - # disable user signup / registration -allow_sign_up = false - -# Set to true to automatically assign new users to the default organization (id 1) -auto_assign_org = true - -# Default role new users will be automatically assigned (if auto_assign_org above is set to true) - auto_assign_org_role = Editor - - -############ Auth Proxy ######## -[auth.proxy] -enabled = true - -# the Header name that contains the authenticated user. -header_name = X-WEBAUTH-USER - -# does the user authenticate against the proxy using a 'username' or an 'email' -header_property = username - -# automatically add the user to the system if they don't already exist. -auto_sign_up = true -``` - -#### Full walk through using Docker. - -##### Grafana Container - -For this example, we use the offical Grafana docker image available at [Docker Hub](https://hub.docker.com/r/grafana/grafana/) +For this example, we use the official Grafana docker image available at [Docker Hub](https://hub.docker.com/r/grafana/grafana/) * Create a file `grafana.ini` with the following contents @@ -166,7 +137,8 @@ header_property = username auto_sign_up = true ``` -* Launch the Grafana container, using our custom grafana.ini to replace `/etc/grafana/grafana.ini`. We dont expose any ports for this container as it will only be connected to by our Apache container. +Launch the Grafana container, using our custom grafana.ini to replace `/etc/grafana/grafana.ini`. We don't expose +any ports for this container as it will only be connected to by our Apache container. ```bash docker run -i -v $(pwd)/grafana.ini:/etc/grafana/grafana.ini --name grafana grafana/grafana @@ -174,7 +146,7 @@ docker run -i -v $(pwd)/grafana.ini:/etc/grafana/grafana.ini --name grafana graf ### Apache Container -For this example we use the offical Apache docker image available at [Docker Hub](https://hub.docker.com/_/httpd/) +For this example we use the official Apache docker image available at [Docker Hub](https://hub.docker.com/_/httpd/) * Create a file `httpd.conf` with the following contents @@ -244,4 +216,4 @@ ProxyPassReverse / http://grafana:3000/ ### Use grafana. -With our Grafana and Apache containers running, you can now connect to http://localhost/ and log in using the username/password we created in the htpasswd file. \ No newline at end of file +With our Grafana and Apache containers running, you can now connect to http://localhost/ and log in using the username/password we created in the htpasswd file. diff --git a/docs/sources/auth/enhanced_ldap.md b/docs/sources/auth/enhanced_ldap.md new file mode 100644 index 00000000000..8eec57b1429 --- /dev/null +++ b/docs/sources/auth/enhanced_ldap.md @@ -0,0 +1,43 @@ ++++ +title = "Enhanced LDAP Integration" +description = "Grafana Enhanced LDAP Integration Guide " +keywords = ["grafana", "configuration", "documentation", "ldap", "active directory", "enterprise"] +type = "docs" +[menu.docs] +name = "Enhanced LDAP" +identifier = "enhanced-ldap" +parent = "authentication" +weight = 3 ++++ + +# Enhanced LDAP Integration + +> Enhanced LDAP Integration is only available in Grafana Enterprise. Read more about [Grafana Enterprise]({{< relref "enterprise/index.md" >}}). + +The enhanced LDAP integration adds additional functionality on top of the [existing LDAP integration]({{< relref "auth/ldap.md" >}}). + +## LDAP Group Synchronization for Teams + +{{< docs-imagebox img="/img/docs/enterprise/team_members_ldap.png" class="docs-image--no-shadow docs-image--right" max-width= "600px" >}} + +With the enhanced LDAP integration it's possible to setup synchronization between LDAP groups and teams. This enables LDAP users which are members +of certain LDAP groups to automatically be added/removed as members to certain teams in Grafana. Currently the synchronization will only happen every +time a user logs in, but an active background synchronization is currently being developed. + +Grafana keeps track of all synchronized users in teams and you can see which users have been synchronized from LDAP in the team members list, see `LDAP` label in screenshot. +This mechanism allows Grafana to remove an existing synchronized user from a team when its LDAP group membership changes. This mechanism also enables you to manually add +a user as member of a team and it will not be removed when the user signs in. This gives you flexibility to combine LDAP group memberships and Grafana team memberships. + +
+ +### Enable LDAP group synchronization for a team + +{{< docs-imagebox img="/img/docs/enterprise/team_add_external_group.png" class="docs-image--no-shadow docs-image--right" max-width= "600px" >}} + +1. Navigate to Configuration / Teams. +2. Select a team. +3. Select the External group sync tab and click on the `Add group` button. +4. Insert LDAP distinguished name (DN) of LDAP group you want to synchronize with the team. +5. Click on `Add group` button to save. + +
diff --git a/docs/sources/auth/generic-oauth.md b/docs/sources/auth/generic-oauth.md new file mode 100644 index 00000000000..6fa6531fc98 --- /dev/null +++ b/docs/sources/auth/generic-oauth.md @@ -0,0 +1,214 @@ ++++ +title = "OAuth authentication" +description = "Grafana OAuthentication Guide " +keywords = ["grafana", "configuration", "documentation", "oauth"] +type = "docs" +[menu.docs] +name = "Generic OAuth" +identifier = "generic_oauth" +parent = "authentication" +weight = 3 ++++ + +# Generic OAuth Authentication + +You can configure many different oauth2 authentication services with Grafana using the generic oauth2 feature. Below you +can find examples using Okta, BitBucket, OneLogin and Azure. + +This callback URL must match the full HTTP address that you use in your browser to access Grafana, but with the prefix path of `/login/generic_oauth`. + +You may have to set the `root_url` option of `[server]` for the callback URL to be +correct. For example in case you are serving Grafana behind a proxy. + +Example config: + +```bash +[auth.generic_oauth] +enabled = true +client_id = YOUR_APP_CLIENT_ID +client_secret = YOUR_APP_CLIENT_SECRET +scopes = +auth_url = +token_url = +api_url = +allowed_domains = mycompany.com mycompany.org +allow_sign_up = true +``` + +Set `api_url` to the resource that returns [OpenID UserInfo](https://connect2id.com/products/server/docs/api/userinfo) compatible information. + +Grafana will attempt to determine the user's e-mail address by querying the OAuth provider as described below in the following order until an e-mail address is found: + +1. Check for the presence of an e-mail address via the `email` field encoded in the OAuth `id_token` parameter. +2. Check for the presence of an e-mail address in the `attributes` map encoded in the OAuth `id_token` parameter. By default Grafana will perform a lookup into the attributes map using the `email:primary` key, however, this is configurable and can be adjusted by using the `email_attribute_name` configuration option. +3. Query the `/emails` endpoint of the OAuth provider's API (configured with `api_url`) and check for the presence of an e-mail address marked as a primary address. +4. If no e-mail address is found in steps (1-3), then the e-mail address of the user is set to the empty string. + +## Set up OAuth2 with Okta + +First set up Grafana as an OpenId client "webapplication" in Okta. Then set the Base URIs to `https:///` and set the Login redirect URIs to `https:///login/generic_oauth`. + +Finally set up the generic oauth module like this: +```bash +[auth.generic_oauth] +name = Okta +enabled = true +scopes = openid profile email +client_id = +client_secret = +auth_url = https:///oauth2/v1/authorize +token_url = https:///oauth2/v1/token +api_url = https:///oauth2/v1/userinfo +``` + +## Set up OAuth2 with Bitbucket + +```bash +[auth.generic_oauth] +name = BitBucket +enabled = true +allow_sign_up = true +client_id = +client_secret = +scopes = account email +auth_url = https://bitbucket.org/site/oauth2/authorize +token_url = https://bitbucket.org/site/oauth2/access_token +api_url = https://api.bitbucket.org/2.0/user +team_ids = +allowed_organizations = +``` + +## Set up OAuth2 with OneLogin + +1. Create a new Custom Connector with the following settings: + - Name: Grafana + - Sign On Method: OpenID Connect + - Redirect URI: `https:///login/generic_oauth` + - Signing Algorithm: RS256 + - Login URL: `https:///login/generic_oauth` + + then: +2. Add an App to the Grafana Connector: + - Display Name: Grafana + + then: +3. Under the SSO tab on the Grafana App details page you'll find the Client ID and Client Secret. + + Your OneLogin Domain will match the url you use to access OneLogin. + + Configure Grafana as follows: + + ```bash + [auth.generic_oauth] + name = OneLogin + enabled = true + allow_sign_up = true + client_id = + client_secret = + scopes = openid email name + auth_url = https://.onelogin.com/oidc/auth + token_url = https://.onelogin.com/oidc/token + api_url = https://.onelogin.com/oidc/me + team_ids = + allowed_organizations = + ``` + +### Set up OAuth2 with Auth0 + +1. Create a new Client in Auth0 + - Name: Grafana + - Type: Regular Web Application + +2. Go to the Settings tab and set: + - Allowed Callback URLs: `https:///login/generic_oauth` + +3. Click Save Changes, then use the values at the top of the page to configure Grafana: + + ```bash + [auth.generic_oauth] + enabled = true + allow_sign_up = true + team_ids = + allowed_organizations = + name = Auth0 + client_id = + client_secret = + scopes = openid profile email + auth_url = https:///authorize + token_url = https:///oauth/token + api_url = https:///userinfo + ``` + +### Set up OAuth2 with Azure Active Directory + +1. Log in to portal.azure.com and click "Azure Active Directory" in the side menu, then click the "Properties" sub-menu item. + +2. Copy the "Directory ID", this is needed for setting URLs later + +3. Click "App Registrations" and add a new application registration: + - Name: Grafana + - Application type: Web app / API + - Sign-on URL: `https:///login/generic_oauth` + +4. Click the name of the new application to open the application details page. + +5. Note down the "Application ID", this will be the OAuth client id. + +6. Click "Settings", then click "Keys" and add a new entry under Passwords + - Key Description: Grafana OAuth + - Duration: Never Expires + +7. Click Save then copy the key value, this will be the OAuth client secret. + +8. Configure Grafana as follows: + + ```bash + [auth.generic_oauth] + name = Azure AD + enabled = true + allow_sign_up = true + client_id = + client_secret = + scopes = openid email name + auth_url = https://login.microsoftonline.com//oauth2/authorize + token_url = https://login.microsoftonline.com//oauth2/token + api_url = + team_ids = + allowed_organizations = + ``` + +> Note: It's important to ensure that the [root_url](/installation/configuration/#root-url) in Grafana is set in your Azure Application Reply URLs (App -> Settings -> Reply URLs) + +## Set up OAuth2 with Centrify + +1. Create a new Custom OpenID Connect application configuration in the Centrify dashboard. + +2. Create a memorable unique Application ID, e.g. "grafana", "grafana_aws", etc. + +3. Put in other basic configuration (name, description, logo, category) + +4. On the Trust tab, generate a long password and put it into the OpenID Connect Client Secret field. + +5. Put the URL to the front page of your Grafana instance into the "Resource Application URL" field. + +6. Add an authorized Redirect URI like https://your-grafana-server/login/generic_oauth + +7. Set up permissions, policies, etc. just like any other Centrify app + +8. Configure Grafana as follows: + + ```bash + [auth.generic_oauth] + name = Centrify + enabled = true + allow_sign_up = true + client_id = + client_secret = .my.centrify.com/OAuth2/Authorize/ + token_url = https://.my.centrify.com/OAuth2/Token/ + ``` + +
+ + diff --git a/docs/sources/auth/github.md b/docs/sources/auth/github.md new file mode 100644 index 00000000000..b4ffc0fc2d4 --- /dev/null +++ b/docs/sources/auth/github.md @@ -0,0 +1,101 @@ ++++ +title = "Google OAuth2 Authentication" +description = "Grafana OAuthentication Guide " +keywords = ["grafana", "configuration", "documentation", "oauth"] +type = "docs" +[menu.docs] +name = "GitHub" +identifier = "github_oauth2" +parent = "authentication" +weight = 4 ++++ + +# GitHub OAuth2 Authentication + +To enable the GitHub OAuth2 you must register your application with GitHub. GitHub will generate a client ID and secret key for you to use. + +## Configure GitHub OAuth application + +You need to create a GitHub OAuth application (you find this under the GitHub +settings page). When you create the application you will need to specify +a callback URL. Specify this as callback: + +```bash +http://:/login/github +``` + +This callback URL must match the full HTTP address that you use in your +browser to access Grafana, but with the prefix path of `/login/github`. +When the GitHub OAuth application is created you will get a Client ID and a +Client Secret. Specify these in the Grafana configuration file. For +example: + +## Enable GitHub in Grafana + +```bash +[auth.github] +enabled = true +allow_sign_up = true +client_id = YOUR_GITHUB_APP_CLIENT_ID +client_secret = YOUR_GITHUB_APP_CLIENT_SECRET +scopes = user:email,read:org +auth_url = https://github.com/login/oauth/authorize +token_url = https://github.com/login/oauth/access_token +api_url = https://api.github.com/user +team_ids = +allowed_organizations = +``` + +You may have to set the `root_url` option of `[server]` for the callback URL to be +correct. For example in case you are serving Grafana behind a proxy. + +Restart the Grafana back-end. You should now see a GitHub login button +on the login page. You can now login or sign up with your GitHub +accounts. + +You may allow users to sign-up via GitHub authentication by setting the +`allow_sign_up` option to `true`. When this option is set to `true`, any +user successfully authenticating via GitHub authentication will be +automatically signed up. + +### team_ids + +Require an active team membership for at least one of the given teams on +GitHub. If the authenticated user isn't a member of at least one of the +teams they will not be able to register or authenticate with your +Grafana instance. For example: + +```bash +[auth.github] +enabled = true +client_id = YOUR_GITHUB_APP_CLIENT_ID +client_secret = YOUR_GITHUB_APP_CLIENT_SECRET +scopes = user:email,read:org +team_ids = 150,300 +auth_url = https://github.com/login/oauth/authorize +token_url = https://github.com/login/oauth/access_token +api_url = https://api.github.com/user +allow_sign_up = true +``` + +### allowed_organizations + +Require an active organization membership for at least one of the given +organizations on GitHub. If the authenticated user isn't a member of at least +one of the organizations they will not be able to register or authenticate with +your Grafana instance. For example + +```bash +[auth.github] +enabled = true +client_id = YOUR_GITHUB_APP_CLIENT_ID +client_secret = YOUR_GITHUB_APP_CLIENT_SECRET +scopes = user:email,read:org +auth_url = https://github.com/login/oauth/authorize +token_url = https://github.com/login/oauth/access_token +api_url = https://api.github.com/user +allow_sign_up = true +# space-delimited organization names +allowed_organizations = github google +``` + diff --git a/docs/sources/auth/gitlab.md b/docs/sources/auth/gitlab.md new file mode 100644 index 00000000000..56fc3b131a5 --- /dev/null +++ b/docs/sources/auth/gitlab.md @@ -0,0 +1,118 @@ ++++ +title = "Google OAuth2 Authentication" +description = "Grafana OAuthentication Guide " +keywords = ["grafana", "configuration", "documentation", "oauth"] +type = "docs" +[menu.docs] +name = "GitLab" +identifier = "gitlab_oauth" +parent = "authentication" +weight = 5 ++++ + +# GitLab OAuth2 Authentication + +To enable the GitLab OAuth2 you must register an application in GitLab. GitLab will generate a client ID and secret key for you to use. + +## Create GitLab OAuth keys + +You need to [create a GitLab OAuth application](https://docs.gitlab.com/ce/integration/oauth_provider.html). +Choose a descriptive *Name*, and use the following *Redirect URI*: + +``` +https://grafana.example.com/login/gitlab +``` + +where `https://grafana.example.com` is the URL you use to connect to Grafana. +Adjust it as needed if you don't use HTTPS or if you use a different port; for +instance, if you access Grafana at `http://203.0.113.31:3000`, you should use + +``` +http://203.0.113.31:3000/login/gitlab +``` + +Finally, select *api* as the *Scope* and submit the form. Note that if you're +not going to use GitLab groups for authorization (i.e. not setting +`allowed_groups`, see below), you can select *read_user* instead of *api* as +the *Scope*, thus giving a more restricted access to your GitLab API. + +You'll get an *Application Id* and a *Secret* in return; we'll call them +`GITLAB_APPLICATION_ID` and `GITLAB_SECRET` respectively for the rest of this +section. + +## Enable GitLab in Grafana + +Add the following to your Grafana configuration file to enable GitLab +authentication: + +```bash +[auth.gitlab] +enabled = false +allow_sign_up = false +client_id = GITLAB_APPLICATION_ID +client_secret = GITLAB_SECRET +scopes = api +auth_url = https://gitlab.com/oauth/authorize +token_url = https://gitlab.com/oauth/token +api_url = https://gitlab.com/api/v4 +allowed_groups = +``` + +You may have to set the `root_url` option of `[server]` for the callback URL to be +correct. For example in case you are serving Grafana behind a proxy. + +Restart the Grafana backend for your changes to take effect. + +If you use your own instance of GitLab instead of `gitlab.com`, adjust +`auth_url`, `token_url` and `api_url` accordingly by replacing the `gitlab.com` +hostname with your own. + +With `allow_sign_up` set to `false`, only existing users will be able to login +using their GitLab account, but with `allow_sign_up` set to `true`, *any* user +who can authenticate on GitLab will be able to login on your Grafana instance; +if you use the public `gitlab.com`, it means anyone in the world would be able +to login on your Grafana instance. + +You can can however limit access to only members of a given group or list of +groups by setting the `allowed_groups` option. + +### allowed_groups + +To limit access to authenticated users that are members of one or more [GitLab +groups](https://docs.gitlab.com/ce/user/group/index.html), set `allowed_groups` +to a comma- or space-separated list of groups. For instance, if you want to +only give access to members of the `example` group, set + + +```ini +allowed_groups = example +``` + +If you want to also give access to members of the subgroup `bar`, which is in +the group `foo`, set + +```ini +allowed_groups = example, foo/bar +``` + +Note that in GitLab, the group or subgroup name doesn't always match its +display name, especially if the display name contains spaces or special +characters. Make sure you always use the group or subgroup name as it appears +in the URL of the group or subgroup. + +Here's a complete example with `allow_sign_up` enabled, and access limited to +the `example` and `foo/bar` groups: + +```ini +[auth.gitlab] +enabled = true +allow_sign_up = true +client_id = GITLAB_APPLICATION_ID +client_secret = GITLAB_SECRET +scopes = api +auth_url = https://gitlab.com/oauth/authorize +token_url = https://gitlab.com/oauth/token +api_url = https://gitlab.com/api/v4 +allowed_groups = example, foo/bar +``` + diff --git a/docs/sources/auth/google.md b/docs/sources/auth/google.md new file mode 100644 index 00000000000..f7faf1a1097 --- /dev/null +++ b/docs/sources/auth/google.md @@ -0,0 +1,58 @@ ++++ +title = "Google OAuth2 Authentication" +description = "Grafana OAuthentication Guide " +keywords = ["grafana", "configuration", "documentation", "oauth"] +type = "docs" +[menu.docs] +name = "Google" +identifier = "ggogle_oauth2" +parent = "authentication" +weight = 3 ++++ + +# Google OAuth2 Authentication + +To enable the Google OAuth2 you must register your application with Google. Google will generate a client ID and secret key for you to use. + +## Create Google OAuth keys + +First, you need to create a Google OAuth Client: + +1. Go to https://console.developers.google.com/apis/credentials +2. Click the 'Create Credentials' button, then click 'OAuth Client ID' in the menu that drops down +3. Enter the following: + - Application Type: Web Application + - Name: Grafana + - Authorized Javascript Origins: https://grafana.mycompany.com + - Authorized Redirect URLs: https://grafana.mycompany.com/login/google + - Replace https://grafana.mycompany.com with the URL of your Grafana instance. +4. Click Create +5. Copy the Client ID and Client Secret from the 'OAuth Client' modal + +## Enable Google OAuth in Grafana + +Specify the Client ID and Secret in the [Grafana configuration file]({{< relref "installation/configuration.md#config-file-locations" >}}). For example: + +```bash +[auth.google] +enabled = true +client_id = CLIENT_ID +client_secret = CLIENT_SECRET +scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email +auth_url = https://accounts.google.com/o/oauth2/auth +token_url = https://accounts.google.com/o/oauth2/token +allowed_domains = mycompany.com mycompany.org +allow_sign_up = true +``` + +You may have to set the `root_url` option of `[server]` for the callback URL to be +correct. For example in case you are serving Grafana behind a proxy. + +Restart the Grafana back-end. You should now see a Google login button +on the login page. You can now login or sign up with your Google +accounts. The `allowed_domains` option is optional, and domains were separated by space. + +You may allow users to sign-up via Google authentication by setting the +`allow_sign_up` option to `true`. When this option is set to `true`, any +user successfully authenticating via Google authentication will be +automatically signed up. diff --git a/docs/sources/auth/index.md b/docs/sources/auth/index.md new file mode 100644 index 00000000000..7fdcc082319 --- /dev/null +++ b/docs/sources/auth/index.md @@ -0,0 +1,12 @@ ++++ +title = "Authentication" +description = "Authentication" +type = "docs" +[menu.docs] +name = "Authentication" +identifier = "authentication" +parent = "admin" +weight = 3 ++++ + + diff --git a/docs/sources/auth/ldap.md b/docs/sources/auth/ldap.md new file mode 100644 index 00000000000..4a884a60d15 --- /dev/null +++ b/docs/sources/auth/ldap.md @@ -0,0 +1,261 @@ ++++ +title = "LDAP Authentication" +description = "Grafana LDAP Authentication Guide " +keywords = ["grafana", "configuration", "documentation", "ldap", "active directory"] +type = "docs" +[menu.docs] +name = "LDAP" +identifier = "ldap" +parent = "authentication" +weight = 2 ++++ + +# LDAP Authentication + +The LDAP integration in Grafana allows your Grafana users to login with their LDAP credentials. You can also specify mappings between LDAP +group memberships and Grafana Organization user roles. + +## Supported LDAP Servers + +Grafana uses a [third-party LDAP library](https://github.com/go-ldap/ldap) under the hood that supports basic LDAP v3 functionality. +This means that you should be able to configure LDAP integration using any compliant LDAPv3 server, for example [OpenLDAP](#openldap) or +[Active Directory](#active-directory) among [others](https://en.wikipedia.org/wiki/Directory_service#LDAP_implementations). + +## Enable LDAP + +In order to use LDAP integration you'll first need to enable LDAP in the [main config file]({{< relref "installation/configuration.md" >}}) as well as specify the path to the LDAP +specific configuration file (default: `/etc/grafana/ldap.toml`). + +```bash +[auth.ldap] +# Set to `true` to enable LDAP integration (default: `false`) +enabled = true + +# Path to the LDAP specific configuration file (default: `/etc/grafana/ldap.toml`) +config_file = /etc/grafana/ldap.toml + +# Allow sign up should almost always be true (default) to allow new Grafana users to be created (if ldap authentication is ok). If set to +# false only pre-existing Grafana users will be able to login (if ldap authentication is ok). +allow_sign_up = true +``` + +## Grafana LDAP Configuration + +Depending on which LDAP server you're using and how that's configured your Grafana LDAP configuration may vary. +See [configuration examples](#configuration-examples) for more information. + +**LDAP specific configuration file (ldap.toml) example:** +```bash +[[servers]] +# Ldap server host (specify multiple hosts space separated) +host = "127.0.0.1" +# Default port is 389 or 636 if use_ssl = true +port = 389 +# Set to true if ldap server supports TLS +use_ssl = false +# Set to true if connect ldap server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS) +start_tls = false +# set to true if you want to skip ssl cert validation +ssl_skip_verify = false +# set to the path to your root CA certificate or leave unset to use system defaults +# root_ca_cert = "/path/to/certificate.crt" +# Authentication against LDAP servers requiring client certificates +# client_cert = "/path/to/client.crt" +# client_key = "/path/to/client.key" + +# Search user bind dn +bind_dn = "cn=admin,dc=grafana,dc=org" +# Search user bind password +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +bind_password = 'grafana' + +# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)" +# Allow login from email or username, example "(|(sAMAccountName=%s)(userPrincipalName=%s))" +search_filter = "(cn=%s)" + +# An array of base dns to search through +search_base_dns = ["dc=grafana,dc=org"] + +# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" +# group_search_filter_user_attribute = "distinguishedName" +# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] + +# Specify names of the ldap attributes your ldap uses +[servers.attributes] +name = "givenName" +surname = "sn" +username = "cn" +member_of = "memberOf" +email = "email" +``` + +### Bind + +#### Bind & Bind Password + +By default the configuration expects you to specify a bind DN and bind password. This should be a read only user that can perform LDAP searches. +When the user DN is found a second bind is performed with the user provided username & password (in the normal Grafana login form). + +```bash +bind_dn = "cn=admin,dc=grafana,dc=org" +bind_password = "grafana" +``` + +#### Single Bind Example + +If you can provide a single bind expression that matches all possible users, you can skip the second bind and bind against the user DN directly. +This allows you to not specify a bind_password in the configuration file. + +```bash +bind_dn = "cn=%s,o=users,dc=grafana,dc=org" +``` + +In this case you skip providing a `bind_password` and instead provide a `bind_dn` value with a `%s` somewhere. This will be replaced with the username entered in on the Grafana login page. +The search filter and search bases settings are still needed to perform the LDAP search to retrieve the other LDAP information (like LDAP groups and email). + +### POSIX schema +If your ldap server does not support the memberOf attribute add these options: + +```bash +## Group search filter, to retrieve the groups of which the user is a member (only set if memberOf attribute is not available) +group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" +## An array of the base DNs to search through for groups. Typically uses ou=groups +group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] +## the %s in the search filter will be replaced with the attribute defined below +group_search_filter_user_attribute = "uid" +``` + +Also set `member_of = "dn"` in the `[servers.attributes]` section. + +### Group Mappings + +In `[[servers.group_mappings]]` you can map an LDAP group to a Grafana organization and role. These will be synced every time the user logs in, with LDAP being +the authoritative source. So, if you change a user's role in the Grafana Org. Users page, this change will be reset the next time the user logs in. If you +change the LDAP groups of a user, the change will take effect the next time the user logs in. + +The first group mapping that an LDAP user is matched to will be used for the sync. If you have LDAP users that fit multiple mappings, the topmost mapping in the +TOML config will be used. + +**LDAP specific configuration file (ldap.toml) example:** +```bash +[[servers]] +# other settings omitted for clarity + +[[servers.group_mappings]] +group_dn = "cn=superadmins,dc=grafana,dc=org" +org_role = "Admin" +grafana_admin = true # Available in Grafana v5.3 and above + +[[servers.group_mappings]] +group_dn = "cn=admins,dc=grafana,dc=org" +org_role = "Admin" + +[[servers.group_mappings]] +group_dn = "cn=users,dc=grafana,dc=org" +org_role = "Editor" + +[[servers.group_mappings]] +group_dn = "*" +org_role = "Viewer" +``` + +Setting | Required | Description | Default +------------ | ------------ | ------------- | ------------- +`group_dn` | Yes | LDAP distinguished name (DN) of LDAP group. If you want to match all (or no LDAP groups) then you can use wildcard (`"*"`) | +`org_role` | Yes | Assign users of `group_dn` the organisation role `"Admin"`, `"Editor"` or `"Viewer"` | +`org_id` | No | The Grafana organization database id. Setting this allows for multiple group_dn's to be assigned to the same `org_role` provided the `org_id` differs | `1` (default org id) +`grafana_admin` | No | When `true` makes user of `group_dn` Grafana server admin. A Grafana server admin has admin access over all organisations and users. Available in Grafana v5.3 and above | `false` + +### Nested/recursive group membership + +Users with nested/recursive group membership must have an LDAP server that supports `LDAP_MATCHING_RULE_IN_CHAIN` +and configure `group_search_filter` in a way that it returns the groups the submitted username is a member of. + +**Active Directory example:** + +Active Directory groups store the Distinguished Names (DNs) of members, so your filter will need to know the DN for the user based only on the submitted username. +Multiple DN templates can be searched by combining filters with the LDAP OR-operator. Examples: + +```bash +group_search_filter = "(member:1.2.840.113556.1.4.1941:=CN=%s,[user container/OU])" +group_search_filter = "(|(member:1.2.840.113556.1.4.1941:=CN=%s,[user container/OU])(member:1.2.840.113556.1.4.1941:=CN=%s,[another user container/OU]))" +group_search_filter_user_attribute = "cn" +``` +For more information on AD searches see [Microsoft's Search Filter Syntax](https://docs.microsoft.com/en-us/windows/desktop/adsi/search-filter-syntax) documentation. + +For troubleshooting, by changing `member_of` in `[servers.attributes]` to "dn" it will show you more accurate group memberships when [debug is enabled](#troubleshooting). + +## Configuration examples + +### OpenLDAP + +[OpenLDAP](http://www.openldap.org/) is an open source directory service. + +**LDAP specific configuration file (ldap.toml):** +```bash +[[servers]] +host = "127.0.0.1" +port = 389 +use_ssl = false +start_tls = false +ssl_skip_verify = false +bind_dn = "cn=admin,dc=grafana,dc=org" +bind_password = 'grafana' +search_filter = "(cn=%s)" +search_base_dns = ["dc=grafana,dc=org"] + +[servers.attributes] +name = "givenName" +surname = "sn" +username = "cn" +member_of = "memberOf" +email = "email" + +# [[servers.group_mappings]] omitted for clarity +``` + +### Active Directory + +[Active Directory](https://technet.microsoft.com/en-us/library/hh831484(v=ws.11).aspx) is a directory service which is commonly used in Windows environments. + +Assuming the following Active Directory server setup: + +* IP address: `10.0.0.1` +* Domain: `CORP` +* DNS name: `corp.local` + +**LDAP specific configuration file (ldap.toml):** +```bash +[[servers]] +host = "10.0.0.1" +port = 3269 +use_ssl = true +start_tls = false +ssl_skip_verify = true +bind_dn = "CORP\\%s" +search_filter = "(sAMAccountName=%s)" +search_base_dns = ["dc=corp,dc=local"] + +[servers.attributes] +name = "givenName" +surname = "sn" +username = "sAMAccountName" +member_of = "memberOf" +email = "mail" + +# [[servers.group_mappings]] omitted for clarity +``` + +#### Port requirements + +In above example SSL is enabled and an encrypted port have been configured. If your Active Directory don't support SSL please change `enable_ssl = false` and `port = 389`. +Please inspect your Active Directory configuration and documentation to find the correct settings. For more information about Active Directory and port requirements see [link](https://technet.microsoft.com/en-us/library/dd772723(v=ws.10)). + +## Troubleshooting + +To troubleshoot and get more log info enable ldap debug logging in the [main config file]({{< relref "installation/configuration.md" >}}). + +```bash +[log] +filters = ldap:debug +``` diff --git a/docs/sources/auth/overview.md b/docs/sources/auth/overview.md new file mode 100644 index 00000000000..a372600ac46 --- /dev/null +++ b/docs/sources/auth/overview.md @@ -0,0 +1,86 @@ ++++ +title = "Overview" +description = "Overview for auth" +type = "docs" +[menu.docs] +name = "Overview" +identifier = "overview-auth" +parent = "authentication" +weight = 1 ++++ + +# User Authentication Overview + +Grafana provides many ways to authenticate users. Some authentication integrations also enable syncing user +permissions and org memberships. + +## OAuth Integrations + +- [Google OAuth]({{< relref "auth/google.md" >}}) +- [GitHub OAuth]({{< relref "auth/github.md" >}}) +- [Gitlab OAuth]({{< relref "auth/gitlab.md" >}}) +- [Generic OAuth]({{< relref "auth/generic-oauth.md" >}}) (Okta2, BitBucket, Azure, OneLogin, Auth0) + +## LDAP integrations + +- [LDAP Authentication]({{< relref "auth/ldap.md" >}}) (OpenLDAP, ActiveDirectory, etc) + +## Auth proxy + +- [Auth Proxy]({{< relref "auth/auth-proxy.md" >}}) If you want to handle authentication outside Grafana using a reverse + proxy. + +## Grafana Auth + +Grafana of course has a built in user authentication system with password authentication enabled by default. You can +disable authentication by enabling anonymous access. You can also hide login form and only allow login through an auth +provider (listed above). There is also options for allowing self sign up. + +### Anonymous authentication + +You can make Grafana accessible without any login required by enabling anonymous access in the configuration file. + +Example: + +```bash +[auth.anonymous] +enabled = true + +# Organization name that should be used for unauthenticated users +org_name = Main Org. + +# Role for unauthenticated users, other valid values are `Editor` and `Admin` +org_role = Viewer +``` + +If you change your organization name in the Grafana UI this setting needs to be updated to match the new name. + +### Basic authentication + +Basic auth is enabled by default and works with the built in Grafana user password authentication system and LDAP +authentication integration. + +To disable basic auth: + +```bash +[auth.basic] +enabled = false +``` + +### Disable login form + +You can hide the Grafana login form using the below configuration settings. + +```bash +[auth] +disable_login_form ⁼ true +``` + +### Hide sign-out menu + +Set to the option detailed below to true to hide sign-out menu link. Useful if you use an auth proxy. + +```bash +[auth] +disable_signout_menu = true +``` diff --git a/docs/sources/contribute/cla.md b/docs/sources/contribute/cla.md index b990187d809..a073a9a4eae 100644 --- a/docs/sources/contribute/cla.md +++ b/docs/sources/contribute/cla.md @@ -1,6 +1,6 @@ +++ title = "Contributor Licence Agreement (CLA)" -description = "Contributer Licence Agreement (CLA)" +description = "Contributor Licence Agreement (CLA)" type = "docs" aliases = ["/project/cla", "docs/contributing/cla.html"] [menu.docs] @@ -101,4 +101,4 @@ TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU [OR US]


-This CLA aggreement is based on the [Harmony Contributor Aggrement Template (combined)](http://www.harmonyagreements.org/agreements.html), [Creative Commons Attribution 3.0 Unported License](https://creativecommons.org/licenses/by/3.0/) +This CLA agreement is based on the [Harmony Contributor Agreement Template (combined)](http://www.harmonyagreements.org/agreements.html), [Creative Commons Attribution 3.0 Unported License](https://creativecommons.org/licenses/by/3.0/) diff --git a/docs/sources/enterprise/index.md b/docs/sources/enterprise/index.md new file mode 100644 index 00000000000..f65fa55f02b --- /dev/null +++ b/docs/sources/enterprise/index.md @@ -0,0 +1,67 @@ ++++ +title = "Grafana Enterprise" +description = "Grafana Enterprise overview" +keywords = ["grafana", "documentation", "datasource", "permissions", "ldap", "licensing", "enterprise"] +type = "docs" +[menu.docs] +name = "Grafana Enterprise" +identifier = "enterprise" +weight = 30 ++++ + +# Grafana Enterprise + +Grafana Enterprise is a commercial edition of Grafana that includes additional features not found in the open source +version. + +Building on everything you already know and love about Grafana, Grafana Enterprise adds premium data sources, +advanced authentication options, more permission controls, 24x7x365 support, and training from the core Grafana team. + +Grafana Enterprise includes all of the features found in the open source edition and more. + +___ + +### Enhanced LDAP Integration + +With Grafana Enterprise you can set up synchronization between LDAP Groups and Teams. [Learn More]({{< relref "auth/enhanced_ldap.md" >}}). + +### Datasource Permissions + +Datasource permissions allow you to restrict query access to only specific Teams and Users. [Learn More]({{< relref "permissions/datasource_permissions.md" >}}). + +### Premium Plugins + +With a Grafana Enterprise licence you will get access to premium plugins, including: + +* [Splunk](https://grafana.com/plugins/grafana-splunk-datasource) +* [AppDynamics](https://grafana.com/plugins/dlopes7-appdynamics-datasource) +* [DataDog](https://grafana.com/plugins/grafana-datadog-datasource) +* [Dynatrace](https://grafana.com/plugins/grafana-dynatrace-datasource) +* [New Relic](https://grafana.com/plugins/grafana-newrelic-datasource) + +## Try Grafana Enterprise + +You can learn more about Grafana Enterprise [here](https://grafana.com/enterprise). To purchase or obtain a trial license contact +the Grafana Labs [Sales Team](https://grafana.com/contact?about=support&topic=Grafana%20Enterprise). + +## License file management + +To download your Grafana Enterprise license log in to your [Grafana.com](https://grafana.com) account and go to your **Org +Profile**. In the side menu there is a section for Grafana Enterprise licenses. At the bottom of the license +details page there is **Download Token** link that will download the *license.jwt* file containing your license. + +Place the *license.jwt* file in Grafana's data folder. This is usually located at `/var/lib/grafana/data` on linux systems. + +You can also configure a custom location for the license file via the ini setting: + +```bash +[enterprise] +license_path = /company/secrets/license.jwt +``` + +This setting can also be set via ENV variable which is useful if you're running Grafana via docker and have a custom +volume where you have placed the license file. In this case set the ENV variable `GF_ENTERPRISE_LICENSE_PATH` to point +to the location of your license file. + + + diff --git a/docs/sources/features/datasources/cloudwatch.md b/docs/sources/features/datasources/cloudwatch.md index f7f8138b5e9..e2bcb50bb1d 100644 --- a/docs/sources/features/datasources/cloudwatch.md +++ b/docs/sources/features/datasources/cloudwatch.md @@ -43,6 +43,42 @@ server is running on AWS you can use IAM Roles and authentication will be handle Checkout AWS docs on [IAM Roles](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) +## IAM Policies + +Grafana needs permissions granted via IAM to be able to read CloudWatch metrics +and EC2 tags/instances/regions. You can attach these permissions to IAM roles and +utilize Grafana's built-in support for assuming roles. + +Here is a minimal policy example: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowReadingMetricsFromCloudWatch", + "Effect": "Allow", + "Action": [ + "cloudwatch:ListMetrics", + "cloudwatch:GetMetricStatistics", + "cloudwatch:GetMetricData" + ], + "Resource": "*" + }, + { + "Sid": "AllowReadingTagsInstancesRegionsFromEC2", + "Effect": "Allow", + "Action": [ + "ec2:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions" + ], + "Resource": "*" + } + ] +} +``` + ### AWS credentials file Create a file at `~/.aws/credentials`. That is the `HOME` path for user running grafana-server. @@ -81,6 +117,8 @@ and `dimension keys/values`. In place of `region` you can specify `default` to use the default region configured in the datasource for the query, e.g. `metrics(AWS/DynamoDB, default)` or `dimension_values(default, ..., ..., ...)`. +Read more about the available dimensions in the [CloudWatch Metrics and Dimensions Reference](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CW_Support_For_AWS.html). + Name | Description ------- | -------- *regions()* | Returns a list of regions AWS provides their service. @@ -173,3 +211,37 @@ Amazon provides 1 million CloudWatch API requests each month at no additional ch it costs $0.01 per 1,000 GetMetricStatistics or ListMetrics requests. For each query Grafana will issue a GetMetricStatistics request and every time you pick a dimension in the query editor Grafana will issue a ListMetrics request. + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +Using a credentials file +```yaml +apiVersion: 1 + +datasources: + - name: Cloudwatch + type: cloudwatch + jsonData: + authType: credentials + defaultRegion: eu-west-2 +``` + +Using `accessKey` and `secretKey` + +```yaml +apiVersion: 1 + +datasources: + - name: Cloudwatch + type: cloudwatch + jsonData: + authType: keys + defaultRegion: eu-west-2 + secureJsonData: + accessKey: "" + secretKey: "" +``` diff --git a/docs/sources/features/datasources/elasticsearch.md b/docs/sources/features/datasources/elasticsearch.md index 6ce17113a9b..aa60eb7cbc1 100644 --- a/docs/sources/features/datasources/elasticsearch.md +++ b/docs/sources/features/datasources/elasticsearch.md @@ -29,13 +29,19 @@ Name | Description *Name* | The data source name. This is how you refer to the data source in panels & queries. *Default* | Default data source means that it will be pre-selected for new panels. *Url* | The HTTP protocol, IP, and port of your Elasticsearch server. -*Access* | Proxy = access via Grafana backend, Direct = access directly from browser. +*Access* | Server (default) = URL needs to be accessible from the Grafana backend/server, Browser = URL needs to be accessible from the browser. -Proxy access means that the Grafana backend will proxy all requests from the browser, and send them on to the Data Source. This is useful because it can eliminate CORS (Cross Origin Site Resource) issues, as well as eliminate the need to disseminate authentication to the browser. +Access mode controls how requests to the data source will be handled. Server should be the preferred way if nothing else stated. -### Direct access +### Server access mode (Default) -If you select direct access you must update your Elasticsearch configuration to allow other domains to access +All requests will be made from the browser to Grafana backend/server which in turn will forward the requests to the data source and by that circumvent possible Cross-Origin Resource Sharing (CORS) requirements. The URL needs to be accessible from the grafana backend/server if you select this access mode. + +### Browser (Direct) access + +All requests will be made from the browser directly to the data source and may be subject to Cross-Origin Resource Sharing (CORS) requirements. The URL needs to be accessible from the browser if you select this access mode. + +If you select Browser access you must update your Elasticsearch configuration to allow other domains to access Elasticsearch from the browser. You do this by specifying these to options in your **elasticsearch.yml** config file. ```bash @@ -45,19 +51,35 @@ http.cors.allow-origin: "*" ### Index settings -![](/img/docs/elasticsearch/elasticsearch_ds_details.png) +![Elasticsearch Datasource Details](/img/docs/elasticsearch/elasticsearch_ds_details.png) Here you can specify a default for the `time field` and specify the name of your Elasticsearch index. You can use a time pattern for the index name or a wildcard. ### Elasticsearch version -Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences how queries are composed. Currently only 2.x and 5.x -are supported. +Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences how queries are composed. +Currently the versions available is 2.x, 5.x, 5.6+ or 6.0+. 5.6+ means a version of 5.6 or less than 6.0. 6.0+ means a version of 6.0 or higher, 6.3.2 for example. + +### Min time interval +A lower limit for the auto group by time interval. Recommended to be set to write frequency, for example `1m` if your data is written every minute. +This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a +number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported: + +Identifier | Description +------------ | ------------- +`y` | year +`M` | month +`w` | week +`d` | day +`h` | hour +`m` | minute +`s` | second +`ms` | millisecond ## Metric Query editor -![](/img/docs/elasticsearch/query_editor.png) +![Elasticsearch Query Editor](/img/docs/elasticsearch/query_editor.png) The Elasticsearch query editor allows you to select multiple metrics and group by multiple terms or filters. Use the plus and minus icons to the right to add/remove metrics or group by clauses. Some metrics and group by clauses haves options, click the option text to expand the row to view and edit metric or group by options. @@ -93,7 +115,7 @@ The Elasticsearch data source supports two types of queries you can use in the * Query | Description ------------ | ------------- -*{"find": "fields", "type": "keyword"} | Returns a list of field names with the index type `keyword`. +*{"find": "fields", "type": "keyword"}* | Returns a list of field names with the index type `keyword`. *{"find": "terms", "field": "@hostname", "size": 1000}* | Returns a list of values for a field using term aggregation. Query will user current dashboard time range as time range for query. *{"find": "terms", "field": "@hostname", "query": ''}* | Returns a list of values for a field using term aggregation & and a specified lucene query filter. Query will use current dashboard time range as time range for query. @@ -137,3 +159,23 @@ Query | You can leave the search query blank or specify a lucene query Time | The name of the time field, needs to be date field. Text | Event description field. Tags | Optional field name to use for event tags (can be an array or a CSV string). + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: Elastic + type: elasticsearch + access: proxy + database: "[metrics-]YYYY.MM.DD" + url: http://localhost:9200 + jsonData: + interval: Daily + timeField: "@timestamp" +``` diff --git a/docs/sources/features/datasources/graphite.md b/docs/sources/features/datasources/graphite.md index 05a7df7fea8..8c819726977 100644 --- a/docs/sources/features/datasources/graphite.md +++ b/docs/sources/features/datasources/graphite.md @@ -20,7 +20,7 @@ queries through the use of query references. ## Adding the data source 1. Open the side menu by clicking the Grafana icon in the top header. -2. In the side menu under the `Dashboards` link you should find a link named `Data Sources`. +2. In the side menu under the `Configuration` link you should find a link named `Data Sources`. 3. Click the `+ Add data source` button in the top header. 4. Select `Graphite` from the *Type* dropdown. @@ -31,20 +31,28 @@ Name | Description *Name* | The data source name. This is how you refer to the data source in panels & queries. *Default* | Default data source means that it will be pre-selected for new panels. *Url* | The HTTP protocol, IP, and port of your graphite-web or graphite-api install. -*Access* | Proxy = access via Grafana backend, Direct = access directly from browser. +*Access* | Server (default) = URL needs to be accessible from the Grafana backend/server, Browser = URL needs to be accessible from the browser. -Proxy access means that the Grafana backend will proxy all requests from the browser, and send them on to the Data Source. This is useful because it can eliminate CORS (Cross Origin Site Resource) issues, as well as eliminate the need to disseminate authentication details to the browser. +Access mode controls how requests to the data source will be handled. Server should be the preferred way if nothing else stated. + +### Server access mode (Default) + +All requests will be made from the browser to Grafana backend/server which in turn will forward the requests to the data source and by that circumvent possible Cross-Origin Resource Sharing (CORS) requirements. The URL needs to be accessible from the grafana backend/server if you select this access mode. + +### Browser access mode + +All requests will be made from the browser directly to the data source and may be subject to Cross-Origin Resource Sharing (CORS) requirements. The URL needs to be accessible from the browser if you select this access mode. ## Metric editor ### Navigate metric segments + Click the ``Select metric`` link to start navigating the metric space. One you start you can continue using the mouse or keyboard arrow keys. You can select a wildcard and still continue. {{< docs-imagebox img="/img/docs/v45/graphite_query1_still.png" animated-gif="/img/docs/v45/graphite_query1.gif" >}} - ### Functions Click the plus icon to the right to add a function. You can search for the function or select it from the menu. Once @@ -55,7 +63,6 @@ by the x icon. {{< docs-imagebox img="/img/docs/v45/graphite_query2_still.png" animated-gif="/img/docs/v45/graphite_query2.gif" >}} - ### Optional parameters Some functions like aliasByNode support an optional second argument. To add this parameter specify for example 3,-2 as the first parameter and the function editor will adapt and move the -2 to a second parameter. To remove the second optional parameter just click on it and leave it blank and the editor will remove it. @@ -63,7 +70,6 @@ Some functions like aliasByNode support an optional second argument. To add this {{< docs-imagebox img="/img/docs/v45/graphite_query3_still.png" animated-gif="/img/docs/v45/graphite_query3.gif" >}} - ### Nested Queries You can reference queries by the row “letter” that they’re on (similar to Microsoft Excel). If you add a second query to a graph, you can reference the first query simply by typing in #A. This provides an easy and convenient way to build compounded queries. @@ -71,11 +77,10 @@ You can reference queries by the row “letter” that they’re on (similar to {{< docs-imagebox img="/img/docs/v45/graphite_nested_queries_still.png" animated-gif="/img/docs/v45/graphite_nested_queries.gif" >}} - ## Point consolidation All Graphite metrics are consolidated so that Graphite doesn't return more data points than there are pixels in the graph. By default, -this consolidation is done using `avg` function. You can how Graphite consolidates metrics by adding the Graphite consolidateBy function. +this consolidation is done using `avg` function. You can control how Graphite consolidates metrics by adding the Graphite consolidateBy function. > *Notice* This means that legend summary values (max, min, total) cannot be all correct at the same time. They are calculated > client side by Grafana. And depending on your consolidation function only one or two can be correct at the same time. @@ -89,6 +94,18 @@ being displayed in your dashboard. Checkout the [Templating]({{< relref "reference/templating.md" >}}) documentation for an introduction to the templating feature and the different types of template variables. +Graphite 1.1 introduced tags and Grafana added support for Graphite queries with tags in version 5.0. To create a variable using tag values, then you need to use the Grafana functions `tags` and `tag_values`. + +Query | Description +------------ | ------------- +*tags()* | Returns all tags. +*tags(server=~backend\*)* | Returns only tags that occur in series matching the filter expression. +*tag_values(server)* | Return tag values for the specified tag. +*tag_values(server, server=~backend\*)* | Returns filtered tag values that occur for the specified tag in series matching those expressions. +*tag_values(server, server=~backend\*, app=~${apps:regex})* | Multiple filter expressions and expressions can contain other variables. + +For more details, see the [Graphite docs on the autocomplete api for tags](http://graphite.readthedocs.io/en/latest/tags.html#auto-complete-support). + ### Query variable The query you specify in the query field should be a metric find type of query. For example, a query like `prod.servers.*` will fill the @@ -97,10 +114,10 @@ variable with all possible values that exist in the wildcard position. You can also create nested variables that use other variables in their definition. For example `apps.$app.servers.*` uses the variable `$app` in its query definition. -### Variable usage +### Variable Usage You can use a variable in a metric node path or as a parameter to a function. -![](/img/docs/v2/templated_variable_parameter.png) +![variable](/img/docs/v2/templated_variable_parameter.png) There are two syntaxes: @@ -113,6 +130,18 @@ the second syntax in expressions like `my.server[[serverNumber]].count`. Example: [Graphite Templated Dashboard](http://play.grafana.org/dashboard/db/graphite-templated-nested) +### Variable Usage in Tag Queries + +Multi-value variables in tag queries use the advanced formatting syntax introduced in Grafana 5.0 for variables: `{var:regex}`. Non-tag queries will use the default glob formatting for multi-value variables. + +Example of a tag expression with regex formatting and using the Equal Tilde operator, `=~`: + +```text +server=~${servers:regex} +``` + +Checkout the [Advanced Formatting Options section in the Variables]({{< relref "reference/templating.md#advanced-formatting-options" >}}) documentation for examples and details. + ## Annotations [Annotations]({{< relref "reference/annotations.md" >}}) allows you to overlay rich event information on top of graphs. You add annotation @@ -120,3 +149,21 @@ queries via the Dashboard menu / Annotations view. Graphite supports two ways to query annotations. A regular metric query, for this you use the `Graphite query` textbox. A Graphite events query, use the `Graphite event tags` textbox, specify a tag or wildcard (leave empty should also work) + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + jsonData: + graphiteVersion: "1.1" +``` diff --git a/docs/sources/features/datasources/index.md b/docs/sources/features/datasources/index.md index 54606d20988..a892f38a448 100644 --- a/docs/sources/features/datasources/index.md +++ b/docs/sources/features/datasources/index.md @@ -30,6 +30,7 @@ The following datasources are officially supported: * [Prometheus]({{< relref "prometheus.md" >}}) * [MySQL]({{< relref "mysql.md" >}}) * [Postgres]({{< relref "postgres.md" >}}) +* [Microsoft SQL Server (MSSQL)]({{< relref "mssql.md" >}}) ## Data source plugins diff --git a/docs/sources/features/datasources/influxdb.md b/docs/sources/features/datasources/influxdb.md index 6d0918a0d01..bc96190e9b1 100644 --- a/docs/sources/features/datasources/influxdb.md +++ b/docs/sources/features/datasources/influxdb.md @@ -28,16 +28,36 @@ Name | Description *Name* | The data source name. This is how you refer to the data source in panels & queries. *Default* | Default data source means that it will be pre-selected for new panels. *Url* | The http protocol, ip and port of you influxdb api (influxdb api port is by default 8086) -*Access* | Proxy = access via Grafana backend, Direct = access directly from browser. +*Access* | Server (default) = URL needs to be accessible from the Grafana backend/server, Browser = URL needs to be accessible from the browser. *Database* | Name of your influxdb database *User* | Name of your database user *Password* | Database user's password -### Proxy vs Direct access +Access mode controls how requests to the data source will be handled. Server should be the preferred way if nothing else stated. -Proxy access means that the Grafana backend will proxy all requests from the browser. So requests to InfluxDB will be channeled through -`grafana-server`. This means that the URL you specify needs to be accessible from the server you are running Grafana on. Proxy access -mode is also more secure as the username & password will never reach the browser. +### Server access mode (Default) + +All requests will be made from the browser to Grafana backend/server which in turn will forward the requests to the data source and by that circumvent possible Cross-Origin Resource Sharing (CORS) requirements. The URL needs to be accessible from the grafana backend/server if you select this access mode. + +### Browser access mode + +All requests will be made from the browser directly to the data source and may be subject to Cross-Origin Resource Sharing (CORS) requirements. The URL needs to be accessible from the browser if you select this access mode. + +### Min time interval +A lower limit for the auto group by time interval. Recommended to be set to write frequency, for example `1m` if your data is written every minute. +This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a +number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported: + +Identifier | Description +------------ | ------------- +`y` | year +`M` | month +`w` | week +`d` | day +`h` | hour +`m` | minute +`s` | second +`ms` | millisecond ## Query Editor @@ -168,9 +188,28 @@ queries via the Dashboard menu / Annotations view. An example query: ```SQL -SELECT title, description from events WHERE $timeFilter order asc +SELECT title, description from events WHERE $timeFilter ORDER BY time ASC ``` For InfluxDB you need to enter a query like in the above example. You need to have the ```where $timeFilter``` part. If you only select one column you will not need to enter anything in the column mapping fields. The Tags field can be a comma separated string. + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: InfluxDB + type: influxdb + access: proxy + database: site + user: grafana + password: grafana + url: http://localhost:8086 +``` diff --git a/docs/sources/features/datasources/mssql.md b/docs/sources/features/datasources/mssql.md new file mode 100644 index 00000000000..cd191f14273 --- /dev/null +++ b/docs/sources/features/datasources/mssql.md @@ -0,0 +1,599 @@ ++++ +title = "Using Microsoft SQL Server in Grafana" +description = "Guide for using Microsoft SQL Server in Grafana" +keywords = ["grafana", "MSSQL", "Microsoft", "SQL", "guide", "Azure SQL Database"] +type = "docs" +[menu.docs] +name = "Microsoft SQL Server" +parent = "datasources" +weight = 8 ++++ + +# Using Microsoft SQL Server in Grafana + +> Only available in Grafana v5.1+. + +Grafana ships with a built-in Microsoft SQL Server (MSSQL) data source plugin that allows you to query and visualize data from any Microsoft SQL Server 2005 or newer, including Microsoft Azure SQL Database. + +## Adding the data source + +1. Open the side menu by clicking the Grafana icon in the top header. +2. In the side menu under the `Configuration` link you should find a link named `Data Sources`. +3. Click the `+ Add data source` button in the top header. +4. Select *Microsoft SQL Server* from the *Type* dropdown. + +### Data source options + +Name | Description +------------ | ------------- +*Name* | The data source name. This is how you refer to the data source in panels & queries. +*Default* | Default data source means that it will be pre-selected for new panels. +*Host* | The IP address/hostname and optional port of your MSSQL instance. If port is omitted, default 1433 will be used. +*Database* | Name of your MSSQL database. +*User* | Database user's login/username +*Password* | Database user's password +*Encrypt* | This option determines whether or to which extent a secure SSL TCP/IP connection will be negotiated with the server, default `false` (Grafana v5.4+). +*Max open* | The maximum number of open connections to the database, default `unlimited` (Grafana v5.4+). +*Max idle* | The maximum number of connections in the idle connection pool, default `2` (Grafana v5.4+). +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400`/4 hours (Grafana v5.4+). + +### Min time interval + +A lower limit for the [$__interval](/reference/templating/#the-interval-variable) and [$__interval_ms](/reference/templating/#the-interval-ms-variable) variables. +Recommended to be set to write frequency, for example `1m` if your data is written every minute. +This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a +number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported: + +Identifier | Description +------------ | ------------- +`y` | year +`M` | month +`w` | week +`d` | day +`h` | hour +`m` | minute +`s` | second +`ms` | millisecond + +### Database User Permissions (Important!) + +The database user you specify when you add the data source should only be granted SELECT permissions on +the specified database & tables you want to query. Grafana does not validate that the query is safe. The query +could include any SQL statement. For example, statements like `DELETE FROM user;` and `DROP TABLE user;` would be +executed. To protect against this we **Highly** recommend you create a specific MSSQL user with restricted permissions. + +Example: + +```sql + CREATE USER grafanareader WITH PASSWORD 'password' + GRANT SELECT ON dbo.YourTable3 TO grafanareader +``` + +Make sure the user does not get any unwanted privileges from the public role. + +### Known Issues + +If you're using an older version of Microsoft SQL Server like 2008 and 2008R2 you may need to disable encryption to be able to connect. +If possible, we recommend you to use the latest service pack available for optimal compatibility. + +## Query Editor + +{{< docs-imagebox img="/img/docs/v51/mssql_query_editor.png" class="docs-image--no-shadow" >}} + +You find the MSSQL query editor in the metrics tab in Graph, Singlestat or Table panel's edit mode. You enter edit mode by clicking the +panel title, then edit. The editor allows you to define a SQL query to select data to be visualized. + +1. Select *Format as* `Time series` (for use in Graph or Singlestat panel's among others) or `Table` (for use in Table panel among others). +2. This is the actual editor where you write your SQL queries. +3. Show help section for MSSQL below the query editor. +4. Show actual executed SQL query. Will be available first after a successful query has been executed. +5. Add an additional query where an additional query editor will be displayed. + +
+ +## Macros + +To simplify syntax and to allow for dynamic parts, like date range filters, the query can contain macros. + +Macro example | Description +------------ | ------------- +*$__time(dateColumn)* | Will be replaced by an expression to rename the column to *time*. For example, *dateColumn as time* +*$__timeEpoch(dateColumn)* | Will be replaced by an expression to convert a DATETIME column type to unix timestamp and rename it to *time*.
For example, *DATEDIFF(second, '1970-01-01', dateColumn) AS time* +*$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name.
For example, *dateColumn BETWEEN '2017-04-21T05:01:17Z' AND '2017-04-21T05:06:17Z'* +*$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *'2017-04-21T05:01:17Z'* +*$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *'2017-04-21T05:06:17Z'* +*$__timeGroup(dateColumn,'5m'[, fillvalue])* | Will be replaced by an expression usable in GROUP BY clause. Providing a *fillValue* of *NULL* or *floating value* will automatically fill empty series in timerange with that value.
For example, *CAST(ROUND(DATEDIFF(second, '1970-01-01', time_column)/300.0, 0) as bigint)\*300*. +*$__timeGroup(dateColumn,'5m', 0)* | Same as above but with a fill parameter so missing points in that series will be added by grafana and 0 will be used as value. +*$__timeGroup(dateColumn,'5m', NULL)* | Same as above but NULL will be used as value for missing points. +*$__timeGroup(dateColumn,'5m', previous)* | Same as above but the previous value in that series will be used as fill value if no value has been seen yet NULL will be used (only available in Grafana 5.3+). +*$__timeGroupAlias(dateColumn,'5m')* | Will be replaced identical to $__timeGroup but with an added column alias (only available in Grafana 5.3+). +*$__unixEpochFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name with times represented as unix timestamp. For example, *dateColumn > 1494410783 AND dateColumn < 1494497183* +*$__unixEpochFrom()* | Will be replaced by the start of the currently active time selection as unix timestamp. For example, *1494410783* +*$__unixEpochTo()* | Will be replaced by the end of the currently active time selection as unix timestamp. For example, *1494497183* +*$__unixEpochGroup(dateColumn,'5m', [fillmode])* | Same as $__timeGroup but for times stored as unix timestamp (only available in Grafana 5.3+). +*$__unixEpochGroupAlias(dateColumn,'5m', [fillmode])* | Same as above but also adds a column alias (only available in Grafana 5.3+). + +We plan to add many more macros. If you have suggestions for what macros you would like to see, please [open an issue](https://github.com/grafana/grafana) in our GitHub repo. + +The query editor has a link named `Generated SQL` that shows up after a query has been executed, while in panel edit mode. Click on it and it will expand and show the raw interpolated SQL string that was executed. + +## Table queries + +If the `Format as` query option is set to `Table` then you can basically do any type of SQL query. The table panel will automatically show the results of whatever columns & rows your query returns. + +**Example database table:** + +```sql +CREATE TABLE [event] ( + time_sec bigint, + description nvarchar(100), + tags nvarchar(100), +) +``` + +```sql +CREATE TABLE [mssql_types] ( + c_bit bit, c_tinyint tinyint, c_smallint smallint, c_int int, c_bigint bigint, c_money money, c_smallmoney smallmoney, c_numeric numeric(10,5), + c_real real, c_decimal decimal(10,2), c_float float, + c_char char(10), c_varchar varchar(10), c_text text, + c_nchar nchar(12), c_nvarchar nvarchar(12), c_ntext ntext, + c_datetime datetime, c_datetime2 datetime2, c_smalldatetime smalldatetime, c_date date, c_time time, c_datetimeoffset datetimeoffset +) + +INSERT INTO [mssql_types] +SELECT + 1, 5, 20020, 980300, 1420070400, '$20000.15', '£2.15', 12345.12, + 1.11, 2.22, 3.33, + 'char10', 'varchar10', 'text', + N'☺nchar12☺', N'☺nvarchar12☺', N'☺text☺', + GETDATE(), CAST(GETDATE() AS DATETIME2), CAST(GETDATE() AS SMALLDATETIME), CAST(GETDATE() AS DATE), CAST(GETDATE() AS TIME), SWITCHOFFSET(CAST(GETDATE() AS DATETIMEOFFSET), '-07:00')) +``` + +Query editor with example query: + +{{< docs-imagebox img="/img/docs/v51/mssql_table_query.png" max-width="500px" class="docs-image--no-shadow" >}} + + +The query: + +```sql +SELECT * FROM [mssql_types] +``` + +You can control the name of the Table panel columns by using regular `AS ` SQL column selection syntax. Example: + +```sql +SELECT + c_bit as [column1], c_tinyint as [column2] +FROM + [mssql_types] +``` + +The resulting table panel: + +{{< docs-imagebox img="/img/docs/v51/mssql_table_result.png" max-width="1489px" class="docs-image--no-shadow" >}} + +## Time series queries + +If you set `Format as` to `Time series`, for use in Graph panel for example, then the query must must have a column named `time` that returns either a sql datetime or any numeric datatype representing unix epoch in seconds. You may return a column named `metric` that is used as metric name for the value column. Any column except `time` and `metric` is treated as a value column. If you omit the `metric` column, the name of the value column will be the metric name. You may select multiple value columns, each will have its name as metric. +If you return multiple value columns and a column named `metric` then this column is used as prefix for the series name (only available in Grafana 5.3+). + +Resultsets of time series queries need to be sorted by time. + +**Example database table:** + +```sql +CREATE TABLE [event] ( + time_sec bigint, + description nvarchar(100), + tags nvarchar(100), +) +``` + + +```sql +CREATE TABLE metric_values ( + time datetime, + measurement nvarchar(100), + valueOne int, + valueTwo int, +) + +INSERT metric_values (time, measurement, valueOne, valueTwo) VALUES('2018-03-15 12:30:00', 'Metric A', 62, 6) +INSERT metric_values (time, measurement, valueOne, valueTwo) VALUES('2018-03-15 12:30:00', 'Metric B', 49, 11) +... +INSERT metric_values (time, measurement, valueOne, valueTwo) VALUES('2018-03-15 13:55:00', 'Metric A', 14, 25) +INSERT metric_values (time, measurement, valueOne, valueTwo) VALUES('2018-03-15 13:55:00', 'Metric B', 48, 10) + +``` + +{{< docs-imagebox img="/img/docs/v51/mssql_time_series_one.png" class="docs-image--no-shadow docs-image--right" >}} + +**Example with one `value` and one `metric` column.** + +```sql +SELECT + time, + valueOne, + measurement as metric +FROM + metric_values +WHERE + $__timeFilter(time) +ORDER BY 1 +``` + +When above query are used in a graph panel the result will be two series named `Metric A` and `Metric B` with value of `valueOne` and `valueTwo` plotted over `time`. + +
+ +{{< docs-imagebox img="/img/docs/v51/mssql_time_series_two.png" class="docs-image--no-shadow docs-image--right" >}} + +**Example with multiple `value` columns:** + +```sql +SELECT + time, + valueOne, + valueTwo +FROM + metric_values +WHERE + $__timeFilter(time) +ORDER BY 1 +``` + +When above query are used in a graph panel the result will be two series named `valueOne` and `valueTwo` with value of `valueOne` and `valueTwo` plotted over `time`. + +
+ +{{< docs-imagebox img="/img/docs/v51/mssql_time_series_three.png" class="docs-image--no-shadow docs-image--right" >}} + +**Example using the $__timeGroup macro:** + +```sql +SELECT + $__timeGroup(time, '3m') as time, + measurement as metric, + avg(valueOne) +FROM + metric_values +WHERE + $__timeFilter(time) +GROUP BY + $__timeGroup(time, '3m'), + measurement +ORDER BY 1 +``` + +When above query are used in a graph panel the result will be two series named `Metric A` and `Metric B` with an average of `valueOne` plotted over `time`. +Any two series lacking a value in a 3 minute window will render a line between those two lines. You'll notice that the graph to the right never goes down to zero. + +
+ +{{< docs-imagebox img="/img/docs/v51/mssql_time_series_four.png" class="docs-image--no-shadow docs-image--right" >}} + +**Example using the $__timeGroup macro with fill parameter set to zero:** + +```sql +SELECT + $__timeGroup(time, '3m', 0) as time, + measurement as metric, + sum(valueTwo) +FROM + metric_values +WHERE + $__timeFilter(time) +GROUP BY + $__timeGroup(time, '3m'), + measurement +ORDER BY 1 +``` + +When above query are used in a graph panel the result will be two series named `Metric A` and `Metric B` with a sum of `valueTwo` plotted over `time`. +Any series lacking a value in a 3 minute window will have a value of zero which you'll see rendered in the graph to the right. + +## Templating + +Instead of hard-coding things like server, application and sensor name in you metric queries you can use variables in their place. Variables are shown as dropdown select boxes at the top of the dashboard. These dropdowns makes it easy to change the data being displayed in your dashboard. + +Checkout the [Templating]({{< relref "reference/templating.md" >}}) documentation for an introduction to the templating feature and the different types of template variables. + +### Query Variable + +If you add a template variable of the type `Query`, you can write a MSSQL query that can +return things like measurement names, key names or key values that are shown as a dropdown select box. + +For example, you can have a variable that contains all values for the `hostname` column in a table if you specify a query like this in the templating variable *Query* setting. + +```sql +SELECT hostname FROM host +``` + +A query can return multiple columns and Grafana will automatically create a list from them. For example, the query below will return a list with values from `hostname` and `hostname2`. + +```sql +SELECT [host].[hostname], [other_host].[hostname2] FROM host JOIN other_host ON [host].[city] = [other_host].[city] +``` + +Another option is a query that can create a key/value variable. The query should return two columns that are named `__text` and `__value`. The `__text` column value should be unique (if it is not unique then the first value is used). The options in the dropdown will have a text and value that allows you to have a friendly name as text and an id as the value. An example query with `hostname` as the text and `id` as the value: + +```sql +SELECT hostname __text, id __value FROM host +``` + +You can also create nested variables. For example if you had another variable named `region`. Then you could have +the hosts variable only show hosts from the current selected region with a query like this (if `region` is a multi-value variable then use the `IN` comparison operator rather than `=` to match against multiple values): + +```sql +SELECT hostname FROM host WHERE region IN ($region) +``` + +### Using Variables in Queries + +> From Grafana 4.3.0 to 4.6.0, template variables are always quoted automatically so if it is a string value do not wrap them in quotes in where clauses. +> +> From Grafana 5.0.0, template variable values are only quoted when the template variable is a `multi-value`. + +If the variable is a multi-value variable then use the `IN` comparison operator rather than `=` to match against multiple values. + +There are two syntaxes: + +`$` Example with a template variable named `hostname`: + +```sql +SELECT + atimestamp time, + aint value +FROM table +WHERE $__timeFilter(atimestamp) and hostname in($hostname) +ORDER BY atimestamp +``` + +`[[varname]]` Example with a template variable named `hostname`: + +```sql +SELECT + atimestamp as time, + aint as value +FROM table +WHERE $__timeFilter(atimestamp) and hostname in([[hostname]]) +ORDER BY atimestamp +``` + +#### Disabling Quoting for Multi-value Variables + +Grafana automatically creates a quoted, comma-separated string for multi-value variables. For example: if `server01` and `server02` are selected then it will be formatted as: `'server01', 'server02'`. Do disable quoting, use the csv formatting option for variables: + +`${servers:csv}` + +Read more about variable formatting options in the [Variables]({{< relref "reference/templating.md#advanced-formatting-options" >}}) documentation. + +## Annotations + +[Annotations]({{< relref "reference/annotations.md" >}}) allows you to overlay rich event information on top of graphs. You add annotation queries via the Dashboard menu / Annotations view. + +**Columns:** + +Name | Description +------------ | ------------- +time | The name of the date/time field. Could be a column with a native sql date/time data type or epoch value. +text | Event description field. +tags | Optional field name to use for event tags as a comma separated string. + +**Example database tables:** + +```sql +CREATE TABLE [events] ( + time_sec bigint, + description nvarchar(100), + tags nvarchar(100), +) +``` + +We also use the database table defined in [Time series queries](#time-series-queries). + +**Example query using time column with epoch values:** + +```sql +SELECT + time_sec as time, + description as [text], + tags +FROM + [events] +WHERE + $__unixEpochFilter(time_sec) +ORDER BY 1 +``` + +**Example query using time column of native sql date/time data type:** + +```sql +SELECT + time, + measurement as text, + convert(varchar, valueOne) + ',' + convert(varchar, valueTwo) as tags +FROM + metric_values +WHERE + $__timeFilter(time_column) +ORDER BY 1 +``` + +## Stored procedure support + +Stored procedures have been verified to work. However, please note that we haven't done anything special to support this why there may exist edge cases where it won't work as you would expect. +Stored procedures should be supported in table, time series and annotation queries as long as you use the same naming of columns and return data in the same format as describe above under respective section. + +Please note that any macro function will not work inside a stored procedure. + +### Examples + +{{< docs-imagebox img="/img/docs/v51/mssql_metrics_graph.png" class="docs-image--no-shadow docs-image--right" >}} +For the following examples the database table defined in [Time series queries](#time-series-queries). Let's say that we want to visualize 4 series in a graph panel, i.e. all combinations of columns `valueOne`, `valueTwo` and `measurement`. Graph panel to the right visualizes what we want to achieve. To solve this we actually need to use two queries: + +**First query:** + +```sql +SELECT + $__timeGroup(time, '5m') as time, + measurement + ' - value one' as metric, + avg(valueOne) as valueOne +FROM + metric_values +WHERE + $__timeFilter(time) +GROUP BY + $__timeGroup(time, '5m'), + measurement +ORDER BY 1 +``` + +**Second query:** +```sql +SELECT + $__timeGroup(time, '5m') as time, + measurement + ' - value two' as metric, + avg(valueTwo) as valueTwo +FROM + metric_values +GROUP BY + $__timeGroup(time, '5m'), + measurement +ORDER BY 1 +``` + +#### Stored procedure using time in epoch format + +We can define a stored procedure that will return all data we need to render 4 series in a graph panel like above. +In this case the stored procedure accepts two parameters `@from` and `@to` of `int` data types which should be a timerange (from-to) in epoch format +which will be used to filter the data to return from the stored procedure. + +We're mimicking the `$__timeGroup(time, '5m')` in the select and group by expressions and that's why there's a lot of lengthy expressions needed - +these could be extracted to MSSQL functions, if wanted. + +```sql +CREATE PROCEDURE sp_test_epoch( + @from int, + @to int +) AS +BEGIN + SELECT + cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time, + measurement + ' - value one' as metric, + avg(valueOne) as value + FROM + metric_values + WHERE + time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01') + GROUP BY + cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int), + measurement + UNION ALL + SELECT + cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time, + measurement + ' - value two' as metric, + avg(valueTwo) as value + FROM + metric_values + WHERE + time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01') + GROUP BY + cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int), + measurement + ORDER BY 1 +END +``` + +Then we can use the following query for our graph panel. + +```sql +DECLARE + @from int = $__unixEpochFrom(), + @to int = $__unixEpochTo() + +EXEC dbo.sp_test_epoch @from, @to +``` + +#### Stored procedure using time in datetime format + +We can define a stored procedure that will return all data we need to render 4 series in a graph panel like above. +In this case the stored procedure accepts two parameters `@from` and `@to` of `datetime` data types which should be a timerange (from-to) +which will be used to filter the data to return from the stored procedure. + +We're mimicking the `$__timeGroup(time, '5m')` in the select and group by expressions and that's why there's a lot of lengthy expressions needed - +these could be extracted to MSSQL functions, if wanted. + +```sql +CREATE PROCEDURE sp_test_datetime( + @from datetime, + @to datetime +) AS +BEGIN + SELECT + cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time, + measurement + ' - value one' as metric, + avg(valueOne) as value + FROM + metric_values + WHERE + time >= @from AND time <= @to + GROUP BY + cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int), + measurement + UNION ALL + SELECT + cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time, + measurement + ' - value two' as metric, + avg(valueTwo) as value + FROM + metric_values + WHERE + time >= @from AND time <= @to + GROUP BY + cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int), + measurement + ORDER BY 1 +END + +``` + +Then we can use the following query for our graph panel. + +```sql +DECLARE + @from datetime = $__timeFrom(), + @to datetime = $__timeTo() + +EXEC dbo.sp_test_datetime @from, @to +``` + +## Alerting + +Time series queries should work in alerting conditions. Table formatted queries are not yet supported in alert rule +conditions. + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: MSSQL + type: mssql + url: localhost:1433 + database: grafana + user: grafana + jsonData: + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + secureJsonData: + password: "Password!" + +``` diff --git a/docs/sources/features/datasources/mysql.md b/docs/sources/features/datasources/mysql.md index 7fae7441b6d..bc4e4df6cf9 100644 --- a/docs/sources/features/datasources/mysql.md +++ b/docs/sources/features/datasources/mysql.md @@ -12,6 +12,8 @@ weight = 7 # Using MySQL in Grafana > Only available in Grafana v4.3+. +> +> Starting from Grafana v5.1 you can name the time column *time* in addition to earlier supported *time_sec*. Usage of *time_sec* will eventually be deprecated. Grafana ships with a built-in MySQL data source plugin that allow you to query any visualize data from a MySQL compatible database. @@ -23,12 +25,44 @@ data from a MySQL compatible database. 3. Click the `+ Add data source` button in the top header. 4. Select *MySQL* from the *Type* dropdown. +### Data source options + +Name | Description +------------ | ------------- +*Name* | The data source name. This is how you refer to the data source in panels & queries. +*Default* | Default data source means that it will be pre-selected for new panels. +*Host* | The IP address/hostname and optional port of your MySQL instance. +*Database* | Name of your MySQL database. +*User* | Database user's login/username +*Password* | Database user's password +*Max open* | The maximum number of open connections to the database, default `unlimited` (Grafana v5.4+). +*Max idle* | The maximum number of connections in the idle connection pool, default `2` (Grafana v5.4+). +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400`/4 hours. This should always be lower than configured [wait_timeout](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_wait_timeout) in MySQL (Grafana v5.4+). + +### Min time interval + +A lower limit for the [$__interval](/reference/templating/#the-interval-variable) and [$__interval_ms](/reference/templating/#the-interval-ms-variable) variables. +Recommended to be set to write frequency, for example `1m` if your data is written every minute. +This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a +number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported: + +Identifier | Description +------------ | ------------- +`y` | year +`M` | month +`w` | week +`d` | day +`h` | hour +`m` | minute +`s` | second +`ms` | millisecond + ### Database User Permissions (Important!) The database user you specify when you add the data source should only be granted SELECT permissions on the specified database & tables you want to query. Grafana does not validate that the query is safe. The query could include any SQL statement. For example, statements like `USE otherdb;` and `DROP TABLE user;` would be -executed. To protect against this we **Highly** recommmend you create a specific mysql user with restricted permissions. +executed. To protect against this we **Highly** recommend you create a specific mysql user with restricted permissions. Example: @@ -39,6 +73,58 @@ Example: You can use wildcards (`*`) in place of database or table if you want to grant access to more databases and tables. +## Query Editor + +> Only available in Grafana v5.4+. + +{{< docs-imagebox img="/img/docs/v54/mysql_query_still.png" class="docs-image--no-shadow" animated-gif="/img/docs/v54/mysql_query.gif" >}} + +You find the MySQL query editor in the metrics tab in a panel's edit mode. You enter edit mode by clicking the +panel title, then edit. + +The query editor has a link named `Generated SQL` that shows up after a query has been executed, while in panel edit mode. Click on it and it will expand and show the raw interpolated SQL string that was executed. + +### Select table, time column and metric column (FROM) + +When you enter edit mode for the first time or add a new query Grafana will try to prefill the query builder with the first table that has a timestamp column and a numeric column. + +In the FROM field, Grafana will suggest tables that are in the configured database. To select a table or view in another database that your database user has access to you can manually enter a fully qualified name (database.table) like `otherDb.metrics`. + +The Time column field refers to the name of the column holding your time values. Selecting a value for the Metric column field is optional. If a value is selected, the Metric column field will be used as the series name. + +The metric column suggestions will only contain columns with a text datatype (text, tinytext, mediumtext, longtext, varchar, char). +If you want to use a column with a different datatype as metric column you may enter the column name with a cast: `CAST(numericColumn as CHAR)`. +You may also enter arbitrary SQL expressions in the metric column field that evaluate to a text datatype like +`CONCAT(column1, " ", CAST(numericColumn as CHAR))`. + +### Columns and Aggregation functions (SELECT) + +In the `SELECT` row you can specify what columns and functions you want to use. +In the column field you may write arbitrary expressions instead of a column name like `column1 * column2 / column3`. + +If you use aggregate functions you need to group your resultset. The editor will automatically add a `GROUP BY time` if you add an aggregate function. + +You may add further value columns by clicking the plus button and selecting `Column` from the menu. Multiple value columns will be plotted as separate series in the graph panel. + +### Filter data (WHERE) +To add a filter click the plus icon to the right of the `WHERE` condition. You can remove filters by clicking on +the filter and selecting `Remove`. A filter for the current selected timerange is automatically added to new queries. + +### Group By +To group by time or any other columns click the plus icon at the end of the GROUP BY row. The suggestion dropdown will only show text columns of your currently selected table but you may manually enter any column. +You can remove the group by clicking on the item and then selecting `Remove`. + +If you add any grouping, all selected columns need to have an aggregate function applied. The query builder will automatically add aggregate functions to all columns without aggregate functions when you add groupings. + +#### Gap Filling + +Grafana can fill in missing values when you group by time. The time function accepts two arguments. The first argument is the time window that you would like to group by, and the second argument is the value you want Grafana to fill missing items with. + +### Text Editor Mode (RAW) +You can switch to the raw query editor mode by clicking the hamburger icon and selecting `Switch editor mode` or by clicking `Edit SQL` below the query. + +> If you use the raw query editor, be sure your query at minimum has `ORDER BY time` and a filter on the returned time range. + ## Macros To simplify syntax and to allow for dynamic parts, like date range filters, the query can contain macros. @@ -46,13 +132,20 @@ To simplify syntax and to allow for dynamic parts, like date range filters, the Macro example | Description ------------ | ------------- *$__time(dateColumn)* | Will be replaced by an expression to convert to a UNIX timestamp and rename the column to `time_sec`. For example, *UNIX_TIMESTAMP(dateColumn) as time_sec* -*$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *dateColumn > FROM_UNIXTIME(1494410783) AND dateColumn < FROM_UNIXTIME(1494497183)* -*$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *FROM_UNIXTIME(1494410783)* -*$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *FROM_UNIXTIME(1494497183)* -*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *cast(cast(UNIX_TIMESTAMP(dateColumn)/(300) as signed)*300 as signed) as time_sec,* +*$__timeEpoch(dateColumn)* | Will be replaced by an expression to convert to a UNIX timestamp and rename the column to `time_sec`. For example, *UNIX_TIMESTAMP(dateColumn) as time_sec* +*$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *dateColumn BETWEEN '2017-04-21T05:01:17Z' AND '2017-04-21T05:06:17Z'* +*$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *'2017-04-21T05:01:17Z'* +*$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *'2017-04-21T05:06:17Z'* +*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *cast(cast(UNIX_TIMESTAMP(dateColumn)/(300) as signed)*300 as signed),* +*$__timeGroup(dateColumn,'5m', 0)* | Same as above but with a fill parameter so missing points in that series will be added by grafana and 0 will be used as value. +*$__timeGroup(dateColumn,'5m', NULL)* | Same as above but NULL will be used as value for missing points. +*$__timeGroup(dateColumn,'5m', previous)* | Same as above but the previous value in that series will be used as fill value if no value has been seen yet NULL will be used (only available in Grafana 5.3+). +*$__timeGroupAlias(dateColumn,'5m')* | Will be replaced identical to $__timeGroup but with an added column alias (only available in Grafana 5.3+). *$__unixEpochFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name with times represented as unix timestamp. For example, *dateColumn > 1494410783 AND dateColumn < 1494497183* *$__unixEpochFrom()* | Will be replaced by the start of the currently active time selection as unix timestamp. For example, *1494410783* *$__unixEpochTo()* | Will be replaced by the end of the currently active time selection as unix timestamp. For example, *1494497183* +*$__unixEpochGroup(dateColumn,'5m', [fillmode])* | Same as $__timeGroup but for times stored as unix timestamp (only available in Grafana 5.3+). +*$__unixEpochGroupAlias(dateColumn,'5m', [fillmode])* | Same as above but also adds a column alias (only available in Grafana 5.3+). We plan to add many more macros. If you have suggestions for what macros you would like to see, please [open an issue](https://github.com/grafana/grafana) in our GitHub repo. @@ -84,39 +177,53 @@ The resulting table panel: ![](/img/docs/v43/mysql_table.png) -### Time series queries +## Time series queries -If you set `Format as` to `Time series`, for use in Graph panel for example, then there are some requirements for -what your query returns. +If you set `Format as` to `Time series`, for use in Graph panel for example, then the query must return a column named `time` that returns either a sql datetime or any numeric datatype representing unix epoch. +Any column except `time` and `metric` is treated as a value column. +You may return a column named `metric` that is used as metric name for the value column. +If you return multiple value columns and a column named `metric` then this column is used as prefix for the series name (only available in Grafana 5.3+). -- Must be a column named `time_sec` representing a unix epoch in seconds. -- Must be a column named `value` representing the time series value. -- Must be a column named `metric` representing the time series name. +Resultsets of time series queries need to be sorted by time. -Example: +**Example with `metric` column:** ```sql SELECT - min(UNIX_TIMESTAMP(time_date_time)) as time_sec, - max(value_double) as value, - metric1 as metric -FROM test_data -WHERE $__timeFilter(time_date_time) -GROUP BY metric1, UNIX_TIMESTAMP(time_date_time) DIV 300 -ORDER BY time_sec asc -``` - -Example with $__timeGroup macro: - -```sql -SELECT - $__timeGroup(time_date_time,'5m') as time_sec, - min(value_double) as value, - metric_name as metric + $__timeGroup(time_date_time,'5m'), + min(value_double), + 'min' as metric FROM test_data WHERE $__timeFilter(time_date_time) -GROUP BY 1, metric_name -ORDER BY 1 +GROUP BY time +ORDER BY time +``` + +**Example using the fill parameter in the $__timeGroup macro to convert null values to be zero instead:** + +```sql +SELECT + $__timeGroup(createdAt,'5m',0), + sum(value_double) as value, + measurement +FROM test_data +WHERE + $__timeFilter(createdAt) +GROUP BY time, measurement +ORDER BY time +``` + +**Example with multiple columns:** + +```sql +SELECT + $__timeGroup(time_date_time,'5m'), + min(value_double) as min_value, + max(value_double) as max_value +FROM test_data +WHERE $__timeFilter(time_date_time) +GROUP BY time +ORDER BY time ``` Currently, there is no support for a dynamic group by time based on time range & panel width. @@ -180,7 +287,7 @@ There are two syntaxes: ```sql SELECT - UNIX_TIMESTAMP(atimestamp) as time_sec, + UNIX_TIMESTAMP(atimestamp) as time, aint as value, avarchar as metric FROM my_table @@ -192,7 +299,7 @@ ORDER BY atimestamp ASC ```sql SELECT - UNIX_TIMESTAMP(atimestamp) as time_sec, + UNIX_TIMESTAMP(atimestamp) as time, aint as value, avarchar as metric FROM my_table @@ -200,28 +307,72 @@ WHERE $__timeFilter(atimestamp) and hostname in([[hostname]]) ORDER BY atimestamp ASC ``` +#### Disabling Quoting for Multi-value Variables + +Grafana automatically creates a quoted, comma-separated string for multi-value variables. For example: if `server01` and `server02` are selected then it will be formatted as: `'server01', 'server02'`. Do disable quoting, use the csv formatting option for variables: + +`${servers:csv}` + +Read more about variable formatting options in the [Variables]({{< relref "reference/templating.md#advanced-formatting-options" >}}) documentation. + ## Annotations -[Annotations]({{< relref "reference/annotations.md" >}}) allows you to overlay rich event information on top of graphs. You add annotation queries via the Dashboard menu / Annotations view. +[Annotations]({{< relref "reference/annotations.md" >}}) allow you to overlay rich event information on top of graphs. You add annotation queries via the Dashboard menu / Annotations view. -An example query: +**Example query using time column with epoch values:** ```sql SELECT - UNIX_TIMESTAMP(atimestamp) as time_sec, - value as text, + epoch_time as time, + metric1 as text, CONCAT(tag1, ',', tag2) as tags -FROM my_table -WHERE $__timeFilter(atimestamp) -ORDER BY atimestamp ASC +FROM + public.test_data +WHERE + $__unixEpochFilter(epoch_time) +``` + +**Example query using time column of native sql date/time data type:** + +```sql +SELECT + native_date_time as time, + metric1 as text, + CONCAT(tag1, ',', tag2) as tags +FROM + public.test_data +WHERE + $__timeFilter(native_date_time) ``` Name | Description ------------ | ------------- -time_sec | The name of the date/time field. +time | The name of the date/time field. Could be a column with a native sql date/time data type or epoch value. text | Event description field. tags | Optional field name to use for event tags as a comma separated string. ## Alerting Time series queries should work in alerting conditions. Table formatted queries is not yet supported in alert rule conditions. + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: MySQL + type: mysql + url: localhost:3306 + database: grafana + user: grafana + password: password + jsonData: + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ +``` diff --git a/docs/sources/features/datasources/opentsdb.md b/docs/sources/features/datasources/opentsdb.md index 03795473ff7..d2cd0b1dc0e 100644 --- a/docs/sources/features/datasources/opentsdb.md +++ b/docs/sources/features/datasources/opentsdb.md @@ -28,11 +28,10 @@ Name | Description *Name* | The data source name. This is how you refer to the data source in panels & queries. *Default* | Default data source means that it will be pre-selected for new panels. *Url* | The http protocol, ip and port of you opentsdb server (default port is usually 4242) -*Access* | Proxy = access via Grafana backend, Direct = access directly from browser. +*Access* | Server (default) = URL needs to be accessible from the Grafana backend/server, Browser = URL needs to be accessible from the browser. *Version* | Version = opentsdb version, either <=2.1 or 2.2 *Resolution* | Metrics from opentsdb may have datapoints with either second or millisecond resolution. - ## Query editor Open a graph in edit mode by click the title. Query editor will differ if the datasource has version <=2.1 or = 2.2. @@ -78,13 +77,32 @@ the existing time series data in OpenTSDB, you need to run `tsdb uid metasync` o ### Nested Templating -One template variable can be used to filter tag values for another template varible. First parameter is the metric name, +One template variable can be used to filter tag values for another template variable. First parameter is the metric name, second parameter is the tag key for which you need to find tag values, and after that all other dependent template variables. Some examples are mentioned below to make nested template queries work successfully. Query | Description ------------ | ------------- *tag_values(cpu, hostname, env=$env)* | Return tag values for cpu metric, selected env tag value and tag key hostname -*tag_values(cpu, hostanme, env=$env, region=$region)* | Return tag values for cpu metric, selected env tag value, selected region tag value and tag key hostname +*tag_values(cpu, hostname, env=$env, region=$region)* | Return tag values for cpu metric, selected env tag value, selected region tag value and tag key hostname For details on OpenTSDB metric queries checkout the official [OpenTSDB documentation](http://opentsdb.net/docs/build/html/index.html) + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: OpenTsdb + type: opentsdb + access: proxy + url: http://localhost:4242 + jsonData: + tsdbResolution: 1 + tsdbVersion: 1 +``` diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index 7d52df2fd3e..52f8804f27f 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -16,16 +16,51 @@ Grafana ships with a built-in PostgreSQL data source plugin that allows you to q ## Adding the data source 1. Open the side menu by clicking the Grafana icon in the top header. -2. In the side menu under the `Dashboards` link you should find a link named `Data Sources`. +2. In the side menu under the `Configuration` icon you should find a link named `Data Sources`. 3. Click the `+ Add data source` button in the top header. 4. Select *PostgreSQL* from the *Type* dropdown. +### Data source options + +Name | Description +------------ | ------------- +*Name* | The data source name. This is how you refer to the data source in panels & queries. +*Default* | Default data source means that it will be pre-selected for new panels. +*Host* | The IP address/hostname and optional port of your PostgreSQL instance. +*Database* | Name of your PostgreSQL database. +*User* | Database user's login/username +*Password* | Database user's password +*SSL Mode* | This option determines whether or with what priority a secure SSL TCP/IP connection will be negotiated with the server. +*Max open* | The maximum number of open connections to the database, default `unlimited` (Grafana v5.4+). +*Max idle* | The maximum number of connections in the idle connection pool, default `2` (Grafana v5.4+). +*Max lifetime* | The maximum amount of time in seconds a connection may be reused, default `14400`/4 hours (Grafana v5.4+). +*Version* | This option determines which functions are available in the query builder (only available in Grafana 5.3+). +*TimescaleDB* | TimescaleDB is a time-series database built as a PostgreSQL extension. If enabled, Grafana will use `time_bucket` in the `$__timeGroup` macro and display TimescaleDB specific aggregate functions in the query builder (only available in Grafana 5.3+). + +### Min time interval + +A lower limit for the [$__interval](/reference/templating/#the-interval-variable) and [$__interval_ms](/reference/templating/#the-interval-ms-variable) variables. +Recommended to be set to write frequency, for example `1m` if your data is written every minute. +This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a +number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported: + +Identifier | Description +------------ | ------------- +`y` | year +`M` | month +`w` | week +`d` | day +`h` | hour +`m` | minute +`s` | second +`ms` | millisecond + ### Database User Permissions (Important!) The database user you specify when you add the data source should only be granted SELECT permissions on the specified database & tables you want to query. Grafana does not validate that the query is safe. The query could include any SQL statement. For example, statements like `DELETE FROM user;` and `DROP TABLE user;` would be -executed. To protect against this we **Highly** recommmend you create a specific postgresql user with restricted permissions. +executed. To protect against this we **highly** recommend you create a specific PostgreSQL user with restricted permissions. Example: @@ -37,26 +72,93 @@ Example: Make sure the user does not get any unwanted privileges from the public role. +## Query Editor + +> Only available in Grafana v5.3+. + +{{< docs-imagebox img="/img/docs/v53/postgres_query_still.png" class="docs-image--no-shadow" animated-gif="/img/docs/v53/postgres_query.gif" >}} + +You find the PostgreSQL query editor in the metrics tab in Graph or Singlestat panel's edit mode. You enter edit mode by clicking the +panel title, then edit. + +The query editor has a link named `Generated SQL` that shows up after a query has been executed, while in panel edit mode. Click on it and it will expand and show the raw interpolated SQL string that was executed. + +### Select table, time column and metric column (FROM) + +When you enter edit mode for the first time or add a new query Grafana will try to prefill the query builder with the first table that has a timestamp column and a numeric column. + +In the FROM field, Grafana will suggest tables that are in the `search_path` of the database user. To select a table or view not in your `search_path` +you can manually enter a fully qualified name (schema.table) like `public.metrics`. + +The Time column field refers to the name of the column holding your time values. Selecting a value for the Metric column field is optional. If a value is selected, the Metric column field will be used as the series name. + +The metric column suggestions will only contain columns with a text datatype (char,varchar,text). +If you want to use a column with a different datatype as metric column you may enter the column name with a cast: `ip::text`. +You may also enter arbitrary SQL expressions in the metric column field that evaluate to a text datatype like +`hostname || ' ' || container_name`. + +### Columns, Window and Aggregation functions (SELECT) + +In the `SELECT` row you can specify what columns and functions you want to use. +In the column field you may write arbitrary expressions instead of a column name like `column1 * column2 / column3`. + +The available functions in the query editor depend on the PostgreSQL version you selected when configuring the datasource. +If you use aggregate functions you need to group your resultset. The editor will automatically add a `GROUP BY time` if you add an aggregate function. + +The editor tries to simplify and unify this part of the query. For example:
+![](/img/docs/v53/postgres_select_editor.png)
+ +The above will generate the following PostgreSQL `SELECT` clause: + +```sql +avg(tx_bytes) OVER (ORDER BY "time" ROWS 5 PRECEDING) AS "tx_bytes" +``` + +You may add further value columns by clicking the plus button and selecting `Column` from the menu. Multiple value columns will be plotted as separate series in the graph panel. + +### Filter data (WHERE) +To add a filter click the plus icon to the right of the `WHERE` condition. You can remove filters by clicking on +the filter and selecting `Remove`. A filter for the current selected timerange is automatically added to new queries. + +### Group By +To group by time or any other columns click the plus icon at the end of the GROUP BY row. The suggestion dropdown will only show text columns of your currently selected table but you may manually enter any column. +You can remove the group by clicking on the item and then selecting `Remove`. + +If you add any grouping, all selected columns need to have an aggregate function applied. The query builder will automatically add aggregate functions to all columns without aggregate functions when you add groupings. + +#### Gap Filling + +Grafana can fill in missing values when you group by time. The time function accepts two arguments. The first argument is the time window that you would like to group by, and the second argument is the value you want Grafana to fill missing items with. + +### Text Editor Mode (RAW) +You can switch to the raw query editor mode by clicking the hamburger icon and selecting `Switch editor mode` or by clicking `Edit SQL` below the query. + +> If you use the raw query editor, be sure your query at minimum has `ORDER BY time` and a filter on the returned time range. + ## Macros -To simplify syntax and to allow for dynamic parts, like date range filters, the query can contain macros. +Macros can be used within a query to simplify syntax and allow for dynamic parts. Macro example | Description ------------ | ------------- *$__time(dateColumn)* | Will be replaced by an expression to rename the column to `time`. For example, *dateColumn as time* *$__timeSec(dateColumn)* | Will be replaced by an expression to rename the column to `time` and converting the value to unix timestamp. For example, *extract(epoch from dateColumn) as time* -*$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *extract(epoch from dateColumn) BETWEEN 1494410783 AND 1494497183* -*$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *to_timestamp(1494410783)* -*$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *to_timestamp(1494497183)* -*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *(extract(epoch from dateColumn)/300)::bigint*300 AS time* -*$__unixEpochFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name with times represented as unix timestamp. For example, *dateColumn > 1494410783 AND dateColumn < 1494497183* +*$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *dateColumn BETWEEN '2017-04-21T05:01:17Z' AND '2017-04-21T05:06:17Z'* +*$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *'2017-04-21T05:01:17Z'* +*$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *'2017-04-21T05:06:17Z'* +*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in a GROUP BY clause. For example, *(extract(epoch from dateColumn)/300)::bigint*300* +*$__timeGroup(dateColumn,'5m', 0)* | Same as above but with a fill parameter so missing points in that series will be added by Grafana and 0 will be used as the value. +*$__timeGroup(dateColumn,'5m', NULL)* | Same as above but NULL will be used as value for missing points. +*$__timeGroup(dateColumn,'5m', previous)* | Same as above but the previous value in that series will be used as fill value. If no value has been seen yet, NULL will be used (only available in Grafana 5.3+). +*$__timeGroupAlias(dateColumn,'5m')* | Will be replaced with an expression identical to $__timeGroup, but with an added column alias (only available in Grafana 5.3+). +*$__unixEpochFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name with times represented as unix timestamps. For example, *dateColumn >= 1494410783 AND dateColumn <= 1494497183* *$__unixEpochFrom()* | Will be replaced by the start of the currently active time selection as unix timestamp. For example, *1494410783* *$__unixEpochTo()* | Will be replaced by the end of the currently active time selection as unix timestamp. For example, *1494497183* +*$__unixEpochGroup(dateColumn,'5m', [fillmode])* | Same as $__timeGroup, but for times stored as unix timestamp (only available in Grafana 5.3+). +*$__unixEpochGroupAlias(dateColumn,'5m', [fillmode])* | Same as above, but also adds a column alias (only available in Grafana 5.3+). We plan to add many more macros. If you have suggestions for what macros you would like to see, please [open an issue](https://github.com/grafana/grafana) in our GitHub repo. -The query editor has a link named `Generated SQL` that shows up after a query as been executed, while in panel edit mode. Click on it and it will expand and show the raw interpolated SQL string that was executed. - ## Table queries If the `Format as` query option is set to `Table` then you can basically do any type of SQL query. The table panel will automatically show the results of whatever columns & rows your query returns. @@ -82,36 +184,53 @@ You can control the name of the Table panel columns by using regular `as ` SQL c The resulting table panel: -![](/img/docs/v46/postgres_table.png) +![postgres table](/img/docs/v46/postgres_table.png) -### Time series queries +## Time series queries -If you set `Format as` to `Time series`, for use in Graph panel for example, then the query must return a column named `time` that returns either a sql datetime or any numeric datatype representing unix epoch in seconds. -Any column except `time` and `metric` is treated as a value column. +If you set `Format as` to `Time series`, for use in Graph panel for example, then the query must return a column named `time` that returns either a SQL datetime or any numeric datatype representing unix epoch. +Any column except `time` and `metric` are treated as a value column. You may return a column named `metric` that is used as metric name for the value column. +If you return multiple value columns and a column named `metric` then this column is used as prefix for the series name (only available in Grafana 5.3+). -Example with `metric` column +Resultsets of time series queries need to be sorted by time. + +**Example with `metric` column:** ```sql SELECT - $__timeGroup(time_date_time,'5m'), - min(value_double), + $__timeGroup("time_date_time",'5m'), + min("value_double"), 'min' as metric FROM test_data -WHERE $__timeFilter(time_date_time) +WHERE $__timeFilter("time_date_time") GROUP BY time ORDER BY time ``` -Example with multiple columns: +**Example using the fill parameter in the $__timeGroup macro to convert null values to be zero instead:** ```sql SELECT - $__timeGroup(time_date_time,'5m'), - min(value_double) as min_value, - max(value_double) as max_value + $__timeGroup("createdAt",'5m',0), + sum(value) as value, + measurement FROM test_data -WHERE $__timeFilter(time_date_time) +WHERE + $__timeFilter("createdAt") +GROUP BY time, measurement +ORDER BY time +``` + +**Example with multiple columns:** + +```sql +SELECT + $__timeGroup("time_date_time",'5m'), + min("value_double") as "min_value", + max("value_double") as "max_value" +FROM test_data +WHERE $__timeFilter("time_date_time") GROUP BY time ORDER BY time ``` @@ -151,7 +270,7 @@ Another option is a query that can create a key/value variable. The query should SELECT hostname AS __text, id AS __value FROM host ``` -You can also create nested variables. For example if you had another variable named `region`. Then you could have +You can also create nested variables. Using a variable named `region`, you could have the hosts variable only show hosts from the current selected region with a query like this (if `region` is a multi-value variable then use the `IN` comparison operator rather than `=` to match against multiple values): ```sql @@ -160,7 +279,7 @@ SELECT hostname FROM host WHERE region IN($region) ### Using Variables in Queries -From Grafana 4.3.0 to 4.6.0, template variables are always quoted automatically so if it is a string value do not wrap them in quotes in where clauses. +From Grafana 4.3.0 to 4.6.0, template variables are always quoted automatically. If your template variables are strings, do not wrap them in quotes in where clauses. From Grafana 4.7.0, template variable values are only quoted when the template variable is a `multi-value`. @@ -190,30 +309,77 @@ WHERE $__timeFilter(atimestamp) and hostname in([[hostname]]) ORDER BY atimestamp ASC ``` +#### Disabling Quoting for Multi-value Variables + +Grafana automatically creates a quoted, comma-separated string for multi-value variables. For example: if `server01` and `server02` are selected then it will be formatted as: `'server01', 'server02'`. To disable quoting, use the csv formatting option for variables: + +`${servers:csv}` + +Read more about variable formatting options in the [Variables]({{< relref "reference/templating.md#advanced-formatting-options" >}}) documentation. + ## Annotations [Annotations]({{< relref "reference/annotations.md" >}}) allow you to overlay rich event information on top of graphs. You add annotation queries via the Dashboard menu / Annotations view. -An example query: +**Example query using time column with epoch values:** ```sql SELECT - extract(epoch from time_date_time) AS time, - metric1 as text, + epoch_time as time, + metric1 as text, concat_ws(', ', metric1::text, metric2::text) as tags FROM public.test_data WHERE - $__timeFilter(time_date_time) + $__unixEpochFilter(epoch_time) +``` + +**Example query using time column of native sql date/time data type:** + +```sql +SELECT + native_date_time as time, + metric1 as text, + concat_ws(', ', metric1::text, metric2::text) as tags +FROM + public.test_data +WHERE + $__timeFilter(native_date_time) ``` Name | Description ------------ | ------------- -time | The name of the date/time field. +time | The name of the date/time field. Could be a column with a native sql date/time data type or epoch value. text | Event description field. tags | Optional field name to use for event tags as a comma separated string. ## Alerting -Time series queries should work in alerting conditions. Table formatted queries is not yet supported in alert rule +Time series queries should work in alerting conditions. Table formatted queries are not yet supported in alert rule conditions. + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: Postgres + type: postgres + url: localhost:5432 + database: grafana + user: grafana + secureJsonData: + password: "Password!" + jsonData: + sslmode: "disable" # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false +``` diff --git a/docs/sources/features/datasources/prometheus.md b/docs/sources/features/datasources/prometheus.md index c9bb16441ca..611a3b4d9e2 100644 --- a/docs/sources/features/datasources/prometheus.md +++ b/docs/sources/features/datasources/prometheus.md @@ -30,11 +30,11 @@ Name | Description *Name* | The data source name. This is how you refer to the data source in panels & queries. *Default* | Default data source means that it will be pre-selected for new panels. *Url* | The http protocol, ip and port of you Prometheus server (default port is usually 9090) -*Access* | Proxy = access via Grafana backend, Direct = access directly from browser. +*Access* | Server (default) = URL needs to be accessible from the Grafana backend/server, Browser = URL needs to be accessible from the browser. *Basic Auth* | Enable basic authentication to the Prometheus data source. *User* | Name of your Prometheus user *Password* | Database user's password -*Scrape interval* | This will be used as a lower limit for the Prometheus step query parameter. Default value is 15s. +*Scrape interval* | This will be used as a lower limit for the Prometheus step query parameter. Default value is 15s. ## Query editor @@ -50,7 +50,7 @@ Name | Description *Min step* | Set a lower limit for the Prometheus step option. Step controls how big the jumps are when the Prometheus query engine performs range queries. Sadly there is no official prometheus documentation to link to for this very important option. *Resolution* | Controls the step option. Small steps create high-resolution graphs but can be slow over larger time ranges, lowering the resolution can speed things up. `1/2` will try to set step option to generate 1 data point for every other pixel. A value of `1/10` will try to set step option so there is a data point every 10 pixels. *Metric lookup* | Search for metric names in this input field. -*Format as* | **(New in v4.3)** Switch between Table & Time series. Table format will only work in the Table panel. +*Format as* | Switch between Table, Time series or Heatmap. Table format will only work in the Table panel. Heatmap format is suitable for displaying metrics having histogram type on Heatmap panel. Under the hood, it converts cumulative histogram to regular and sorts series by the bucket bound. ## Templating @@ -75,6 +75,32 @@ Name | Description For details of *metric names*, *label names* and *label values* are please refer to the [Prometheus documentation](http://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). + +#### Using interval and range variables + +> Support for `$__range`, `$__range_s` and `$__range_ms` only available from Grafana v5.3 + +It's possible to use some global built-in variables in query variables; `$__interval`, `$__interval_ms`, `$__range`, `$__range_s` and `$__range_ms`, see [Global built-in variables](/reference/templating/#global-built-in-variables) for more information. These can be convenient to use in conjunction with the `query_result` function when you need to filter variable queries since +`label_values` function doesn't support queries. + +Make sure to set the variable's `refresh` trigger to be `On Time Range Change` to get the correct instances when changing the time range on the dashboard. + +**Example usage:** + +Populate a variable with the the busiest 5 request instances based on average QPS over the time range shown in the dashboard: + +``` +Query: query_result(topk(5, sum(rate(http_requests_total[$__range])) by (instance))) +Regex: /"([^"]+)"/ +``` + +Populate a variable with the instances having a certain state over the time range shown in the dashboard, using the more precise `$__range_s`: + +``` +Query: query_result(max_over_time([${__range_s}s]) != ) +Regex: +``` + ### Using variables in queries There are two syntaxes: @@ -100,3 +126,19 @@ The step option is useful to limit the number of events returned from your query ## Getting Grafana metrics into Prometheus Since 4.6.0 Grafana exposes metrics for Prometheus on the `/metrics` endpoint. We also bundle a dashboard within Grafana so you can get started viewing your metrics faster. You can import the bundled dashboard by going to the data source edit page and click the dashboard tab. There you can find a dashboard for Grafana and one for Prometheus. Import and start viewing all the metrics! + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here are some provisioning examples for this datasource. + +```yaml +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://localhost:9090 +``` diff --git a/docs/sources/features/datasources/stackdriver.md b/docs/sources/features/datasources/stackdriver.md new file mode 100644 index 00000000000..d19dbe4ea50 --- /dev/null +++ b/docs/sources/features/datasources/stackdriver.md @@ -0,0 +1,253 @@ ++++ +title = "Using Stackdriver in Grafana" +description = "Guide for using Stackdriver in Grafana" +keywords = ["grafana", "stackdriver", "google", "guide"] +type = "docs" +aliases = ["/datasources/stackdriver"] +[menu.docs] +name = "Stackdriver" +parent = "datasources" +weight = 11 ++++ + +# Using Google Stackdriver in Grafana + +> Only available in Grafana v5.3+. +> The datasource is currently a beta feature and is subject to change. + +Grafana ships with built-in support for Google Stackdriver. Just add it as a datasource and you are ready to build dashboards for your Stackdriver metrics. + +## Adding the data source to Grafana + +1. Open the side menu by clicking the Grafana icon in the top header. +2. In the side menu under the `Dashboards` link you should find a link named `Data Sources`. +3. Click the `+ Add data source` button in the top header. +4. Select `Stackdriver` from the _Type_ dropdown. +5. Upload or paste in the Service Account Key file. See below for steps on how to create a Service Account Key file. + +> NOTE: If you're not seeing the `Data Sources` link in your side menu it means that your current user does not have the `Admin` role for the current organization. + +| Name | Description | +| --------------------- | ----------------------------------------------------------------------------------- | +| _Name_ | The datasource name. This is how you refer to the datasource in panels & queries. | +| _Default_ | Default datasource means that it will be pre-selected for new panels. | +| _Service Account Key_ | Service Account Key File for a GCP Project. Instructions below on how to create it. | + +## Authentication + +There are two ways to authenticate the Stackdriver plugin - either by uploading a Google JWT file, or by automatically retrieving credentials from Google metadata server. The latter option is only available when running Grafana on GCE virtual machine. + +### Using a Google Service Account Key File + +To authenticate with the Stackdriver API, you need to create a Google Cloud Platform (GCP) Service Account for the Project you want to show data for. A Grafana datasource integrates with one GCP Project. If you want to visualize data from multiple GCP Projects then you need to create one datasource per GCP Project. + +#### Enable APIs + +The following APIs need to be enabled first: + +* [Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com) +* [Cloud Resource Manager API](https://console.cloud.google.com/apis/library/cloudresourcemanager.googleapis.com) + +Click on the links above and click the `Enable` button: + +{{< docs-imagebox img="/img/docs/v53/stackdriver_enable_api.png" class="docs-image--no-shadow" caption="Enable GCP APIs" >}} + +#### Create a GCP Service Account for a Project + +1. Navigate to the [APIs & Services Credentials page](https://console.cloud.google.com/apis/credentials). +2. Click on the `Create credentials` dropdown/button and choose the `Service account key` option. + + {{< docs-imagebox img="/img/docs/v53/stackdriver_create_service_account_button.png" class="docs-image--no-shadow" caption="Create service account button" >}} + +3. On the `Create service account key` page, choose key type `JSON`. Then in the `Service Account` dropdown, choose the `New service account` option: + + {{< docs-imagebox img="/img/docs/v53/stackdriver_create_service_account_key.png" class="docs-image--no-shadow" caption="Create service account key" >}} + +4. Some new fields will appear. Fill in a name for the service account in the `Service account name` field and then choose the `Monitoring Viewer` role from the `Role` dropdown: + + {{< docs-imagebox img="/img/docs/v53/stackdriver_service_account_choose_role.png" class="docs-image--no-shadow" caption="Choose role" >}} + +5. Click the Create button. A JSON key file will be created and downloaded to your computer. Store this file in a secure place as it allows access to your Stackdriver data. +6. Upload it to Grafana on the datasource Configuration page. You can either upload the file or paste in the contents of the file. + + {{< docs-imagebox img="/img/docs/v53/stackdriver_grafana_upload_key.png" class="docs-image--no-shadow" caption="Upload service key file to Grafana" >}} + +7. The file contents will be encrypted and saved in the Grafana database. Don't forget to save after uploading the file! + + {{< docs-imagebox img="/img/docs/v53/stackdriver_grafana_key_uploaded.png" class="docs-image--no-shadow" caption="Service key file is uploaded to Grafana" >}} + +### Using GCE Default Service Account + +If Grafana is running on a Google Compute Engine (GCE) virtual machine, it is possible for Grafana to automatically retrieve default credentials from the metadata server. This has the advantage of not needing to generate a private key file for the service account and also not having to upload the file to Grafana. However for this to work, there are a few preconditions that need to be met. + +1. First of all, you need to create a Service Account that can be used by the GCE virtual machine. See detailed instructions on how to do that [here](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#createanewserviceaccount). +2. Make sure the GCE virtual machine instance is being run as the service account that you just created. See instructions [here](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#using). +3. Allow access to the `Stackdriver Monitoring API` scope. See instructions [here](changeserviceaccountandscopes). + +Read more about creating and enabling service accounts for GCE VM instances [here](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances). + +## Metric Query Editor + +{{< docs-imagebox img="/img/docs/v53/stackdriver_query_editor.png" max-width= "400px" class="docs-image--right" >}} + +The Stackdriver query editor allows you to select metrics, group/aggregate by labels and by time, and use filters to specify which time series you want in the results. + +Begin by choosing a `Service` and then a metric from the `Metric` dropdown. Use the plus and minus icons in the filter and group by sections to add/remove filters or group by clauses. + +Stackdriver metrics can be of different kinds (GAUGE, DELTA, CUMULATIVE) and these kinds have support for different aggregation options (reducers and aligners). The Grafana query editor shows the list of available aggregation methods for a selected metric and sets a default reducer and aligner when you select the metric. Units for the Y-axis are also automatically selected by the query editor. + +### Filter + +To add a filter, click the plus icon and choose a field to filter by and enter a filter value e.g. `instance_name = grafana-1`. You can remove the filter by clicking on the filter name and select `--remove filter--`. + +#### Simple wildcards + +When the operator is set to `=` or `!=` it is possible to add wildcards to the filter value field. E.g `us-*` will capture all values that starts with "us-" and `*central-a` will capture all values that ends with "central-a". `*-central-*` captures all values that has the substring of -central-. Simple wildcards are less expensive than regular expressions. + +#### Regular expressions + +When the operator is set to `=~` or `!=~` it is possible to add regular expressions to the filter value field. E.g `us-central[1-3]-[af]` would match all values that starts with "us-central", is followed by a number in the range of 1 to 3, a dash and then either an "a" or an "f". Leading and trailing slashes are not needed when creating regular expressions. + +### Aggregation + +The aggregation field lets you combine time series based on common statistics. Read more about this option [here](https://cloud.google.com/monitoring/charts/metrics-selector#aggregation-options). + +The `Aligner` field allows you to align multiple time series after the same group by time interval. Read more about how it works [here](https://cloud.google.com/monitoring/charts/metrics-selector#alignment). + +#### Alignment Period/Group by Time + +The `Alignment Period` groups a metric by time if an aggregation is chosen. The default is to use the GCP Stackdriver default groupings (which allows you to compare graphs in Grafana with graphs in the Stackdriver UI). +The option is called `Stackdriver auto` and the defaults are: + +* 1m for time ranges < 23 hours +* 5m for time ranges >= 23 hours and < 6 days +* 1h for time ranges >= 6 days + +The other automatic option is `Grafana auto`. This will automatically set the group by time depending on the time range chosen and the width of the graph panel. Read more about the details [here](http://docs.grafana.org/reference/templating/#the-interval-variable). + +It is also possible to choose fixed time intervals to group by, like `1h` or `1d`. + +### Group By + +Group by resource or metric labels to reduce the number of time series and to aggregate the results by a group by. E.g. Group by instance_name to see an aggregated metric for a Compute instance. + +### Alias Patterns + +The Alias By field allows you to control the format of the legend keys. The default is to show the metric name and labels. This can be long and hard to read. Using the following patterns in the alias field, you can format the legend key the way you want it. + +#### Metric Type Patterns + +| Alias Pattern | Description | Example Result | +| -------------------- | ---------------------------- | ------------------------------------------------- | +| `{{metric.type}}` | returns the full Metric Type | `compute.googleapis.com/instance/cpu/utilization` | +| `{{metric.name}}` | returns the metric name part | `instance/cpu/utilization` | +| `{{metric.service}}` | returns the service part | `compute` | + +#### Label Patterns + +In the Group By dropdown, you can see a list of metric and resource labels for a metric. These can be included in the legend key using alias patterns. + +| Alias Pattern Format | Description | Alias Pattern Example | Example Result | +| ------------------------ | -------------------------------- | -------------------------------- | ---------------- | +| `{{metric.label.xxx}}` | returns the metric label value | `{{metric.label.instance_name}}` | `grafana-1-prod` | +| `{{resource.label.xxx}}` | returns the resource label value | `{{resource.label.zone}}` | `us-east1-b` | + +Example Alias By: `{{metric.type}} - {{metric.labels.instance_name}}` + +Example Result: `compute.googleapis.com/instance/cpu/usage_time - server1-prod` + +It is also possible to resolve the name of the Monitored Resource Type. + +| Alias Pattern Format | Description | Example Result | +| ------------------------ | ------------------------------------------------| ---------------- | +| `{{resource.type}}` | returns the name of the monitored resource type | `gce_instance` | + +Example Alias By: `{{resource.type}} - {{metric.type}}` + +Example Result: `gce_instance - compute.googleapis.com/instance/cpu/usage_time` + +## Templating + +Instead of hard-coding things like server, application and sensor name in you metric queries you can use variables in their place. +Variables are shown as dropdown select boxes at the top of the dashboard. These dropdowns makes it easy to change the data +being displayed in your dashboard. + +Checkout the [Templating]({{< relref "reference/templating.md" >}}) documentation for an introduction to the templating feature and the different +types of template variables. + +### Query Variable + +Writing variable queries is not supported yet. + +### Using variables in queries + +There are two syntaxes: + +* `$` Example: `metric.label.$metric_label` +* `[[varname]]` Example: `metric.label.[[metric_label]]` + +Why two ways? The first syntax is easier to read and write but does not allow you to use a variable in the middle of a word. When the _Multi-value_ or _Include all value_ options are enabled, Grafana converts the labels from plain text to a regex compatible string, which means you have to use `=~` instead of `=`. + +## Annotations + +{{< docs-imagebox img="/img/docs/v53/stackdriver_annotations_query_editor.png" max-width= "400px" class="docs-image--right" >}} + +[Annotations]({{< relref "reference/annotations.md" >}}) allows you to overlay rich event information on top of graphs. You add annotation +queries via the Dashboard menu / Annotations view. Annotation rendering is expensive so it is important to limit the number of rows returned. There is no support for showing Stackdriver annotations and events yet but it works well with [custom metrics](https://cloud.google.com/monitoring/custom-metrics/) in Stackdriver. + +With the query editor for annotations, you can select a metric and filters. The `Title` and `Text` fields support templating and can use data returned from the query. For example, the Title field could have the following text: + +`{{metric.type}} has value: {{metric.value}}` + +Example Result: `monitoring.googleapis.com/uptime_check/http_status has this value: 502` + +### Patterns for the Annotation Query Editor + +| Alias Pattern Format | Description | Alias Pattern Example | Example Result | +| ------------------------ | -------------------------------- | -------------------------------- | ------------------------------------------------- | +| `{{metric.value}}` | value of the metric/point | `{{metric.value}}` | `555` | +| `{{metric.type}}` | returns the full Metric Type | `{{metric.type}}` | `compute.googleapis.com/instance/cpu/utilization` | +| `{{metric.name}}` | returns the metric name part | `{{metric.name}}` | `instance/cpu/utilization` | +| `{{metric.service}}` | returns the service part | `{{metric.service}}` | `compute` | +| `{{metric.label.xxx}}` | returns the metric label value | `{{metric.label.instance_name}}` | `grafana-1-prod` | +| `{{resource.label.xxx}}` | returns the resource label value | `{{resource.label.zone}}` | `us-east1-b` | + +## Configure the Datasource with Provisioning + +It's now possible to configure datasources using config files with Grafana's provisioning system. You can read more about how it works and all the settings you can set for datasources on the [provisioning docs page](/administration/provisioning/#datasources) + +Here is a provisioning example using the JWT (Service Account key file) authentication type. + +```yaml +apiVersion: 1 + +datasources: + - name: Stackdriver + type: stackdriver + access: proxy + jsonData: + tokenUri: https://oauth2.googleapis.com/token + clientEmail: stackdriver@myproject.iam.gserviceaccount.com + authenticationType: jwt + defaultProject: my-project-name + secureJsonData: + privateKey: | + -----BEGIN PRIVATE KEY----- + POSEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCb1u1Srw8ICYHS + ... + yA+23427282348234= + -----END PRIVATE KEY----- +``` + +Here is a provisioning example using GCE Default Service Account authentication. + +```yaml +apiVersion: 1 + +datasources: + - name: Stackdriver + type: stackdriver + access: proxy + jsonData: + authenticationType: gce +``` diff --git a/docs/sources/features/panels/alertlist.md b/docs/sources/features/panels/alertlist.md index 9307bb71391..a1ea8f0f600 100644 --- a/docs/sources/features/panels/alertlist.md +++ b/docs/sources/features/panels/alertlist.md @@ -14,7 +14,7 @@ weight = 4 {{< docs-imagebox img="/img/docs/v45/alert-list-panel.png" max-width="850px" >}} -The alert list panel allows you to display your dashbords alerts. The list can be configured to show current state or recent state changes. You can read more about alerts [here](http://docs.grafana.org/alerting/rules). +The alert list panel allows you to display your dashboards alerts. The list can be configured to show current state or recent state changes. You can read more about alerts [here](http://docs.grafana.org/alerting/rules). ## Alert List Options @@ -22,6 +22,6 @@ The alert list panel allows you to display your dashbords alerts. The list can b 1. **Show**: Lets you choose between current state or recent state changes. 2. **Max Items**: Max items set the maximum of items in a list. -3. **Sort Order**: Lets you sort your list alphabeticaly(asc/desc) or by importance. +3. **Sort Order**: Lets you sort your list alphabetically(asc/desc) or by importance. 4. **Alerts From** This Dashboard`: Shows alerts only from the dashboard the alert list is in. 5. **State Filter**: Here you can filter your list by one or more parameters. diff --git a/docs/sources/features/panels/dashlist.md b/docs/sources/features/panels/dashlist.md index 8a4ed60875d..2ee578c5b7e 100644 --- a/docs/sources/features/panels/dashlist.md +++ b/docs/sources/features/panels/dashlist.md @@ -25,7 +25,7 @@ The dashboard list panel allows you to display dynamic links to other dashboards 1. **Starred**: The starred dashboard selection displays starred dashboards in alphabetical order. 2. **Recently Viewed**: The recently viewed dashboard selection displays recently viewed dashboards in alphabetical order. 3. **Search**: The search dashboard selection displays dashboards by search query or tag(s). -4. **Show Headings**: When show headings is ticked the choosen list selection(Starred, Recently Viewed, Search) is shown as a heading. +4. **Show Headings**: When show headings is ticked the chosen list selection(Starred, Recently Viewed, Search) is shown as a heading. 5. **Max Items**: Max items set the maximum of items in a list. 6. **Query**: Here is where you enter your query you want to search by. Queries are case-insensitive, and partial values are accepted. 7. **Tags**: Here is where you enter your tag(s) you want to search by. Note that existing tags will not appear as you type, and *are* case sensitive. To see a list of existing tags, you can always return to the dashboard, open the Dashboard Picker at the top and click `tags` link in the search bar. diff --git a/docs/sources/features/panels/graph.md b/docs/sources/features/panels/graph.md index c3b0260c98b..44fa0e7c0db 100644 --- a/docs/sources/features/panels/graph.md +++ b/docs/sources/features/panels/graph.md @@ -22,15 +22,18 @@ options for the panel. ## General -{{< docs-imagebox img="/img/docs/v43/graph_general.png" max-width= "900px" >}} +{{< docs-imagebox img="/img/docs/v51/graph_general.png" max-width= "800px" >}} The general tab allows customization of a panel's appearance and menu options. -### General Options +### Info -- **Title** - The panel title on the dashboard -- **Span** - The panel width in columns -- **Height** - The panel contents height in pixels +- **Title** - The panel title of the dashboard, displayed at the top. +- **Description** - The panel description, displayed on hover of info icon in the upper left corner of the panel. +- **Transparent** - If checked, removes the solid background of the panel (default not checked). + +### Repeat +Repeat a panel for each value of a variable. Repeating panels are described in more detail [here]({{< relref "reference/templating.md#repeating-panels" >}}). ### Drilldown / detail link @@ -54,47 +57,65 @@ options. ## Axes -{{< docs-imagebox img="/img/docs/v43/graph_axes_grid_options.png" max-width= "900px" >}} +{{< docs-imagebox img="/img/docs/v51/graph_axes_grid_options.png" max-width= "800px" >}} -The Axes tab controls the display of axes, grids and legend. The **Left Y** and **Right Y** can be customized using: +The Axes tab controls the display of axes. + +### Left Y/Right Y + +The **Left Y** and **Right Y** can be customized using: - **Unit** - The display unit for the Y value -- **Scale** - +- **Scale** - The scale to use for the Y value, linear or logarithmic. (default linear) - **Y-Min** - The minimum Y value. (default auto) - **Y-Max** - The maximum Y value. (default auto) +- **Decimals** - Controls how many decimals are displayed for Y value (default auto) - **Label** - The Y axis label (default "") Axes can also be hidden by unchecking the appropriate box from **Show**. -### X-Axis Mode +### X-Axis -There are three options: +Axis can be hidden by unchecking **Show**. + +For **Mode** there are three options: - The default option is **Time** and means the x-axis represents time and that the data is grouped by time (for example, by hour or by minute). - The **Series** option means that the data is grouped by series and not by time. The y-axis still represents the value. - {{< docs-imagebox img="/img/docs/v45/graph-x-axis-mode-series.png" max-width="700px">}} + {{< docs-imagebox img="/img/docs/v51/graph-x-axis-mode-series.png" max-width="800px">}} - The **Histogram** option converts the graph into a histogram. A Histogram is a kind of bar chart that groups numbers into ranges, often called buckets or bins. Taller bars show that more data falls in that range. Histograms and buckets are described in more detail [here](http://docs.grafana.org/features/panels/heatmap/#histograms-and-buckets). -### Legend -The legend hand be hidden by checking the **Show** checkbox. If it's shown, it can be -displayed as a table of values by checking the **Table** checkbox. Series with no -values can be hidden from the legend using the **Hide empty** checkbox. +### Y-Axes -### Legend Values +- **Align** - Check to align left and right Y-axes by value (default unchecked/false) +- **Level** - Available when *Align* is checked. Value to use for alignment of left and right Y-axes, starting from Y=0 (default 0) + +## Legend + +{{< docs-imagebox img="/img/docs/v51/graph-legend.png" max-width= "800px" >}} + +### Options + +- **Show** - Uncheck to hide the legend (default checked/true) +- **Table** - Check to display legend in table (default unchecked/false) +- **To the right** - Check to display legend to the right (default unchecked/false) +- **Width** - Available when *To the right* is checked. Value to control the minimum width for the legend (default 0) + +### Values Additional values can be shown along-side the legend names: -- **Total** - Sum of all values returned from metric query -- **Current** - Last value returned from the metric query - **Min** - Minimum of all values returned from metric query - **Max** - Maximum of all values returned from the metric query - **Avg** - Average of all values returned from metric query +- **Current** - Last value returned from the metric query +- **Total** - Sum of all values returned from metric query - **Decimals** - Controls how many decimals are displayed for legend values (and graph hover tooltips) The legend values are calculated client side by Grafana and depend on what type of @@ -103,63 +124,80 @@ be correct at the same time. For example if you plot a rate like requests/second using average as aggregator, then the Total in the legend will not represent the total number of requests. It is just the sum of all data points received by Grafana. +### Hide series + +Hide series when all values of a series from a metric query are of a specific value: + +- **With only nulls** - Value=*null* (default unchecked) +- **With only zeros** - Value=*zero* (default unchecked) + ## Display styles -{{< docs-imagebox img="/img/docs/v43/graph_display_styles.png" max-width= "900px" >}} +{{< docs-imagebox img="/img/docs/v51/graph_display_styles.png" max-width= "800px" >}} Display styles control visual properties of the graph. -### Thresholds +### Draw Options -Thresholds allow you to add arbitrary lines or sections to the graph to make it easier to see when -the graph crosses a particular threshold. - - -### Chart Options +#### Draw Modes - **Bar** - Display values as a bar chart - **Lines** - Display values as a line graph - **Points** - Display points for values -### Line Options +#### Mode Options -- **Line Fill** - Amount of color fill for a series. 0 is none. -- **Line Width** - The width of the line for a series. -- **Null point mode** - How null values are displayed -- **Staircase line** - Draws adjacent points as staircase +- **Fill** - Amount of color fill for a series (default 1). 0 is none. +- **Line Width** - The width of the line for a series (default 1). +- **Staircase** - Draws adjacent points as staircase +- **Points Radius** - Adjust the size of points when *Points* are selected as *Draw Mode*. -### Multiple Series +#### Hover tooltip + +- **Mode** - Controls how many series to display in the tooltip when hover over a point in time, All series or single (default All series). +- **Sort order** - Controls how series displayed in tooltip are sorted, None, Ascending or Descending (default None). +- **Stacked value** - Available when *Stack* are checked and controls how stacked values are displayed in tooltip (default Individual). + - Individual: the value for the series you hover over + - Cumulative - sum of series below plus the series you hover over + +#### Stacking & Null value If there are multiple series, they can be displayed as a group. - **Stack** - Each series is stacked on top of another -- **Percent** - Each series is drawn as a percentage of the total of all series +- **Percent** - Available when *Stack* are checked. Each series is drawn as a percentage of the total of all series +- **Null value** - How null values are displayed -If you have stack enabled, you can select what the mouse hover feature should show. +### Series overrides -- Cumulative - Sum of series below plus the series you hover over -- Individual - Just the value for the series you hover over - -### Rendering - -- **Flot** - Render the graphs in the browser using Flot (default) -- **Graphite PNG** - Render the graph on the server using graphite's render API. - -### Tooltip - -- **All series** - Show all series on the same tooltip and a x crosshairs to help follow all series - -### Series Specific Overrides +{{< docs-imagebox img="/img/docs/v51/graph_display_overrides.png" max-width= "800px" >}} The section allows a series to be rendered differently from the others. For example, one series can be given -a thicker line width to make it stand out. +a thicker line width to make it stand out and/or be moved to the right Y-axis. #### Dashes Drawing Style There is an option under Series overrides to draw lines as dashes. Set Dashes to the value True to override the line draw setting for a specific series. +### Thresholds + +{{< docs-imagebox img="/img/docs/v51/graph_display_thresholds.png" max-width= "800px" >}} + +Thresholds allow you to add arbitrary lines or sections to the graph to make it easier to see when +the graph crosses a particular threshold. + +### Time Regions + +> Only available in Grafana v5.4 and above. + +{{< docs-imagebox img="/img/docs/v54/graph_time_regions.png" max-width= "800px" >}} + +Time regions allow you to highlight certain time regions of the graph to make it easier to see for example weekends, business hours and/or off work hours. + ## Time Range -The time range tab allows you to override the dashboard time range and specify a panel specific time. Either through a relative from now time option or through a timeshift. +{{< docs-imagebox img="/img/docs/v51/graph-time-range.png" max-width= "900px" >}} -{{< docs-imagebox img="/img/docs/v45/graph-time-range.png" max-width= "900px" >}} +The time range tab allows you to override the dashboard time range and specify a panel specific time. +Either through a relative from now time option or through a timeshift. +Panel time overrides & timeshift are described in more detail [here]({{< relref "reference/timerange.md#panel-time-overrides-timeshift" >}}). diff --git a/docs/sources/features/panels/heatmap.md b/docs/sources/features/panels/heatmap.md index e44527f8695..aa87fbef1df 100644 --- a/docs/sources/features/panels/heatmap.md +++ b/docs/sources/features/panels/heatmap.md @@ -56,26 +56,39 @@ Data and bucket options can be found in the `Axes` tab. Data format | Description ------------ | ------------- *Time series* | Grafana does the bucketing by going through all time series values. The bucket sizes & intervals will be determined using the Buckets options. -*Time series buckets* | Each time series already represents a Y-Axis bucket. The time series name (alias) needs to be a numeric value representing the upper interval for the bucket. Grafana does no bucketing so the bucket size options are hidden. +*Time series buckets* | Each time series already represents a Y-Axis bucket. The time series name (alias) needs to be a numeric value representing the upper or lower interval for the bucket. Grafana does no bucketing so the bucket size options are hidden. + +### Bucket bound + +When Data format is *Time series buckets* datasource returns series with names representing bucket bound. But depending +on datasource, a bound may be *upper* or *lower*. This option allows to adjust a bound type. If *Auto* is set, a bound +option will be chosen based on panels' datasource type. ### Bucket Size The Bucket count & size options are used by Grafana to calculate how big each cell in the heatmap is. You can define the bucket size either by count (the first input box) or by specifying a size interval. For the Y-Axis the size interval is just a value but for the X-bucket you can specify a time range in the *Size* input, for example, -the time range `1h`. This will make the cells 1h wide on the X-axis. +the time range `1h`. This will make the cells 1h wide on the X-axis. ### Pre-bucketed data -If you have a data that is already organized into buckets you can use the `Time series buckets` data format. This format requires that your metric query return regular time series and that each time series has a numeric name -that represent the upper or lower bound of the interval. +If you have a data that is already organized into buckets you can use the `Time series buckets` data format. This format +requires that your metric query return regular time series and that each time series has a numeric name that represent +the upper or lower bound of the interval. -The only data source that supports histograms over time is Elasticsearch. You do this by adding a *Histogram* -bucket aggregation before the *Date Histogram*. +There are a number of datasources supporting histogram over time like Elasticsearch (by using a Histogram bucket +aggregation) or Prometheus (with [histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) metric type +and *Format as* option set to Heatmap). But generally, any datasource could be used if it meets the requirements: +returns series with names representing bucket bound or returns series sorted by the bound in ascending order. -![](/img/docs/v43/elastic_histogram.png) +With Elasticsearch you control the size of the buckets using the Histogram interval (Y-Axis) and the Date Histogram interval (X-axis). -You control the size of the buckets using the Histogram interval (Y-Axis) and the Date Histogram interval (X-axis). +![Elastic histogram](/img/docs/v43/elastic_histogram.png) + +With Prometheus you can only control X-axis by adjusting *Min step* and *Resolution* options. + +![Prometheus histogram](/img/docs/v51/prometheus_histogram.png) ## Display Options @@ -100,8 +113,8 @@ but include a group by time interval or maxDataPoints limit coupled with an aggr This all depends on the time range of your query of course. But the important point is to know that the Histogram bucketing that Grafana performs may be done on already aggregated and averaged data. To get more accurate heatmaps it is better -to do the bucketing during metric collection or store the data in Elasticsearch, which currently is the only data source -data supports doing Histogram bucketing on the raw data. +to do the bucketing during metric collection or store the data in Elasticsearch, or in the other data source which +supports doing Histogram bucketing on the raw data. If you remove or lower the group by time (or raise maxDataPoints) in your query to return more data points your heatmap will be more accurate but this can also be very CPU & Memory taxing for your browser and could cause hangs and crashes if the number of diff --git a/docs/sources/features/panels/singlestat.md b/docs/sources/features/panels/singlestat.md index 510642337ff..e16f182f9cd 100644 --- a/docs/sources/features/panels/singlestat.md +++ b/docs/sources/features/panels/singlestat.md @@ -30,7 +30,7 @@ The singlestat panel has a normal query editor to allow you define your exact me * **total** - The sum of all the non-null values in the series * **first** - The first value in the series * **delta** - The total incremental increase (of a counter) in the series. An attempt is made to account for counter resets, but this will only be accurate for single instance metrics. Used to show total counter increase in time series. - * **diff** - The difference betwen 'current' (last value) and 'first'. + * **diff** - The difference between 'current' (last value) and 'first'. * **range** - The difference between 'min' and 'max'. Useful the show the range of change for a gauge. 2. **Prefix/Postfix**: The Prefix/Postfix fields let you define a custom label to appear *before/after* the value. The `$__name` variable can be used here to use the series name or alias from the metric query. 3. **Units**: Units are appended to the the Singlestat within the panel, and will respect the color and threshold settings for the value. @@ -70,18 +70,18 @@ Gauges gives a clear picture of how high a value is in it's context. It's a grea {{< docs-imagebox img="/img/docs/v45/singlestat-gauge-options.png" max-width="500px" class="docs-image--right docs-image--no-shadow">}} -1. **Show**: The show checkbox will toggle wether the gauge is shown in the panel. When unselected, only the Singlestat value will appear. +1. **Show**: The show checkbox will toggle whether the gauge is shown in the panel. When unselected, only the Singlestat value will appear. 2. **Min/Max**: This sets the start and end point for the gauge. 3. **Threshold Labels**: Check if you want to show the threshold labels. Thresholds are set in the color options. 4. **Threshold Markers**: Check if you want to have a second meter showing the thresholds.
-### Value to text mapping +### Value/Range to text mapping {{< docs-imagebox img="/img/docs/v45/singlestat-value-mapping.png" class="docs-image--right docs-image--no-shadow">}} -Value to text mapping allows you to translate the value of the summary stat into explicit text. The text will respect all styling, thresholds and customization defined for the value. This can be useful to translate the number of the main Singlestat value into a context-specific human-readable word or message. +Value/Range to text mapping allows you to translate the value of the summary stat into explicit text. The text will respect all styling, thresholds and customization defined for the value. This can be useful to translate the number of the main Singlestat value into a context-specific human-readable word or message.
diff --git a/docs/sources/features/panels/table_panel.md b/docs/sources/features/panels/table_panel.md index a3e56c72960..2cbb601820e 100644 --- a/docs/sources/features/panels/table_panel.md +++ b/docs/sources/features/panels/table_panel.md @@ -14,11 +14,53 @@ weight = 2 -The new table panel is very flexible, supporting both multiple modes for time series as well as for +The table panel is very flexible, supporting both multiple modes for time series as well as for table, annotation and raw JSON data. It also provides date formatting and value formatting and coloring options. To view table panels in action and test different configurations with sample data, check out the [Table Panel Showcase in the Grafana Playground](http://play.grafana.org/dashboard/db/table-panel-showcase). +## Querying Data + +The table panel displays the results of a query specified in the **Metrics** tab. +The result being displayed depends on the datasource and the query, but generally there is one row per datapoint, with extra columns for associated keys and values, as well as one column for the numeric value of the datapoint. +You can change the behavior in the section **Data to Table** below. + +### Merge Multiple Queries per Table + +> Only available in Grafana v5.0+. + +Sometimes it is useful to display the results of multiple queries in the same table on corresponding rows, e.g., when comparing capacity and actual usage of resources. +In this example usage and capacity are metrics that will have corresponding datapoints, while their associated keys and values can be used to match them. +(This matching is only available with the **Table Transform** set to **Table**.) + +In its simplest case, both queries return time-series data with a numeric value and a timestamp. +If the timestamps are the same, datapoints will be matched and rendered on the same row. +Some datasources return keys and values (labels, tags) associated with the datapoint. +These are being matched as well if they are present in both results and have the same value. +The following datapoints will end up on the same row with one time column, two label columns ("host" and "job") and two value columns: + +``` +Datapoint for query A: {time: 1, host: "node-2", job: "job-8", value: 3} +Datapoint for query B: {time: 1, host: "node-2", value: 4} +``` + +The following two results cannot be matched and will be rendered on separate rows: + +``` +Different time +Datapoint for query A: {time: 1, host: "node-2", job: "job-8", value: 3} +Datapoint for query B: {time: 2, host: "node-2", value: 4} + +Different label "host" +Datapoint for query A: {time: 1, host: "node-2", job: "job-8", value: 3} +Datapoint for query B: {time: 1, host: "node-9", value: 4} +``` + +You can still merge both of the above cases by changing the conflicting column's **Type** to **hidden** in the **Column Styles**. + +Note that if each datapoint of your query results have multiple value fields like max, min, mean, etc., they will likely have different values and therefore will not match and render on separate rows. +If you intend for rows to be merged but see them rendered on separate rows, check the query results in the **Query Inspector** for field values being identical across datapoints that should be merged into a row. + ## Options overview The table panel has many ways to manipulate your data for optimal presentation. @@ -97,3 +139,14 @@ The column styles allow you control how dates and numbers are formatted. 4. **Thresholds and Coloring**: Specify color mode and thresholds limits. 5. **Type**: The three supported types of types are **Number**, **String** and **Date**. **Unit** and **Decimals**: Specify unit and decimal precision for numbers. **Format**: Specify date format for dates. + +### String +#### Value/Range to text mapping + +> Only available in Grafana v5.1+. + +{{< docs-imagebox img="/img/docs/v51/table-value-mapping.png" class="docs-image--right docs-image--no-shadow">}} + +Value/range to text mapping allows you to translate numeric values into explicit text. The text will respect all styling, thresholds and customization defined for the value. This can be useful to translate the numeric values into a context-specific human-readable word or message. + +
diff --git a/docs/sources/features/shortcuts.md b/docs/sources/features/shortcuts.md index cbcf3670c83..88c645eafdf 100644 --- a/docs/sources/features/shortcuts.md +++ b/docs/sources/features/shortcuts.md @@ -42,6 +42,7 @@ Hit `?` on your keyboard to open the shortcuts help modal. - `e` Toggle panel edit view - `v` Toggle panel fullscreen view - `p` `s` Open Panel Share Modal +- `p` `d` Duplicate Panel - `p` `r` Remove Panel ### Time Range diff --git a/docs/sources/guides/basic_concepts.md b/docs/sources/guides/basic_concepts.md index b710a227a79..d3f8dd0ba63 100644 --- a/docs/sources/guides/basic_concepts.md +++ b/docs/sources/guides/basic_concepts.md @@ -54,7 +54,7 @@ We utilize a unit abstraction so that Grafana looks great on all screens both sm > Note: With MaxDataPoint functionality, Grafana can show you the perfect amount of datapoints no matter your resolution or time-range. -Utilize the [Repeating Row functionality](/reference/templating/#utilizing-template-variables-with-repeating-panels-and-repeating-rows) to dynamically create or remove entire Rows (that can be filled with Panels), based on the Template variables selected. +Utilize the [Repeating Rows functionality](/reference/templating/#repeating-rows) to dynamically create or remove entire Rows (that can be filled with Panels), based on the Template variables selected. Rows can be collapsed by clicking on the Row Title. If you save a Dashboard with a Row collapsed, it will save in that state and will not preload those graphs until the row is expanded. @@ -72,7 +72,7 @@ Panels like the [Graph](/reference/graph/) panel allow you to graph as many metr Panels can be made more dynamic by utilizing [Dashboard Templating](/reference/templating/) variable strings within the panel configuration (including queries to your Data Source configured via the Query Editor). -Utilize the [Repeating Panel](/reference/templating/#utilizing-template-variables-with-repeating-panels-and-repeating-rows) functionality to dynamically create or remove Panels based on the [Templating Variables](/reference/templating/#utilizing-template-variables-with-repeating-panels-and-repeating-rows) selected. +Utilize the [Repeating Panel](/reference/templating/#repeating-panels) functionality to dynamically create or remove Panels based on the [Templating Variables](/reference/templating/#repeating-panels) selected. The time range on Panels is normally what is set in the [Dashboard time picker](/reference/timerange/) but this can be overridden by utilizes [Panel specific time overrides](/reference/timerange/#panel-time-overrides-timeshift). diff --git a/docs/sources/guides/getting_started.md b/docs/sources/guides/getting_started.md index f724504156f..27957990265 100644 --- a/docs/sources/guides/getting_started.md +++ b/docs/sources/guides/getting_started.md @@ -13,7 +13,35 @@ weight = 1 # Getting started -This guide will help you get started and acquainted with Grafana. It assumes you have a working Grafana server up and running and have added at least one [Data Source](/features/datasources/). +This guide will help you get started and acquainted with Grafana. It assumes you have a working Grafana server up and running. If not please read the [installation guide](/installation/). + +## Logging in for the first time + +To run Grafana open your browser and go to http://localhost:3000/. 3000 is the default http port that Grafana listens to if you haven't [configured a different port](/installation/configuration/#http-port). + +There you will see the login page. Default username is admin and default password is admin. When you log in for the first time you will be asked to change your password. We strongly encourage you to +follow Grafana’s best practices and change the default administrator password. You can later go to user preferences and change your user name. + + +## How to add a data source + +{{< docs-imagebox img="/img/docs/v52/sidemenu-datasource.png" max-width="250px" class="docs-image--right docs-image--no-shadow">}} + +Before you create your first dashboard you need to add your data source. + +First move your cursor to the cog on the side menu which will show you the configuration menu. If the side menu is not visible click the Grafana icon in the upper left corner. The first item on the configuration menu is data sources, click on that and you'll be taken to the data sources page where you can add and edit data sources. You can also simply click the cog. + + +Click Add data source and you will come to the settings page of your new data source. + +{{< docs-imagebox img="/img/docs/v52/add-datasource.png" max-width="700px" class="docs-image--no-shadow">}} + +First, give the data source a Name and then select which Type of data source you'll want to create, see [Supported data sources](/features/datasources/#supported-data-sources/) for more information and how to configure your data source. + + +{{< docs-imagebox img="/img/docs/v52/datasource-settings.png" max-width="700px" class="docs-image--no-shadow">}} + +After you have configuered your data source you are ready to save and test. ## Beginner guides @@ -41,7 +69,7 @@ The image above shows you the top header for a Dashboard. ## Dashboards, Panels, the building blocks of Grafana... -Dashboards are at the core of what Grafana is all about. Dashboards are composed of individual Panels arranged on a grid. Grafana ships with a variety of Panels. Grafana makes it easy to construct the right queries, and customize the display properties so that you can create the perfect Dashboard for your need. Each Panel can interact with data from any configured Grafana Data Source (currently InfluxDB, Graphite, OpenTSDB, Prometheus and Cloudwatch). The [Basic Concepts](/guides/basic_concepts) guide explores these key ideas in detail. +Dashboards are at the core of what Grafana is all about. Dashboards are composed of individual Panels arranged on a grid. Grafana ships with a variety of Panels. Grafana makes it easy to construct the right queries, and customize the display properties so that you can create the perfect Dashboard for your need. Each Panel can interact with data from any configured Grafana Data Source (currently Graphite, Prometheus, Elasticsearch, InfluxDB, OpenTSDB, MySQL, PostgreSQL, Microsoft SQL Server and AWS Cloudwatch). The [Basic Concepts](/guides/basic_concepts) guide explores these key ideas in detail. diff --git a/docs/sources/guides/whats-new-in-v2-5.md b/docs/sources/guides/whats-new-in-v2-5.md index 90270ea1121..08d51ba5bd7 100644 --- a/docs/sources/guides/whats-new-in-v2-5.md +++ b/docs/sources/guides/whats-new-in-v2-5.md @@ -25,7 +25,7 @@ correctly in UTC mode.
This release brings a fully featured query editor for Elasticsearch. You will now be able to visualize -logs or any kind of data stored in Elasticserarch. The query editor allows you to build both simple +logs or any kind of data stored in Elasticsearch. The query editor allows you to build both simple and complex queries for logs or metrics. - Compute metrics from your documents, supported Elasticsearch aggregations: diff --git a/docs/sources/guides/whats-new-in-v2-6.md b/docs/sources/guides/whats-new-in-v2-6.md index b8996680ce6..1e6f30c597b 100644 --- a/docs/sources/guides/whats-new-in-v2-6.md +++ b/docs/sources/guides/whats-new-in-v2-6.md @@ -15,7 +15,7 @@ support for multiple Cloudwatch credentials. The new table panel is very flexible, supporting both multiple modes for time series as well as for -table, annotation and raw JSON data. It also provides date formating and value formating and coloring options. +table, annotation and raw JSON data. It also provides date formatting and value formatting and coloring options. ### Time series to rows diff --git a/docs/sources/guides/whats-new-in-v2.md b/docs/sources/guides/whats-new-in-v2.md index 499849c8d83..28d068b1cd6 100644 --- a/docs/sources/guides/whats-new-in-v2.md +++ b/docs/sources/guides/whats-new-in-v2.md @@ -34,7 +34,7 @@ Organizations via a role. That role can be: There are currently no permissions on individual dashboards. -Read more about Grafanas new user model on the [Admin section](../reference/admin/) +Read more about Grafana's new user model on the [Admin section](../reference/admin/) ## Dashboard Snapshot sharing diff --git a/docs/sources/guides/whats-new-in-v3-1.md b/docs/sources/guides/whats-new-in-v3-1.md index 1e8ef87297b..ab6c5281275 100644 --- a/docs/sources/guides/whats-new-in-v3-1.md +++ b/docs/sources/guides/whats-new-in-v3-1.md @@ -21,7 +21,7 @@ The export feature is now accessed from the share menu. Dashboards exported from Grafana 3.1 are now more portable and easier for others to import than before. The export process extracts information data source types used by panels and adds these to a new `inputs` section in the dashboard json. So when you or another person tries to import the dashboard they will be asked to -select data source and optional metrix prefix options. +select data source and optional metric prefix options. @@ -53,7 +53,7 @@ Grafana url to share with a colleague without having to use the Share modal. ## Internal metrics -Do you want metrics about viewing metrics? Ofc you do! In this release we added support for sending metrics about Grafana to graphite. +Do you want metrics about viewing metrics? Of course you do! In this release we added support for sending metrics about Grafana to graphite. You can configure interval and server in the config file. ## Logging diff --git a/docs/sources/guides/whats-new-in-v3.md b/docs/sources/guides/whats-new-in-v3.md index d82a833ec90..dbd9b685a2b 100644 --- a/docs/sources/guides/whats-new-in-v3.md +++ b/docs/sources/guides/whats-new-in-v3.md @@ -197,7 +197,7 @@ you can install it manually from [Grafana.com](https://grafana.com) ## Plugin showcase Discovering and installing plugins is very quick and easy with Grafana 3.0 and [Grafana.com](https://grafana.com). Here -are a couple that I incurage you try! +are a couple that I encourage you try! #### [Clock Panel](https://grafana.com/plugins/grafana-clock-panel) Support's both current time and count down mode. diff --git a/docs/sources/guides/whats-new-in-v4-1.md b/docs/sources/guides/whats-new-in-v4-1.md index bd2b0f1b75f..0cecff68cf5 100644 --- a/docs/sources/guides/whats-new-in-v4-1.md +++ b/docs/sources/guides/whats-new-in-v4-1.md @@ -11,7 +11,7 @@ weight = 3 +++ -## Whats new in Grafana v4.1 +## What's new in Grafana v4.1 - **Graph**: Support for shared tooltip on all graphs as you hover over one graph. [#1578](https://github.com/grafana/grafana/pull/1578), [#6274](https://github.com/grafana/grafana/pull/6274) - **Victorops**: Add VictorOps notification integration [#6411](https://github.com/grafana/grafana/issues/6411), thx [@ichekrygin](https://github.com/ichekrygin) - **Opsgenie**: Add OpsGenie notification integratiion [#6687](https://github.com/grafana/grafana/issues/6687), thx [@kylemcc](https://github.com/kylemcc) @@ -24,7 +24,7 @@ weight = 3 {{< imgbox max-width="60%" img="/img/docs/v41/shared_tooltip.gif" caption="Shared tooltip" >}} -Showing the tooltip on all panels at the same time has been a long standing request in Grafana and we are really happy to finally be able to release it. +Showing the tooltip on all panels at the same time has been a long standing request in Grafana and we are really happy to finally be able to release it. You can enable/disable the shared tooltip from the dashboard settings menu or cycle between default, shared tooltip and shared crosshair by pressing `CTRL + O` or `CMD + O`.
@@ -33,7 +33,7 @@ You can enable/disable the shared tooltip from the dashboard settings menu or cy {{< imgbox max-width="60%" img="/img/docs/v41/helptext_for_panel_settings.png" caption="Hovering help text" >}} -You can set a help text in the general tab on any panel. The help text is using Markdown to enable better formating and linking to other sites that can provide more information. +You can set a help text in the general tab on any panel. The help text is using Markdown to enable better formatting and linking to other sites that can provide more information.
@@ -50,7 +50,7 @@ Panels with a help text available have a little indicator in the top left corner In Grafana 4.1.0 you can configure your Cloudwatch data source with `access key` and `secret key` directly in the data source configuration page. This enables people to use the Cloudwatch data source without having access to the filesystem where Grafana is running. -Once the `access key` and `secret key` have been saved the user will no longer be able to view them. +Once the `access key` and `secret key` have been saved the user will no longer be able to view them.
## Upgrade & Breaking changes diff --git a/docs/sources/guides/whats-new-in-v4-2.md b/docs/sources/guides/whats-new-in-v4-2.md index 4b140a9027e..7a00023172a 100644 --- a/docs/sources/guides/whats-new-in-v4-2.md +++ b/docs/sources/guides/whats-new-in-v4-2.md @@ -10,7 +10,7 @@ parent = "whatsnew" weight = -1 +++ -## Whats new in Grafana v4.2 +## What's new in Grafana v4.2 Grafana v4.2 Beta is now [available for download](https://grafana.com/grafana/download/4.2.0). Just like the last release this one contains lots bug fixes and minor improvements. @@ -45,7 +45,7 @@ We might add more global built in variables in the future and if we do we will p ### Dedupe alert notifications when running multiple servers -In this release we will dedupe alert notificiations when you are running multiple servers. +In this release we will dedupe alert notifications when you are running multiple servers. This makes it possible to run alerting on multiple servers and only get one notification. We currently solve this with sql transactions which puts some limitations for how many servers you can use to execute the same rules. @@ -67,7 +67,7 @@ Making it possible to have users in multiple groups and have detailed access con ## Upgrade & Breaking changes -If your using https in grafana we now force you to use tls 1.2 and the most secure ciphers. +If you're using https in grafana we now force you to use tls 1.2 and the most secure ciphers. We think its better to be secure by default rather then making it configurable. If you want to run https with lower versions of tls we suggest you put a reserve proxy in front of grafana. diff --git a/docs/sources/guides/whats-new-in-v4-5.md b/docs/sources/guides/whats-new-in-v4-5.md index b2de451308a..c6cfcf64720 100644 --- a/docs/sources/guides/whats-new-in-v4-5.md +++ b/docs/sources/guides/whats-new-in-v4-5.md @@ -12,7 +12,7 @@ weight = -4 # What's New in Grafana v4.5 -## Hightlights +## Highlights ### New prometheus query editor @@ -45,7 +45,7 @@ More information [here](https://community.grafana.com/t/using-grafanas-query-ins ### Enhancements * **GitHub OAuth**: Support for GitHub organizations with 100+ teams. [#8846](https://github.com/grafana/grafana/issues/8846), thx [@skwashd](https://github.com/skwashd) -* **Graphite**: Calls to Graphite api /metrics/find now include panel or dashboad time range (from & until) in most cases, [#8055](https://github.com/grafana/grafana/issues/8055) +* **Graphite**: Calls to Graphite api /metrics/find now include panel or dashboard time range (from & until) in most cases, [#8055](https://github.com/grafana/grafana/issues/8055) * **Graphite**: Added new graphite 1.0 functions, available if you set version to 1.0.x in data source settings. New Functions: mapSeries, reduceSeries, isNonNull, groupByNodes, offsetToZero, grep, weightedAverage, removeEmptySeries, aggregateLine, averageOutsidePercentile, delay, exponentialMovingAverage, fallbackSeries, integralByInterval, interpolate, invert, linearRegression, movingMin, movingMax, movingSum, multiplySeriesWithWildcards, pow, powSeries, removeBetweenPercentile, squareRoot, timeSlice, closes [#8261](https://github.com/grafana/grafana/issues/8261) - **Elasticsearch**: Ad-hoc filters now use query phrase match filters instead of term filters, works on non keyword/raw fields [#9095](https://github.com/grafana/grafana/issues/9095). @@ -53,7 +53,7 @@ More information [here](https://community.grafana.com/t/using-grafanas-query-ins * **InfluxDB/Elasticsearch**: The panel & data source option named "Group by time interval" is now named "Min time interval" and does now always define a lower limit for the auto group by time. Without having to use `>` prefix (that prefix still works). This should in theory have close to zero actual impact on existing dashboards. It does mean that if you used this setting to define a hard group by time interval of, say "1d", if you zoomed to a time range wide enough the time range could increase above the "1d" range as the setting is now always considered a lower limit. -This option is now rennamed (and moved to Options sub section above your queries): +This option is now renamed (and moved to Options sub section above your queries): ![image|519x120](upload://ySjHOVpavV6yk9LHQxL9nq2HIsT.png) Datas source selection & options & help are now above your metric queries. @@ -62,7 +62,7 @@ Datas source selection & options & help are now above your metric queries. ### Minor Changes * **InfluxDB**: Change time range filter for absolute time ranges to be inclusive instead of exclusive [#8319](https://github.com/grafana/grafana/issues/8319), thx [@Oxydros](https://github.com/Oxydros) -* **InfluxDB**: Added paranthesis around tag filters in queries [#9131](https://github.com/grafana/grafana/pull/9131) +* **InfluxDB**: Added parenthesis around tag filters in queries [#9131](https://github.com/grafana/grafana/pull/9131) ## Bug Fixes diff --git a/docs/sources/guides/whats-new-in-v4-6.md b/docs/sources/guides/whats-new-in-v4-6.md index fd75384761f..91fa74084a8 100644 --- a/docs/sources/guides/whats-new-in-v4-6.md +++ b/docs/sources/guides/whats-new-in-v4-6.md @@ -45,7 +45,7 @@ This makes exploring and filtering Prometheus data much easier. * **GCS**: Adds support for Google Cloud Storage [#8370](https://github.com/grafana/grafana/issues/8370) thx [@chuhlomin](https://github.com/chuhlomin) * **Prometheus**: Adds /metrics endpoint for exposing Grafana metrics. [#9187](https://github.com/grafana/grafana/pull/9187) -* **Graph**: Add support for local formating in axis. [#1395](https://github.com/grafana/grafana/issues/1395), thx [@m0nhawk](https://github.com/m0nhawk) +* **Graph**: Add support for local formatting in axis. [#1395](https://github.com/grafana/grafana/issues/1395), thx [@m0nhawk](https://github.com/m0nhawk) * **Jaeger**: Add support for open tracing using jaeger in Grafana. [#9213](https://github.com/grafana/grafana/pull/9213) * **Unit types**: New date & time unit types added, useful in singlestat to show dates & times. [#3678](https://github.com/grafana/grafana/issues/3678), [#6710](https://github.com/grafana/grafana/issues/6710), [#2764](https://github.com/grafana/grafana/issues/2764) * **CLI**: Make it possible to install plugins from any url [#5873](https://github.com/grafana/grafana/issues/5873) @@ -61,10 +61,10 @@ This makes exploring and filtering Prometheus data much easier. ### Minor Changes * **SMTP**: Make it possible to set specific EHLO for smtp client. [#9319](https://github.com/grafana/grafana/issues/9319) -* **Dataproxy**: Allow grafan to renegotiate tls connection [#9250](https://github.com/grafana/grafana/issues/9250) +* **Dataproxy**: Allow Grafana to renegotiate tls connection [#9250](https://github.com/grafana/grafana/issues/9250) * **HTTP**: set net.Dialer.DualStack to true for all http clients [#9367](https://github.com/grafana/grafana/pull/9367) * **Alerting**: Add diff and percent diff as series reducers [#9386](https://github.com/grafana/grafana/pull/9386), thx [@shanhuhai5739](https://github.com/shanhuhai5739) -* **Slack**: Allow images to be uploaded to slack when Token is precent [#7175](https://github.com/grafana/grafana/issues/7175), thx [@xginn8](https://github.com/xginn8) +* **Slack**: Allow images to be uploaded to slack when Token is present [#7175](https://github.com/grafana/grafana/issues/7175), thx [@xginn8](https://github.com/xginn8) * **Opsgenie**: Use their latest API instead of old version [#9399](https://github.com/grafana/grafana/pull/9399), thx [@cglrkn](https://github.com/cglrkn) * **Table**: Add support for displaying the timestamp with milliseconds [#9429](https://github.com/grafana/grafana/pull/9429), thx [@s1061123](https://github.com/s1061123) * **Hipchat**: Add metrics, message and image to hipchat notifications [#9110](https://github.com/grafana/grafana/issues/9110), thx [@eloo](https://github.com/eloo) diff --git a/docs/sources/guides/whats-new-in-v5-1.md b/docs/sources/guides/whats-new-in-v5-1.md new file mode 100644 index 00000000000..1f2be3bfedf --- /dev/null +++ b/docs/sources/guides/whats-new-in-v5-1.md @@ -0,0 +1,125 @@ ++++ +title = "What's New in Grafana v5.1" +description = "Feature & improvement highlights for Grafana v5.1" +keywords = ["grafana", "new", "documentation", "5.1"] +type = "docs" +[menu.docs] +name = "Version 5.1" +identifier = "v5.1" +parent = "whatsnew" +weight = -7 ++++ + +# What's New in Grafana v5.1 + +Grafana v5.1 brings new features, many enhancements and bug fixes. This article will detail the major new features and enhancements. + +* [Improved scrolling experience]({{< relref "#improved-scrolling-experience" >}}) +* [Improved docker image]({{< relref "#improved-docker-image-breaking-change" >}}) with a breaking change! +* [Heatmap support for Prometheus]({{< relref "#prometheus" >}}) +* [Microsoft SQL Server]({{< relref "#microsoft-sql-server" >}}) as metric & table datasource! +* [Dashboards & Panels]({{< relref "#dashboards-panels" >}}) Improved adding panels to dashboards and enhancements to Graph and Table panels. +* [New variable interpolation syntax]({{< relref "#new-variable-interpolation-syntax" >}}) +* [Improved workflow for provisioned dashboards]({{< relref "#improved-workflow-for-provisioned-dashboards" >}}) + +## Improved scrolling experience + +In Grafana v5.0 we introduced a new scrollbar component. Unfortunately this introduced a lot of issues and in some scenarios removed +the native scrolling functionality. Grafana v5.1 ships with a native scrollbar for all pages together with a scrollbar component for +the dashboard grid and panels that's not overriding the native scrolling functionality. We hope that these changes and improvements should +make the Grafana user experience much better! + +## Improved docker image (breaking change) + +Grafana v5.1 brings an improved official docker image which should make it easier to run and use the Grafana docker image and at the same time give more control to the user how to use/run it. + +We've switched the id of the grafana user running Grafana inside a docker container. Unfortunately this means that files created prior to 5.1 won't have the correct permissions for later versions and thereby this introduces a breaking change. +We made this change so that it would be easier for you to control what user Grafana is executed as (see examples below). + +Version | User | User ID +--------|---------|--------- +< 5.1 | grafana | 104 +>= 5.1 | grafana | 472 + +Please read the [updated documentation](/installation/docker/#migration-from-a-previous-version-of-the-docker-container-to-5-1-or-later) which includes migration instructions and more information. + +## Prometheus + +{{< docs-imagebox img="/img/docs/v51/prometheus_heatmap.png" max-width="800px" class="docs-image--right" >}} + +The Prometheus datasource now support transforming Prometheus histograms to the heatmap panel. Prometheus histogram is a powerful feature, and we're +really happy to finally allow our users to render those as heatmaps. Please read [Heatmap panel documentation](/features/panels/heatmap/#pre-bucketed-data) +for more information on how to use it. + +Prometheus query editor also got support for autocomplete of template variables. More information in the [Prometheus data source documentation](/features/datasources/prometheus/). + +
+ +## Microsoft SQL Server + +{{< docs-imagebox img="/img/docs/v51/mssql_query_editor_showcase.png" max-width= "800px" class="docs-image--right" >}} + +Grafana v5.1 now ships with a built-in Microsoft SQL Server (MSSQL) data source plugin that allows you to query and visualize data from any +Microsoft SQL Server 2005 or newer, including Microsoft Azure SQL Database. Do you have metric or log data in MSSQL? You can now visualize +that data and define alert rules on it like with any of Grafana's other core datasources. + +Please read [Using Microsoft SQL Server in Grafana documentation](/features/datasources/mssql/) for more detailed information on how to get started and use it. + +
+ +## Dashboards & Panels + +### Adding new panels to dashboards + +{{< docs-imagebox img="/img/docs/v51/dashboard_add_panel.png" max-width= "800px" class="docs-image--right" >}} + +The control for adding new panels to dashboards have got some enhancements and now includes functionality to search for the type of panel +you want to add. Further, the control has tabs separating functionality for adding new panels and pasting +copied panels. + +By copying a panel in a dashboard it will be displayed in the `Paste` tab in *any* dashboard and allows you to paste the +copied panel into the current dashboard. + +{{< docs-imagebox img="/img/docs/v51/dashboard_panel_copy.png" max-width= "300px" >}} + +
+ +### Graph Panel + +New enhancements includes support for multiple series stacking in histogram mode, thresholds for right Y axis, aligning left and right Y-axes to one level and additional units. More information in the [Graph panel documentation](/features/panels/graph/). + +### Table Panel + +New enhancements includes support for mapping a numeric value/range to text and additional units. More information in the [Table panel documentation](/features/panels/table_panel/#string). + +## New variable interpolation syntax + +We now support a new option for rendering variables that gives the user full control of how the value(s) should be rendered. +In the table below you can see some examples and you can find all different options in the [Variables documentation](http://docs.grafana.org/reference/templating/#advanced-formatting-options). + +Filter Option | Example | Raw | Interpolated | Description +------------ | ------------- | ------------- | ------------- | ------------- +`glob` | ${servers:glob} | `'test1', 'test2'` | `{test1,test2}` | Formats multi-value variable into a glob +`regex` | ${servers:regex} | `'test.', 'test2'` | (test\.|test2) | Formats multi-value variable into a regex string +`pipe` | ${servers:pipe} | `'test.', 'test2'` | test.|test2 | Formats multi-value variable into a pipe-separated string +`csv`| ${servers:csv} | `'test1', 'test2'` | `test1,test2` | Formats multi-value variable as a comma-separated string + +## Improved workflow for provisioned dashboards + +{{< docs-imagebox img="/img/docs/v51/provisioning_cannot_save_dashboard.png" max-width="800px" class="docs-image--right" >}} + +Grafana v5.1 brings an improved workflow for provisioned dashboards: + +* A populated `id` property in JSON is now automatically removed when provisioning dashboards. +* When making changes to a provisioned dashboard you can `Save` the dashboard which now will bring up a *Cannot save provisioned dashboard* dialog like seen in the screenshot to the right. + + +Available options in the dialog will let you `Copy JSON to Clipboard` and/or `Save JSON to file` which can help you synchronize your dashboard changes back to the provisioning source. +More information in the [Provisioning documentation](/administration/provisioning/). + +
+ +## Changelog + +Checkout the [CHANGELOG.md](https://github.com/grafana/grafana/blob/master/CHANGELOG.md) file for a complete list +of new features, changes, and bug fixes. diff --git a/docs/sources/guides/whats-new-in-v5-2.md b/docs/sources/guides/whats-new-in-v5-2.md new file mode 100644 index 00000000000..e084f8618e4 --- /dev/null +++ b/docs/sources/guides/whats-new-in-v5-2.md @@ -0,0 +1,101 @@ ++++ +title = "What's New in Grafana v5.2" +description = "Feature & improvement highlights for Grafana v5.2" +keywords = ["grafana", "new", "documentation", "5.2"] +type = "docs" +[menu.docs] +name = "Version 5.2" +identifier = "v5.2" +parent = "whatsnew" +weight = -8 ++++ + +# What's New in Grafana v5.2 + +Grafana v5.2 brings new features, many enhancements and bug fixes. This article will detail the major new features and enhancements. + +- [Elasticsearch alerting]({{< relref "#elasticsearch-alerting" >}}) it's finally here! +- [Native builds for ARM]({{< relref "#native-builds-for-arm" >}}) native builds of Grafana for many more platforms! +- [Improved Docker image]({{< relref "#improved-docker-image" >}}) with support for docker secrets +- [Security]({{< relref "#security" >}}) make your Grafana instance more secure +- [Prometheus]({{< relref "#prometheus" >}}) with alignment enhancements +- [InfluxDB]({{< relref "#influxdb" >}}) now supports the `mode` function +- [Alerting]({{< relref "#alerting" >}}) with alert notification channel type for Discord +- [Dashboards & Panels]({{< relref "#dashboards-panels" >}}) with save & import enhancements + +## Elasticsearch alerting + +{{< docs-imagebox img="/img/docs/v52/elasticsearch_alerting.png" max-width="800px" class="docs-image--right" >}} + +Grafana v5.2 ships with an updated Elasticsearch datasource with support for alerting. Alerting support for Elasticsearch has been one of +the most requested features by our community and now it's finally here. Please try it out and let us know what you think. + +
+ +## Native builds for ARM + +Grafana v5.2 brings an improved build pipeline with cross-platform support. This enables native builds of Grafana for ARMv7 (x32) and ARM64 (x64). +We've been longing for native ARM build support for ages. With the help from our amazing community this is now finally available. +Please try it out and let us know what you think. + +Another great addition with the improved build pipeline is that binaries for MacOS/Darwin (x64) and Windows (x64) are now automatically built and +published for both stable and nightly builds. + +## Improved Docker image + +The Grafana docker image adds support for Docker secrets which enables you to supply Grafana with configuration through files. More +information in the [Installing using Docker documentation](/installation/docker/#reading-secrets-from-files-support-for-docker-secrets). + +## Security + +{{< docs-imagebox img="/img/docs/v52/login_change_password.png" max-width="800px" class="docs-image--right" >}} + +Starting from Grafana v5.2, when you login with the administrator account using the default password you'll be presented with a form to change the password. +We hope this encourages users to follow Grafana's best practices and change the default administrator password. + +
+ +## Prometheus + +The Prometheus datasource now aligns the start/end of the query sent to Prometheus with the step, which ensures PromQL expressions with *rate* +functions get consistent results, and thus avoids graphs jumping around on reload. + +## InfluxDB + +The InfluxDB datasource now includes support for the *mode* function which returns the most frequent value in a list of field values. + +## Alerting + +By popular demand Grafana now includes support for an alert notification channel type for [Discord](https://discordapp.com/). + +## Dashboards & Panels + +### Modified time range and variables are no longer saved by default + +{{< docs-imagebox img="/img/docs/v52/dashboard_save_modal.png" max-width="800px" class="docs-image--right" >}} + +Starting from Grafana v5.2, a modified time range or variable are no longer saved by default. To save a modified +time range or variable, you'll need to actively select that when saving a dashboard, see screenshot. +This should hopefully make it easier to have sane defaults for time and variables in dashboards and make it more explicit +when you actually want to overwrite those settings. + +
+ +### Import dashboard enhancements + +{{< docs-imagebox img="/img/docs/v52/dashboard_import.png" max-width="800px" class="docs-image--right" >}} + +Grafana v5.2 adds support for specifying an existing folder or creating a new one when importing a dashboard - a long-awaited feature since +Grafana v5.0 introduced support for dashboard folders and permissions. The import dashboard page has also got some general improvements +and should now make it more clear if a possible import will overwrite an existing dashboard, or not. + +This release also adds some improvements for those users only having editor or admin permissions in certain folders. The links to +*Create Dashboard* and *Import Dashboard* are now available in the side navigation, in dashboard search and on the manage dashboards/folder page for a +user that has editor role in an organization or the edit permission in at least one folder. + +
+ +## Changelog + +Checkout the [CHANGELOG.md](https://github.com/grafana/grafana/blob/master/CHANGELOG.md) file for a complete list +of new features, changes, and bug fixes. diff --git a/docs/sources/guides/whats-new-in-v5-3.md b/docs/sources/guides/whats-new-in-v5-3.md new file mode 100644 index 00000000000..10592f51648 --- /dev/null +++ b/docs/sources/guides/whats-new-in-v5-3.md @@ -0,0 +1,92 @@ ++++ +title = "What's New in Grafana v5.3" +description = "Feature & improvement highlights for Grafana v5.3" +keywords = ["grafana", "new", "documentation", "5.3"] +type = "docs" +[menu.docs] +name = "Version 5.3" +identifier = "v5.3" +parent = "whatsnew" +weight = -9 ++++ + +# What's New in Grafana v5.3 + +Grafana v5.3 brings new features, many enhancements and bug fixes. This article will detail the major new features and enhancements. + +- [Google Stackdriver]({{< relref "#google-stackdriver" >}}) as a core datasource! +- [TV mode]({{< relref "#tv-and-kiosk-mode" >}}) is improved and more accessible +- [Alerting]({{< relref "#notification-reminders" >}}) with notification reminders +- [Postgres]({{< relref "#postgres-query-builder" >}}) gets a new query builder! +- [OAuth]({{< relref "#improved-oauth-support-for-gitlab" >}}) support for GitLab is improved +- [Annotations]({{< relref "#annotations" >}}) with template variable filtering +- [Variables]({{< relref "#variables" >}}) with free text support + +## Google Stackdriver + +{{< docs-imagebox img="/img/docs/v53/stackdriver-with-heatmap.png" max-width= "600px" class="docs-image--no-shadow docs-image--right" >}} + +Grafana v5.3 ships with built-in support for [Google Stackdriver](https://cloud.google.com/stackdriver/) and enables you to visualize your Stackdriver metrics in Grafana. + +Getting started with the plugin is easy. Simply create a GCE Service account that has access to the Stackdriver API scope, download the Service Account key file from Google and upload it on the Stackdriver datasource config page in Grafana and you should have a secure server-to-server authentication setup. Like other core plugins, Stackdriver has built-in support for alerting. It also comes with support for heatmaps and basic variables. + +If you're already accustomed to the Stackdriver Metrics Explorer UI, you'll notice that there are a lot of similarities to the query editor in Grafana. It is possible to add filters using wildcards and regular expressions. You can do Group By, Primary Aggregation and Alignment. + +Alias By allows you to format the legend the way you want, and it's a feature that is not yet present in the Metrics Explorer. Two other features that are only supported in the Grafana plugin are the abilities to manually set the Alignment Period in the query editor and to add Annotations queries. + +The Grafana Stackdriver plugin comes with support for automatic unit detection. Grafana will try to map the Stackdriver unit type to a corresponding unit type in Grafana, and if successful the panel Y-axes will be updated accordingly to display the correct unit of measure. This is the first core plugin to provide support for unit detection, and it is our intention to provide support for this in other core plugins in the near future. + +The datasource is still in the `beta` phase, meaning it's currently in active development and is still missing one important feature - templating queries. +Please try it out, but be aware of that it might be subject to changes and possible bugs. We would love to hear your feedback. + +Please read [Using Google Stackdriver in Grafana](/features/datasources/stackdriver/) for more detailed information on how to get started and use it. + +## TV and Kiosk Mode + +{{< docs-imagebox img="/img/docs/v53/tv_mode_still.png" max-width="600px" class="docs-image--no-shadow docs-image--right" animated-gif="/img/docs/v53/tv_mode.gif" >}} + +We've improved the TV & kiosk mode to make it easier to use. There's now an icon in the top bar that will let you cycle through the different view modes. + +1. In the first view mode, the sidebar and most of the buttons in the top bar will be hidden. +2. In the second view mode, the top bar is completely hidden so that only the dashboard itself is shown. +3. Hit the escape key to go back to the default view mode. + +When switching view modes, the url will be updated to reflect the view mode selected. This allows a dashboard to be opened with a +certain view mode enabled. Additionally, this also enables [playlists](/reference/playlist) to be started with a certain view mode enabled. + +
+ +## Notification Reminders + +Do you use Grafana alerting and have some notifications that are more important than others? Then it's possible to set reminders so that you continue to be alerted until the problem is fixed. This is done on the notification channel itself and will affect all alerts that use that channel. +For additional examples of why reminders might be useful for you, see [multiple series](/alerting/rules/#multiple-series). + +Learn how to enable and configure reminders [here](/alerting/notifications/#send-reminders). + +## Postgres Query Builder + +Grafana 5.3 comes with a new graphical query builder for Postgres. This brings Postgres integration more in line with some of the other datasources and makes it easier for both advanced users and beginners to work with timeseries in Postgres. Learn more about it in the [documentation](/features/datasources/postgres/#query-editor). + +{{< docs-imagebox img="/img/docs/v53/postgres_query_still.png" class="docs-image--no-shadow" animated-gif="/img/docs/v53/postgres_query.gif" >}} + +## Improved OAuth Support for GitLab + +Grafana 5.3 comes with a new OAuth integration for GitLab that enables configuration to only allow users that are a member of certain GitLab groups to authenticate. This makes it possible to use GitLab OAuth with Grafana in a shared environment without giving everyone access to Grafana. +Learn how to enable and configure it in the [documentation](/auth/gitlab/). + +## Annotations + +Grafana 5.3 brings improved support for [native annotations](/reference/annotations/#native-annotations) and makes it possible to use template variables when filtering by tags. +Learn more about it in the [documentation](/reference/annotations/#query-by-tag). + +{{< docs-imagebox img="/img/docs/v53/annotation_tag_filter_variable.png" max-width="600px" >}} + +## Variables + +Grafana 5.3 ships with a brand new variable type named `Text box` which makes it easier and more convenient to provide free text input to a variable. +This new variable type will display as a free text input field with an optional prefilled default value. + +## Changelog + +Checkout the [CHANGELOG.md](https://github.com/grafana/grafana/blob/master/CHANGELOG.md) file for a complete list +of new features, changes, and bug fixes. diff --git a/docs/sources/http_api/admin.md b/docs/sources/http_api/admin.md index 0194c69caac..2d4be21bb78 100644 --- a/docs/sources/http_api/admin.md +++ b/docs/sources/http_api/admin.md @@ -36,11 +36,10 @@ HTTP/1.1 200 Content-Type: application/json { -"DEFAULT": -{ - "app_mode":"production"}, - "analytics": - { + "DEFAULT": { + "app_mode":"production" + }, + "analytics": { "google_analytics_ua_id":"", "reporting_enabled":"false" }, @@ -195,15 +194,16 @@ HTTP/1.1 200 Content-Type: application/json { - "user_count":2, - "org_count":1, - "dashboard_count":4, - "db_snapshot_count":2, - "db_tag_count":6, - "data_source_count":1, - "playlist_count":1, - "starred_db_count":2, - "grafana_admin_count":2 + "users":2, + "orgs":1, + "dashboards":4, + "snapshots":2, + "tags":6, + "datasources":1, + "playlists":1, + "stars":2, + "alerts":2, + "activeUsers":1 } ``` @@ -340,4 +340,4 @@ HTTP/1.1 200 Content-Type: application/json {state: "new state", message: "alerts pause/un paused", "alertsAffected": 100} -``` \ No newline at end of file +``` diff --git a/docs/sources/http_api/alerting.md b/docs/sources/http_api/alerting.md index 3860ae490b1..2d70a6d2017 100644 --- a/docs/sources/http_api/alerting.md +++ b/docs/sources/http_api/alerting.md @@ -35,32 +35,34 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk `/api/alerts?dashboardId=1` - - **dashboardId** – Return alerts for a specified dashboard. - - **panelId** – Return alerts for a specified panel on a dashboard. - - **limit** - Limit response to x number of alerts. + - **dashboardId** – Limit response to alerts in specified dashboard(s). You can specify multiple dashboards, e.g. dashboardId=23&dashboardId=35. + - **panelId** – Limit response to alert for a specified panel on a dashboard. + - **query** - Limit response to alerts having a name like this value. - **state** - Return alerts with one or more of the following alert states: `ALL`,`no_data`, `paused`, `alerting`, `ok`, `pending`. To specify multiple states use the following format: `?state=paused&state=alerting` + - **limit** - Limit response to *X* number of alerts. + - **folderId** – Limit response to alerts of dashboards in specified folder(s). You can specify multiple folders, e.g. folderId=23&folderId=35. + - **dashboardQuery** - Limit response to alerts having a dashboard name like this value. + - **dashboardTag** - Limit response to alerts of dashboards with specified tags. To do an "AND" filtering with multiple tags, specify the tags parameter multiple times e.g. dashboardTag=tag1&dashboardTag=tag2. + **Example Response**: ```http HTTP/1.1 200 Content-Type: application/json + [ { "id": 1, "dashboardId": 1, + "dashboardUId": "ABcdEFghij" + "dashboardSlug": "sensors", "panelId": 1, "name": "fire place sensor", - "message": "Someone is trying to break in through the fire place", "state": "alerting", + "newStateDate": "2018-05-14T05:55:20+02:00", "evalDate": "0001-01-01T00:00:00Z", - "evalData": [ - { - "metric": "fire", - "tags": null, - "value": 5.349999999999999 - } - "newStateDate": "2016-12-25", + "evalData": null, "executionError": "", "url": "http://grafana.com/dashboard/db/sensors" } @@ -85,19 +87,39 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ```http HTTP/1.1 200 Content-Type: application/json + { "id": 1, "dashboardId": 1, + "dashboardUId": "ABcdEFghij" + "dashboardSlug": "sensors", "panelId": 1, "name": "fire place sensor", - "message": "Someone is trying to break in through the fire place", "state": "alerting", - "newStateDate": "2016-12-25", + "message": "Someone is trying to break in through the fire place", + "newStateDate": "2018-05-14T05:55:20+02:00", + "evalDate": "0001-01-01T00:00:00Z", + "evalData": "evalMatches": [ + { + "metric": "movement", + "tags": { + "name": "fireplace_chimney" + }, + "value": 98.765 + } + ], "executionError": "", "url": "http://grafana.com/dashboard/db/sensors" } ``` +**Important Note**: +"evalMatches" data is cached in the db when and only when the state of the alert changes +(e.g. transitioning from "ok" to "alerting" state). + +If data from one server triggers the alert first and, before that server is seen leaving alerting state, +a second server also enters a state that would trigger the alert, the second server will not be visible in "evalMatches" data. + ## Pause alert `POST /api/alerts/:id/pause` @@ -126,6 +148,7 @@ JSON Body Schema: ```http HTTP/1.1 200 Content-Type: application/json + { "alertId": 1, "state": "Paused", @@ -157,6 +180,7 @@ JSON Body Schema: ```http HTTP/1.1 200 Content-Type: application/json + { "state": "Paused", "message": "alert paused", @@ -184,19 +208,26 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk HTTP/1.1 200 Content-Type: application/json -{ - "id": 1, - "name": "Team A", - "type": "email", - "isDefault": true, - "created": "2017-01-01 12:45", - "updated": "2017-01-01 12:45" -} +[ + { + "id": 1, + "name": "Team A", + "type": "email", + "isDefault": false, + "sendReminder": false, + "settings": { + "addresses": "carl@grafana.com;dev@grafana.com" + }, + "created": "2018-04-23T14:44:09+02:00", + "updated": "2018-08-20T15:47:49+02:00" + } +] + ``` ## Create alert notification -You can find the full list of [supported notifers](/alerting/notifications/#all-supported-notifier) at the alert notifiers page. +You can find the full list of [supported notifiers](/alerting/notifications/#all-supported-notifier) at the alert notifiers page. `POST /api/alert-notifications` @@ -212,6 +243,7 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk "name": "new alert notification", //Required "type": "email", //Required "isDefault": false, + "sendReminder": false, "settings": { "addresses": "carl@grafana.com;dev@grafana.com" } @@ -223,14 +255,18 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ```http HTTP/1.1 200 Content-Type: application/json + { "id": 1, "name": "new alert notification", "type": "email", "isDefault": false, - "settings": { addresses: "carl@grafana.com;dev@grafana.com"} } - "created": "2017-01-01 12:34", - "updated": "2017-01-01 12:34" + "sendReminder": false, + "settings": { + "addresses": "carl@grafana.com;dev@grafana.com" + }, + "created": "2018-04-23T14:44:09+02:00", + "updated": "2018-08-20T15:47:49+02:00" } ``` @@ -251,8 +287,10 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk "name": "new alert notification", //Required "type": "email", //Required "isDefault": false, + "sendReminder": true, + "frequency": "15m", "settings": { - "addresses: "carl@grafana.com;dev@grafana.com" + "addresses": "carl@grafana.com;dev@grafana.com" } } ``` @@ -262,12 +300,17 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ```http HTTP/1.1 200 Content-Type: application/json + { "id": 1, "name": "new alert notification", "type": "email", "isDefault": false, - "settings": { addresses: "carl@grafana.com;dev@grafana.com"} } + "sendReminder": true, + "frequency": "15m", + "settings": { + "addresses": "carl@grafana.com;dev@grafana.com" + }, "created": "2017-01-01 12:34", "updated": "2017-01-01 12:34" } @@ -291,6 +334,7 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ```http HTTP/1.1 200 Content-Type: application/json + { "message": "Notification deleted" } diff --git a/docs/sources/http_api/annotations.md b/docs/sources/http_api/annotations.md index 19c2a5c386c..6633714d77b 100644 --- a/docs/sources/http_api/annotations.md +++ b/docs/sources/http_api/annotations.md @@ -32,10 +32,12 @@ Query Parameters: - `from`: epoch datetime in milliseconds. Optional. - `to`: epoch datetime in milliseconds. Optional. -- `limit`: number. Optional - default is 10. Max limit for results returned. +- `limit`: number. Optional - default is 100. Max limit for results returned. - `alertId`: number. Optional. Find annotations for a specified alert. - `dashboardId`: number. Optional. Find annotations that are scoped to a specific dashboard - `panelId`: number. Optional. Find annotations that are scoped to a specific panel +- `userId`: number. Optional. Find annotations created by a specific user +- `type`: string. Optional. `alert`|`annotation` Return alerts or user created annotations - `tags`: string. Optional. Use this to filter global annotations. Global annotations are annotations from an annotation data source that are not connected specifically to a dashboard or panel. To do an "AND" filtering with multiple tags, specify the tags parameter multiple times e.g. `tags=tag1&tags=tag2`. **Example Response**: @@ -180,14 +182,14 @@ Content-Type: application/json ## Delete Annotation By Id -`DELETE /api/annotation/:id` +`DELETE /api/annotations/:id` Deletes the annotation that matches the specified id. **Example Request**: ```http -DELETE /api/annotation/1 HTTP/1.1 +DELETE /api/annotations/1 HTTP/1.1 Accept: application/json Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk @@ -204,14 +206,14 @@ Content-Type: application/json ## Delete Annotation By RegionId -`DELETE /api/annotation/region/:id` +`DELETE /api/annotations/region/:id` Deletes the annotation that matches the specified region id. A region is an annotation that covers a timerange and has a start and end time. In the Grafana database, this is a stored as two annotations connected by a region id. **Example Request**: ```http -DELETE /api/annotation/region/1 HTTP/1.1 +DELETE /api/annotations/region/1 HTTP/1.1 Accept: application/json Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk diff --git a/docs/sources/http_api/auth.md b/docs/sources/http_api/auth.md index 166a5a4fdb9..e87d3571322 100644 --- a/docs/sources/http_api/auth.md +++ b/docs/sources/http_api/auth.md @@ -5,7 +5,7 @@ keywords = ["grafana", "http", "documentation", "api", "authentication"] aliases = ["/http_api/authentication/"] type = "docs" [menu.docs] -name = "Authentication" +name = "Authentication HTTP API" parent = "http_api" +++ @@ -44,6 +44,14 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk The `Authorization` header value should be `Bearer `. +The API Token can also be passed as a Basic authorization password with the special username `api_key`: + +curl example: +```bash +?curl http://api_key:eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk@localhost:3000/api/org +{"id":1,"name":"Main Org."} +``` + # Auth HTTP resources / actions ## Api Keys diff --git a/docs/sources/http_api/dashboard.md b/docs/sources/http_api/dashboard.md index ea1bd7f2ef7..3df36894901 100644 --- a/docs/sources/http_api/dashboard.md +++ b/docs/sources/http_api/dashboard.md @@ -85,7 +85,7 @@ Status Codes: - **403** – Access denied - **412** – Precondition failed -The **412** status code is used for explaing that you cannot create the dashboard and why. +The **412** status code is used for explaining that you cannot create the dashboard and why. There can be different reasons for this: - The dashboard has been changed by someone else, `status=version-mismatch` diff --git a/docs/sources/http_api/dashboard_permissions.md b/docs/sources/http_api/dashboard_permissions.md index 26aa1550d7c..b8f6b06928c 100644 --- a/docs/sources/http_api/dashboard_permissions.md +++ b/docs/sources/http_api/dashboard_permissions.md @@ -106,6 +106,7 @@ Accept: application/json Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +{ "items": [ { "role": "Viewer", diff --git a/docs/sources/http_api/dashboard_versions.md b/docs/sources/http_api/dashboard_versions.md index 3d0ec27a3a3..0be22674997 100644 --- a/docs/sources/http_api/dashboard_versions.md +++ b/docs/sources/http_api/dashboard_versions.md @@ -291,7 +291,7 @@ Content-Type: text/html; charset=UTF-8

``` -The response is a textual respresentation of the diff, with the dashboard values being in JSON, similar to the diffs seen on sites like GitHub or GitLab. +The response is a textual representation of the diff, with the dashboard values being in JSON, similar to the diffs seen on sites like GitHub or GitLab. Status Codes: diff --git a/docs/sources/http_api/data_source.md b/docs/sources/http_api/data_source.md index 364b55b0cfc..9aaf29ec5f4 100644 --- a/docs/sources/http_api/data_source.md +++ b/docs/sources/http_api/data_source.md @@ -188,8 +188,8 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk "defaultRegion": "us-west-1" }, "secureJsonData": { - "accessKey": "Ol4pIDpeKSA6XikgOl4p", - "secretKey": "dGVzdCBrZXkgYmxlYXNlIGRvbid0IHN0ZWFs" + "accessKey": "Ol4pIDpeKSA6XikgOl4p", //should not be encoded + "secretKey": "dGVzdCBrZXkgYmxlYXNlIGRvbid0IHN0ZWFs" //should be Base-64 encoded } } ``` diff --git a/docs/sources/http_api/datasource_permissions.md b/docs/sources/http_api/datasource_permissions.md new file mode 100644 index 00000000000..226beac3728 --- /dev/null +++ b/docs/sources/http_api/datasource_permissions.md @@ -0,0 +1,249 @@ ++++ +title = "Datasource Permissions HTTP API " +description = "Grafana Datasource Permissions HTTP API" +keywords = ["grafana", "http", "documentation", "api", "datasource", "permission", "permissions", "acl", "enterprise"] +aliases = ["/http_api/datasourcepermissions/"] +type = "docs" +[menu.docs] +name = "Datasource Permissions" +parent = "http_api" ++++ + +# Datasource Permissions API + +> Datasource Permissions is only available in Grafana Enterprise. Read more about [Grafana Enterprise]({{< relref "enterprise/index.md" >}}). + +This API can be used to enable, disable, list, add and remove permissions for a datasource. + +Permissions can be set for a user or a team. Permissions cannot be set for Admins - they always have access to everything. + +The permission levels for the permission field: + +- 1 = Query + +## Enable permissions for a datasource + +`POST /api/datasources/:id/enable-permissions` + +Enables permissions for the datasource with the given `id`. No one except Org Admins will be able to query the datasource until permissions have been added which permit certain users or teams to query the datasource. + +**Example request**: + +```http +POST /api/datasources/1/enable-permissions +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{} +``` + +**Example response**: + +```http +HTTP/1.1 200 OK +Content-Type: application/json; charset=UTF-8 +Content-Length: 35 + +{"message":"Datasource permissions enabled"} +``` + +Status Codes: + +- **200** - Ok +- **400** - Permissions cannot be enabled, see response body for details +- **401** - Unauthorized +- **403** - Access denied +- **404** - Datasource not found + +## Disable permissions for a datasource + +`POST /api/datasources/:id/disable-permissions` + +Disables permissions for the datasource with the given `id`. All existing permissions will be removed and anyone will be able to query the datasource. + +**Example request**: + +```http +POST /api/datasources/1/disable-permissions +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{} +``` + +**Example response**: + +```http +HTTP/1.1 200 OK +Content-Type: application/json; charset=UTF-8 +Content-Length: 35 + +{"message":"Datasource permissions disabled"} +``` + +Status Codes: + +- **200** - Ok +- **400** - Permissions cannot be disabled, see response body for details +- **401** - Unauthorized +- **403** - Access denied +- **404** - Datasource not found + +## Get permissions for a datasource + +`GET /api/datasources/:id/permissions` + +Gets all existing permissions for the datasource with the given `id`. + +**Example request**: + +```http +GET /api/datasources/1/permissions HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response** + +```http +HTTP/1.1 200 OK +Content-Type: application/json; charset=UTF-8 +Content-Length: 551 + +{ + "datasourceId": 1, + "enabled": true, + "permissions": + [ + { + "id": 1, + "datasourceId": 1, + "userId": 1, + "userLogin": "user", + "userEmail": "user@test.com", + "userAvatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56", + "permission": 1, + "permissionName": "Query", + "created": "2017-06-20T02:00:00+02:00", + "updated": "2017-06-20T02:00:00+02:00", + }, + { + "id": 2, + "datasourceId": 1, + "teamId": 1, + "team": "A Team", + "teamAvatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56", + "permission": 1, + "permissionName": "Query", + "created": "2017-06-20T02:00:00+02:00", + "updated": "2017-06-20T02:00:00+02:00", + } + ] +} +``` + +Status Codes: + +- **200** - Ok +- **401** - Unauthorized +- **403** - Access denied +- **404** - Datasource not found + +## Add permission for a datasource + +`POST /api/datasources/:id/permissions` + +Adds a user permission for the datasource with the given `id`. + +**Example request**: + +```http +POST /api/datasources/1/permissions +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{ + "userId": 1, + "permission": 1 +} +``` + +**Example response**: + +```http +HTTP/1.1 200 OK +Content-Type: application/json; charset=UTF-8 +Content-Length: 35 + +{"message":"Datasource permission added"} +``` + +Adds a team permission for the datasource with the given `id`. + +**Example request**: + +```http +POST /api/datasources/1/permissions +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{ + "teamId": 1, + "permission": 1 +} +``` + +**Example response**: + +```http +HTTP/1.1 200 OK +Content-Type: application/json; charset=UTF-8 +Content-Length: 35 + +{"message":"Datasource permission added"} +``` + +Status Codes: + +- **200** - Ok +- **400** - Permission cannot be added, see response body for details +- **401** - Unauthorized +- **403** - Access denied +- **404** - Datasource not found + +## Remove permission for a datasource + +`DELETE /api/datasources/:id/permissions/:permissionId` + +Removes the permission with the given `permissionId` for the datasource with the given `id`. + +**Example request**: + +```http +DELETE /api/datasources/1/permissions/2 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example response**: + +```http +HTTP/1.1 200 OK +Content-Type: application/json; charset=UTF-8 +Content-Length: 35 + +{"message":"Datasource permission removed"} +``` + +Status Codes: + +- **200** - Ok +- **401** - Unauthorized +- **403** - Access denied +- **404** - Datasource not found or permission not found diff --git a/docs/sources/http_api/external_group_sync.md b/docs/sources/http_api/external_group_sync.md new file mode 100644 index 00000000000..2ce06c2c94e --- /dev/null +++ b/docs/sources/http_api/external_group_sync.md @@ -0,0 +1,111 @@ ++++ +title = "External Group Sync HTTP API " +description = "Grafana External Group Sync HTTP API" +keywords = ["grafana", "http", "documentation", "api", "team", "teams", "group", "member", "enterprise"] +aliases = ["/http_api/external_group_sync/"] +type = "docs" +[menu.docs] +name = "External Group Sync" +parent = "http_api" ++++ + +# External Group Synchronization API + +> External Group Synchronization is only available in Grafana Enterprise. Read more about [Grafana Enterprise]({{< relref "enterprise/index.md" >}}). + +## Get External Groups + +`GET /api/teams/:teamId/groups` + +**Example Request**: + +```http +GET /api/teams/1/groups HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Basic YWRtaW46YWRtaW4= +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +[ + { + "orgId": 1, + "teamId": 1, + "groupId": "cn=editors,ou=groups,dc=grafana,dc=org" + } +] +``` + +Status Codes: + +- **200** - Ok +- **401** - Unauthorized +- **403** - Permission denied + +## Add External Group + +`POST /api/teams/:teamId/groups` + +**Example Request**: + +```http +POST /api/teams/1/members HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Basic YWRtaW46YWRtaW4= + +{ + "groupId": "cn=editors,ou=groups,dc=grafana,dc=org" +} +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{"message":"Group added to Team"} +``` + +Status Codes: + +- **200** - Ok +- **400** - Group is already added to this team +- **401** - Unauthorized +- **403** - Permission denied +- **404** - Team not found + +## Remove External Group + +`DELETE /api/teams/:teamId/groups/:groupId` + +**Example Request**: + +```http +DELETE /api/teams/1/groups/cn=editors,ou=groups,dc=grafana,dc=org HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Basic YWRtaW46YWRtaW4= +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{"message":"Team Group removed"} +``` + +Status Codes: + +- **200** - Ok +- **401** - Unauthorized +- **403** - Permission denied +- **404** - Team not found/Group not found diff --git a/docs/sources/http_api/folder.md b/docs/sources/http_api/folder.md index 7ee1f737799..e8845c3b125 100644 --- a/docs/sources/http_api/folder.md +++ b/docs/sources/http_api/folder.md @@ -19,6 +19,10 @@ The unique identifier (uid) of a folder can be used for uniquely identify folder The uid can have a maximum length of 40 characters. +## A note about the General folder + +The General folder (id=0) is special and is not part of the Folder API which means +that you cannot use this API for retrieving information about the General folder. ## Get all folders @@ -219,7 +223,7 @@ Status Codes: - **404** – Folder not found - **412** – Precondition failed -The **412** status code is used for explaing that you cannot update the folder and why. +The **412** status code is used for explaining that you cannot update the folder and why. There can be different reasons for this: - The folder has been changed by someone else, `status=version-mismatch` @@ -273,14 +277,14 @@ Status Codes: ## Get folder by id -`GET /api/folders/:id` +`GET /api/folders/id/:id` Will return the folder identified by id. **Example Request**: ```http -GET /api/folders/1 HTTP/1.1 +GET /api/folders/id/1 HTTP/1.1 Accept: application/json Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk diff --git a/docs/sources/http_api/org.md b/docs/sources/http_api/org.md index 4c1dff904c8..c55107d42f8 100644 --- a/docs/sources/http_api/org.md +++ b/docs/sources/http_api/org.md @@ -12,7 +12,13 @@ parent = "http_api" # Organisation API -## Get current Organisation +The Organisation HTTP API is divided in two resources, `/api/org` (current organisation) +and `/api/orgs` (admin organisations). One big difference between these are that +the admin of all organisations API only works with basic authentication, see [Admin Organisations API](#admin-organisations-api) for more information. + +## Current Organisation API + +### Get current Organisation `GET /api/org/` @@ -37,135 +43,7 @@ Content-Type: application/json } ``` -## Get Organisation by Id - -`GET /api/orgs/:orgId` - -**Example Request**: - -```http -GET /api/orgs/1 HTTP/1.1 -Accept: application/json -Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk -``` -Note: The api will only work when you pass the admin name and password -to the request http url, like http://admin:admin@localhost:3000/api/orgs/1 - -**Example Response**: - -```http -HTTP/1.1 200 -Content-Type: application/json - -{ - "id":1, - "name":"Main Org.", - "address":{ - "address1":"", - "address2":"", - "city":"", - "zipCode":"", - "state":"", - "country":"" - } -} -``` -## Get Organisation by Name - -`GET /api/orgs/name/:orgName` - -**Example Request**: - -```http -GET /api/orgs/name/Main%20Org%2E HTTP/1.1 -Accept: application/json -Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk -``` -Note: The api will only work when you pass the admin name and password -to the request http url, like http://admin:admin@localhost:3000/api/orgs/name/Main%20Org%2E - -**Example Response**: - -```http -HTTP/1.1 200 -Content-Type: application/json - -{ - "id":1, - "name":"Main Org.", - "address":{ - "address1":"", - "address2":"", - "city":"", - "zipCode":"", - "state":"", - "country":"" - } -} -``` - -## Create Organisation - -`POST /api/orgs` - -**Example Request**: - -```http -POST /api/orgs HTTP/1.1 -Accept: application/json -Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk - -{ - "name":"New Org." -} -``` -Note: The api will work in the following two ways -1) Need to set GF_USERS_ALLOW_ORG_CREATE=true -2) Set the config users.allow_org_create to true in ini file - -**Example Response**: - -```http -HTTP/1.1 200 -Content-Type: application/json - -{ - "orgId":"1", - "message":"Organization created" -} -``` - - -## Update current Organisation - -`PUT /api/org` - -**Example Request**: - -```http -PUT /api/org HTTP/1.1 -Accept: application/json -Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk - -{ - "name":"Main Org." -} -``` - -**Example Response**: - -```http -HTTP/1.1 200 -Content-Type: application/json - -{"message":"Organization updated"} -``` - -## Get all users within the actual organisation +### Get all users within the current organisation `GET /api/org/users` @@ -195,36 +73,7 @@ Content-Type: application/json ] ``` -## Add a new user to the actual organisation - -`POST /api/org/users` - -Adds a global user to the actual organisation. - -**Example Request**: - -```http -POST /api/org/users HTTP/1.1 -Accept: application/json -Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk - -{ - "role": "Admin", - "loginOrEmail": "admin" -} -``` - -**Example Response**: - -```http -HTTP/1.1 200 -Content-Type: application/json - -{"message":"User added to organization"} -``` - -## Updates the given user +### Updates the given user `PATCH /api/org/users/:userId` @@ -250,7 +99,7 @@ Content-Type: application/json {"message":"Organization user updated"} ``` -## Delete user in actual organisation +### Delete user in current organisation `DELETE /api/org/users/:userId` @@ -272,19 +121,181 @@ Content-Type: application/json {"message":"User removed from organization"} ``` -# Organisations +### Update current Organisation -## Search all Organisations +`PUT /api/org` + +**Example Request**: + +```http +PUT /api/org HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{ + "name":"Main Org." +} +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{"message":"Organization updated"} +``` + +### Add a new user to the current organisation + +`POST /api/org/users` + +Adds a global user to the current organisation. + +**Example Request**: + +```http +POST /api/org/users HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{ + "role": "Admin", + "loginOrEmail": "admin" +} +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{"message":"User added to organization"} +``` + +## Admin Organisations API + +The Admin Organisations HTTP API does not currently work with an API Token. API Tokens are currently +only linked to an organization and an organization role. They cannot be given the permission of server +admin, only users can be given that permission. So in order to use these API calls you will have to +use Basic Auth and the Grafana user must have the Grafana Admin permission (The default admin user +is called `admin` and has permission to use this API). + +### Get Organisation by Id + +`GET /api/orgs/:orgId` + +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + +**Example Request**: + +```http +GET /api/orgs/1 HTTP/1.1 +Accept: application/json +Content-Type: application/json +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "id":1, + "name":"Main Org.", + "address":{ + "address1":"", + "address2":"", + "city":"", + "zipCode":"", + "state":"", + "country":"" + } +} +``` +### Get Organisation by Name + +`GET /api/orgs/name/:orgName` + +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + +**Example Request**: + +```http +GET /api/orgs/name/Main%20Org%2E HTTP/1.1 +Accept: application/json +Content-Type: application/json +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "id":1, + "name":"Main Org.", + "address":{ + "address1":"", + "address2":"", + "city":"", + "zipCode":"", + "state":"", + "country":"" + } +} +``` + +### Create Organisation + +`POST /api/orgs` + +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + +**Example Request**: + +```http +POST /api/orgs HTTP/1.1 +Accept: application/json +Content-Type: application/json + +{ + "name":"New Org." +} +``` +Note: The api will work in the following two ways +1) Need to set GF_USERS_ALLOW_ORG_CREATE=true +2) Set the config users.allow_org_create to true in ini file + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "orgId":"1", + "message":"Organization created" +} +``` + +### Search all Organisations `GET /api/orgs` +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + **Example Request**: ```http GET /api/orgs HTTP/1.1 Accept: application/json Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ``` Note: The api will only work when you pass the admin name and password to the request http url, like http://admin:admin@localhost:3000/api/orgs @@ -303,11 +314,12 @@ Content-Type: application/json ] ``` -## Update Organisation +### Update Organisation `PUT /api/orgs/:orgId` -Update Organisation, fields *Adress 1*, *Adress 2*, *City* are not implemented yet. +Update Organisation, fields *Address 1*, *Address 2*, *City* are not implemented yet. +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). **Example Request**: @@ -315,7 +327,6 @@ Update Organisation, fields *Adress 1*, *Adress 2*, *City* are not implemented y PUT /api/orgs/1 HTTP/1.1 Accept: application/json Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk { "name":"Main Org 2." @@ -331,17 +342,40 @@ Content-Type: application/json {"message":"Organization updated"} ``` -## Get Users in Organisation +### Delete Organisation + +`DELETE /api/orgs/:orgId` + +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + +**Example Request**: + +```http +DELETE /api/orgs/1 HTTP/1.1 +Accept: application/json +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{"message":"Organization deleted"} +``` + +### Get Users in Organisation `GET /api/orgs/:orgId/users` +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + **Example Request**: ```http GET /api/orgs/1/users HTTP/1.1 Accept: application/json Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ``` Note: The api will only work when you pass the admin name and password to the request http url, like http://admin:admin@localhost:3000/api/orgs/1/users @@ -363,17 +397,18 @@ Content-Type: application/json ] ``` -## Add User in Organisation +### Add User in Organisation `POST /api/orgs/:orgId/users` +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + **Example Request**: ```http POST /api/orgs/1/users HTTP/1.1 Accept: application/json Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk { "loginOrEmail":"user", @@ -390,17 +425,18 @@ Content-Type: application/json {"message":"User added to organization"} ``` -## Update Users in Organisation +### Update Users in Organisation `PATCH /api/orgs/:orgId/users/:userId` +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + **Example Request**: ```http PATCH /api/orgs/1/users/2 HTTP/1.1 Accept: application/json Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk { "role":"Admin" @@ -416,17 +452,18 @@ Content-Type: application/json {"message":"Organization user updated"} ``` -## Delete User in Organisation +### Delete User in Organisation `DELETE /api/orgs/:orgId/users/:userId` +Only works with Basic Authentication (username and password), see [introduction](#admin-organisations-api). + **Example Request**: ```http DELETE /api/orgs/1/users/2 HTTP/1.1 Accept: application/json Content-Type: application/json -Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ``` **Example Response**: @@ -436,4 +473,4 @@ HTTP/1.1 200 Content-Type: application/json {"message":"User removed from organization"} -``` \ No newline at end of file +``` diff --git a/docs/sources/http_api/playlist.md b/docs/sources/http_api/playlist.md new file mode 100644 index 00000000000..7c33900969b --- /dev/null +++ b/docs/sources/http_api/playlist.md @@ -0,0 +1,286 @@ ++++ +title = "Playlist HTTP API " +description = "Playlist Admin HTTP API" +keywords = ["grafana", "http", "documentation", "api", "playlist"] +aliases = ["/http_api/playlist/"] +type = "docs" +[menu.docs] +name = "Playlist" +parent = "http_api" ++++ + +# Playlist API + +## Search Playlist + +`GET /api/playlists` + +Get all existing playlist for the current organization using pagination + +**Example Request**: + +```bash +GET /api/playlists HTTP/1.1 +Accept: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + + Querystring Parameters: + + These parameters are used as querystring parameters. + + - **query** - Limit response to playlist having a name like this value. + - **limit** - Limit response to *X* number of playlist. + +**Example Response**: + +```json +HTTP/1.1 200 +Content-Type: application/json +[ + { + "id": 1, + "name": "my playlist", + "interval": "5m" + } +] +``` + +## Get one playlist + +`GET /api/playlists/:id` + +**Example Request**: + +```bash +GET /api/playlists/1 HTTP/1.1 +Accept: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```json +HTTP/1.1 200 +Content-Type: application/json +{ + "id" : 1, + "name": "my playlist", + "interval": "5m", + "orgId": "my org", + "items": [ + { + "id": 1, + "playlistId": 1, + "type": "dashboard_by_id", + "value": "3", + "order": 1, + "title":"my third dasboard" + }, + { + "id": 2, + "playlistId": 1, + "type": "dashboard_by_tag", + "value": "myTag", + "order": 2, + "title":"my other dasboard" + } + ] +} +``` + +## Get Playlist items + +`GET /api/playlists/:id/items` + +**Example Request**: + +```bash +GET /api/playlists/1/items HTTP/1.1 +Accept: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```json +HTTP/1.1 200 +Content-Type: application/json +[ + { + "id": 1, + "playlistId": 1, + "type": "dashboard_by_id", + "value": "3", + "order": 1, + "title":"my third dasboard" + }, + { + "id": 2, + "playlistId": 1, + "type": "dashboard_by_tag", + "value": "myTag", + "order": 2, + "title":"my other dasboard" + } +] +``` + +## Get Playlist dashboards + +`GET /api/playlists/:id/dashboards` + +**Example Request**: + +```bash +GET /api/playlists/1/dashboards HTTP/1.1 +Accept: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```json +HTTP/1.1 200 +Content-Type: application/json +[ + { + "id": 3, + "title": "my third dasboard", + "order": 1, + }, + { + "id": 5, + "title":"my other dasboard" + "order": 2, + + } +] +``` + +## Create a playlist + +`POST /api/playlists/` + +**Example Request**: + +```bash +PUT /api/playlists/1 HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + { + "name": "my playlist", + "interval": "5m", + "items": [ + { + "type": "dashboard_by_id", + "value": "3", + "order": 1, + "title":"my third dasboard" + }, + { + "type": "dashboard_by_tag", + "value": "myTag", + "order": 2, + "title":"my other dasboard" + } + ] + } +``` + +**Example Response**: + +```json +HTTP/1.1 200 +Content-Type: application/json + { + "id": 1, + "name": "my playlist", + "interval": "5m" + } +``` + +## Update a playlist + +`PUT /api/playlists/:id` + +**Example Request**: + +```bash +PUT /api/playlists/1 HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + { + "name": "my playlist", + "interval": "5m", + "items": [ + { + "playlistId": 1, + "type": "dashboard_by_id", + "value": "3", + "order": 1, + "title":"my third dasboard" + }, + { + "playlistId": 1, + "type": "dashboard_by_tag", + "value": "myTag", + "order": 2, + "title":"my other dasboard" + } + ] + } +``` + +**Example Response**: + +```json +HTTP/1.1 200 +Content-Type: application/json +{ + "id" : 1, + "name": "my playlist", + "interval": "5m", + "orgId": "my org", + "items": [ + { + "id": 1, + "playlistId": 1, + "type": "dashboard_by_id", + "value": "3", + "order": 1, + "title":"my third dasboard" + }, + { + "id": 2, + "playlistId": 1, + "type": "dashboard_by_tag", + "value": "myTag", + "order": 2, + "title":"my other dasboard" + } + ] +} +``` + +## Delete a playlist + +`DELETE /api/playlists/:id` + +**Example Request**: + +```bash +DELETE /api/playlists/1 HTTP/1.1 +Accept: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```json +HTTP/1.1 200 +Content-Type: application/json +{} +``` diff --git a/docs/sources/http_api/snapshot.md b/docs/sources/http_api/snapshot.md index dce3b0a9160..5a76d0118b3 100644 --- a/docs/sources/http_api/snapshot.md +++ b/docs/sources/http_api/snapshot.md @@ -70,7 +70,7 @@ JSON Body schema: Content-Type: application/json { "deleteKey":"XXXXXXX", - "deleteUrl":"myurl/dashboard/snapshot/XXXXXXX", + "deleteUrl":"myurl/api/snapshots-delete/XXXXXXX", "key":"YYYYYYY", "url":"myurl/dashboard/snapshot/YYYYYYY" } @@ -81,7 +81,46 @@ Keys: - **deleteKey** – Key generated to delete the snapshot - **key** – Key generated to share the dashboard -## Get Snapshot by Id +## Get list of Snapshots + +`GET /api/dashboard/snapshots` + +Query parameters: + +- **query** – Search Query +- **limit** – Limit the number of returned results + +**Example Request**: + +```http +GET /api/dashboard/snapshots HTTP/1.1 +Accept: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +[ + { + "id":8, + "name":"Home", + "key":"YYYYYYY", + "orgId":1, + "userId":1, + "external":false, + "externalUrl":"", + "expires":"2200-13-32T25:23:23+02:00", + "created":"2200-13-32T28:24:23+02:00", + "updated":"2200-13-32T28:24:23+02:00" + } +] +``` + +## Get Snapshot by Key `GET /api/snapshots/:key` @@ -90,7 +129,6 @@ Keys: ```http GET /api/snapshots/YYYYYYY HTTP/1.1 Accept: application/json -Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ``` @@ -140,16 +178,15 @@ Content-Type: application/json } ``` -## Delete Snapshot by deleteKey +## Delete Snapshot by Key -`GET /api/snapshots-delete/:deleteKey` +`DELETE /api/snapshots/:key` **Example Request**: ```http -GET /api/snapshots/YYYYYYY HTTP/1.1 +DELETE /api/snapshots/YYYYYYY HTTP/1.1 Accept: application/json -Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk ``` @@ -159,5 +196,27 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk HTTP/1.1 200 Content-Type: application/json -{"message":"Snapshot deleted. It might take an hour before it's cleared from a CDN cache."} +{"message":"Snapshot deleted. It might take an hour before it's cleared from any CDN caches."} +``` + +## Delete Snapshot by deleteKey + +This API call can be used without authentication by using the secret delete key for the snapshot. + +`GET /api/snapshots-delete/:deleteKey` + +**Example Request**: + +```http +GET /api/snapshots-delete/XXXXXXX HTTP/1.1 +Accept: application/json +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{"message":"Snapshot deleted. It might take an hour before it's cleared from any CDN caches."} ``` \ No newline at end of file diff --git a/docs/sources/http_api/team.md b/docs/sources/http_api/team.md index 94ea4108481..5024ec69522 100644 --- a/docs/sources/http_api/team.md +++ b/docs/sources/http_api/team.md @@ -30,7 +30,7 @@ Authorization: Basic YWRtaW46YWRtaW4= ### Using the query parameter -Default value for the `perpage` parameter is `1000` and for the `page` parameter is `1`. +Default value for the `perpage` parameter is `1000` and for the `page` parameter is `1`. The `totalCount` field in the response can be used for pagination of the teams list E.g. if `totalCount` is equal to 100 teams and the `perpage` parameter is set to 10 then there are 10 pages of teams. @@ -314,3 +314,67 @@ Status Codes: - **401** - Unauthorized - **403** - Permission denied - **404** - Team not found/Team member not found + +## Get Team Preferences + +`GET /api/teams/:teamId/preferences` + +**Example Request**: + +```http +GET /api/teams/2/preferences HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +{ + "theme": "", + "homeDashboardId": 0, + "timezone": "" +} +``` + +## Update Team Preferences + +`PUT /api/teams/:teamId/preferences` + +**Example Request**: + +```http +PUT /api/teams/2/preferences HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{ + "theme": "dark", + "homeDashboardId": 39, + "timezone": "utc" +} +``` + +JSON Body Schema: + +- **theme** - One of: ``light``, ``dark``, or an empty string for the default theme +- **homeDashboardId** - The numerical ``:id`` of a dashboard, default: ``0`` +- **timezone** - One of: ``utc``, ``browser``, or an empty string for the default + +Omitting a key will cause the current value to be replaced with the system default value. + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: text/plain; charset=utf-8 + +{ + "message":"Preferences updated" +} +``` diff --git a/docs/sources/http_api/user.md b/docs/sources/http_api/user.md index 134c1842851..b9047187b2d 100644 --- a/docs/sources/http_api/user.md +++ b/docs/sources/http_api/user.md @@ -363,6 +363,39 @@ Content-Type: application/json ] ``` +## Teams that the actual User is member of + +`GET /api/user/teams` + +Return a list of all teams that the current user is member of. + +**Example Request**: + +```http +GET /api/user/teams HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +``` + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json + +[ + { + "id": 1, + "orgId": 1, + "name": "MyTestTeam", + "email": "", + "avatarUrl": "\/avatar\/3f49c15916554246daa714b9bd0ee398", + "memberCount": 1 + } +] +``` + ## Star a dashboard `POST /api/user/stars/dashboard/:dashboardId` diff --git a/docs/sources/index.md b/docs/sources/index.md index 3c59b9baba0..e9a900d93f1 100644 --- a/docs/sources/index.md +++ b/docs/sources/index.md @@ -60,9 +60,9 @@ aliases = ["v1.1", "guides/reference/admin"]

Provisioning

A guide to help you automate your Grafana setup & configuration.

- }}" class="nav-cards__item nav-cards__item--guide"> -

What's new in v5.0

-

Article on all the new cool features and enhancements in v5.0

+ }}" class="nav-cards__item nav-cards__item--guide"> +

What's new in v5.3

+

Article on all the new cool features and enhancements in v5.3

 }}" class="nav-cards__item nav-cards__item--guide">

Screencasts

@@ -88,9 +88,13 @@ aliases = ["v1.1", "guides/reference/admin"]
Prometheus
 - }}" class="nav-cards__item nav-cards__item--ds"> - -
OpenTSDB
+ }}" class="nav-cards__item nav-cards__item--ds"> + +
Google Stackdriver
+ + }}" class="nav-cards__item nav-cards__item--ds"> + +
Cloudwatch
 }}" class="nav-cards__item nav-cards__item--ds"> @@ -100,8 +104,12 @@ aliases = ["v1.1", "guides/reference/admin"]
Postgres
 - }}" class="nav-cards__item nav-cards__item--ds"> - -
Cloudwatch
+ }}" class="nav-cards__item nav-cards__item--ds"> + +
Microsoft SQL Server
+ + }}" class="nav-cards__item nav-cards__item--ds"> + +
OpenTSDB
 diff --git a/docs/sources/installation/behind_proxy.md b/docs/sources/installation/behind_proxy.md index f1a00a5b1cc..6e3884456ac 100644 --- a/docs/sources/installation/behind_proxy.md +++ b/docs/sources/installation/behind_proxy.md @@ -26,7 +26,7 @@ Otherwise Grafana will not behave correctly. See example below. ## Examples Here are some example configurations for running Grafana behind a reverse proxy. -### Grafana configuration (ex http://foo.bar.com) +### Grafana configuration (ex http://foo.bar) ```bash [server] @@ -47,13 +47,13 @@ server { } ``` -### Examples with **sub path** (ex http://foo.bar.com/grafana) +### Examples with **sub path** (ex http://foo.bar/grafana) #### Grafana configuration with sub path ```bash [server] domain = foo.bar -root_url = %(protocol)s://%(domain)s:/grafana +root_url = %(protocol)s://%(domain)s/grafana/ ``` #### Nginx configuration with sub path @@ -98,7 +98,7 @@ Given: ```bash [server] domain = localhost:8080 - root_url = %(protocol)s://%(domain)s:/grafana + root_url = %(protocol)s://%(domain)s/grafana/ ``` Create an Inbound Rule for the parent website (localhost:8080 in this example) in IIS Manager with the following settings: diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 66072a98f84..8d156e739bf 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -15,6 +15,8 @@ weight = 1 The Grafana back-end has a number of configuration options that can be specified in a `.ini` configuration file or specified using environment variables. +> **Note.** Grafana needs to be restarted for any configuration changes to take effect. + ## Comments In .ini Files Semicolons (the `;` char) are the standard way to comment out lines in a `.ini` file. @@ -80,6 +82,11 @@ Path to where Grafana stores the sqlite3 database (if used), file based sessions (if used), and other data. This path is usually specified via command line in the init.d script or the systemd service file. +### temp_data_lifetime + +How long temporary images in `data` directory should be kept. Defaults to: `24h`. Supported modifiers: `h` (hours), +`m` (minutes), for example: `168h`, `30m`, `10h30m`. Use `0` to never clean up temporary files. + ### logs Path to where Grafana will store logs. This path is usually specified via @@ -93,8 +100,6 @@ Directory where grafana will automatically scan and look for plugins ### provisioning -> This feature is available in 5.0+ - Folder that contains [provisioning](/administration/provisioning) config files that grafana will apply on startup. Dashboards will be reloaded when the json files changes ## [server] @@ -122,10 +127,13 @@ Another way is put a webserver like Nginx or Apache in front of Grafana and have ### protocol -`http` or `https` +`http`,`https` or `socket` > **Note** Grafana versions earlier than 3.0 are vulnerable to [POODLE](https://en.wikipedia.org/wiki/POODLE). So we strongly recommend to upgrade to 3.x or use a reverse proxy for ssl termination. +### socket +Path where the socket should be created when `protocol=socket`. Please make sure that Grafana has appropriate permissions. + ### domain This setting is only used in as a part of the `root_url` setting (see below). Important if you @@ -176,7 +184,7 @@ embedded database (included in the main Grafana binary). ### url -Use either URL or or the other fields below to configure the database +Use either URL or the other fields below to configure the database Example: `mysql://user:secret@host:port/database` ### type @@ -190,9 +198,9 @@ will be stored. ### host -Only applicable to MySQL or Postgres. Includes IP or hostname and port. +Only applicable to MySQL or Postgres. Includes IP or hostname and port or in case of unix sockets the path to it. For example, for MySQL running on the same host as Grafana: `host = -127.0.0.1:3306` +127.0.0.1:3306` or with unix sockets: `host = /var/run/mysqld/mysqld.sock` ### name @@ -234,7 +242,12 @@ The maximum number of connections in the idle connection pool. ### max_open_conn The maximum number of open connections to the database. +### conn_max_lifetime + +Sets the maximum amount of time a connection may be reused. The default is 14400 (which means 14400 seconds or 4 hours). For MySQL, this setting should be shorter than the [`wait_timeout`](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_wait_timeout) variable. + ### log_queries + Set to `true` to log the sql calls and execution times.
@@ -256,7 +269,8 @@ The number of days the keep me logged in / remember me cookie lasts. ### secret_key -Used for signing keep me logged in / remember me cookies. +Used for signing some datasource settings like secrets and passwords. Cannot be changed without requiring an update +to datasource settings to re-encode them. ### disable_gravatar @@ -288,6 +302,12 @@ Set to `true` to automatically add new users to the main organization (id 1). When set to `false`, new users will automatically cause a new organization to be created for that new user. +### auto_assign_org_id + +Set this value to automatically add new users to the provided org. +This requires `auto_assign_org` to be set to `true`. Please make sure +that this organization does already exists. + ### auto_assign_org_role The role new users will be assigned for the main organization (if the @@ -305,356 +325,17 @@ Defaults to `false`. ## [auth] -### disable_login_form - -Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false. - -### disable_signout_menu - -Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false. - -
- -## [auth.anonymous] - -### enabled - -Set to `true` to enable anonymous access. Defaults to `false` - -### org_name - -Set the organization name that should be used for anonymous users. If -you change your organization name in the Grafana UI this setting needs -to be updated to match the new name. - -### org_role - -Specify role for anonymous users. Defaults to `Viewer`, other valid -options are `Editor` and `Admin`. - -## [auth.github] - -You need to create a GitHub OAuth application (you find this under the GitHub -settings page). When you create the application you will need to specify -a callback URL. Specify this as callback: - -```bash -http://:/login/github -``` - -This callback URL must match the full HTTP address that you use in your -browser to access Grafana, but with the prefix path of `/login/github`. -When the GitHub OAuth application is created you will get a Client ID and a -Client Secret. Specify these in the Grafana configuration file. For -example: - -```bash -[auth.github] -enabled = true -allow_sign_up = true -client_id = YOUR_GITHUB_APP_CLIENT_ID -client_secret = YOUR_GITHUB_APP_CLIENT_SECRET -scopes = user:email,read:org -auth_url = https://github.com/login/oauth/authorize -token_url = https://github.com/login/oauth/access_token -api_url = https://api.github.com/user -team_ids = -allowed_organizations = -``` - -Restart the Grafana back-end. You should now see a GitHub login button -on the login page. You can now login or sign up with your GitHub -accounts. - -You may allow users to sign-up via GitHub authentication by setting the -`allow_sign_up` option to `true`. When this option is set to `true`, any -user successfully authenticating via GitHub authentication will be -automatically signed up. - -### team_ids - -Require an active team membership for at least one of the given teams on -GitHub. If the authenticated user isn't a member of at least one of the -teams they will not be able to register or authenticate with your -Grafana instance. For example: - -```bash -[auth.github] -enabled = true -client_id = YOUR_GITHUB_APP_CLIENT_ID -client_secret = YOUR_GITHUB_APP_CLIENT_SECRET -scopes = user:email,read:org -team_ids = 150,300 -auth_url = https://github.com/login/oauth/authorize -token_url = https://github.com/login/oauth/access_token -api_url = https://api.github.com/user -allow_sign_up = true -``` - -### allowed_organizations - -Require an active organization membership for at least one of the given -organizations on GitHub. If the authenticated user isn't a member of at least -one of the organizations they will not be able to register or authenticate with -your Grafana instance. For example - -```bash -[auth.github] -enabled = true -client_id = YOUR_GITHUB_APP_CLIENT_ID -client_secret = YOUR_GITHUB_APP_CLIENT_SECRET -scopes = user:email,read:org -auth_url = https://github.com/login/oauth/authorize -token_url = https://github.com/login/oauth/access_token -api_url = https://api.github.com/user -allow_sign_up = true -# space-delimited organization names -allowed_organizations = github google -``` - -
- -## [auth.google] - -You need to create a Google project. You can do this in the [Google -Developer Console](https://console.developers.google.com/project). When -you create the project you will need to specify a callback URL. Specify -this as callback: - -```bash -http://:/login/google -``` - -This callback URL must match the full HTTP address that you use in your -browser to access Grafana, but with the prefix path of `/login/google`. -When the Google project is created you will get a Client ID and a Client -Secret. Specify these in the Grafana configuration file. For example: - -```bash -[auth.google] -enabled = true -client_id = YOUR_GOOGLE_APP_CLIENT_ID -client_secret = YOUR_GOOGLE_APP_CLIENT_SECRET -scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email -auth_url = https://accounts.google.com/o/oauth2/auth -token_url = https://accounts.google.com/o/oauth2/token -allowed_domains = mycompany.com mycompany.org -allow_sign_up = true -``` - -Restart the Grafana back-end. You should now see a Google login button -on the login page. You can now login or sign up with your Google -accounts. The `allowed_domains` option is optional, and domains were separated by space. - -You may allow users to sign-up via Google authentication by setting the -`allow_sign_up` option to `true`. When this option is set to `true`, any -user successfully authenticating via Google authentication will be -automatically signed up. - -## [auth.generic_oauth] - -This option could be used if have your own oauth service. - -This callback URL must match the full HTTP address that you use in your -browser to access Grafana, but with the prefix path of `/login/generic_oauth`. - -```bash -[auth.generic_oauth] -enabled = true -client_id = YOUR_APP_CLIENT_ID -client_secret = YOUR_APP_CLIENT_SECRET -scopes = -auth_url = -token_url = -api_url = -allowed_domains = mycompany.com mycompany.org -allow_sign_up = true -``` - -Set api_url to the resource that returns [OpenID UserInfo](https://connect2id.com/products/server/docs/api/userinfo) compatible information. - -### Set up oauth2 with Okta - -First set up Grafana as an OpenId client "webapplication" in Okta. Then set the Base URIs to `https:///` and set the Login redirect URIs to `https:///login/generic_oauth`. - -Finaly set up the generic oauth module like this: -```bash -[auth.generic_oauth] -name = Okta -enabled = true -scopes = openid profile email -client_id = -client_secret = -auth_url = https:///oauth2/v1/authorize -token_url = https:///oauth2/v1/token -api_url = https:///oauth2/v1/userinfo -``` - -### Set up oauth2 with Bitbucket - -```bash -[auth.generic_oauth] -name = BitBucket -enabled = true -allow_sign_up = true -client_id = -client_secret = -scopes = account email -auth_url = https://bitbucket.org/site/oauth2/authorize -token_url = https://bitbucket.org/site/oauth2/access_token -api_url = https://api.bitbucket.org/2.0/user -team_ids = -allowed_organizations = -``` - -### Set up oauth2 with OneLogin - -1. Create a new Custom Connector with the following settings: - - Name: Grafana - - Sign On Method: OpenID Connect - - Redirect URI: `https:///login/generic_oauth` - - Signing Algorithm: RS256 - - Login URL: `https:///login/generic_oauth` - - then: -2. Add an App to the Grafana Connector: - - Display Name: Grafana - - then: -3. Under the SSO tab on the Grafana App details page you'll find the Client ID and Client Secret. - - Your OneLogin Domain will match the url you use to access OneLogin. - - Configure Grafana as follows: - - ```bash - [auth.generic_oauth] - name = OneLogin - enabled = true - allow_sign_up = true - client_id = - client_secret = - scopes = openid email name - auth_url = https://.onelogin.com/oidc/auth - token_url = https://.onelogin.com/oidc/token - api_url = https://.onelogin.com/oidc/me - team_ids = - allowed_organizations = - ``` - -### Set up oauth2 with Auth0 - -1. Create a new Client in Auth0 - - Name: Grafana - - Type: Regular Web Application - -2. Go to the Settings tab and set: - - Allowed Callback URLs: `https:///login/generic_oauth` - -3. Click Save Changes, then use the values at the top of the page to configure Grafana: - - ```bash - [auth.generic_oauth] - enabled = true - allow_sign_up = true - team_ids = - allowed_organizations = - name = Auth0 - client_id = - client_secret = - scopes = openid profile email - auth_url = https:///authorize - token_url = https:///oauth/token - api_url = https:///userinfo - ``` - -### Set up oauth2 with Azure Active Directory - -1. Log in to portal.azure.com and click "Azure Active Directory" in the side menu, then click the "Properties" sub-menu item. - -2. Copy the "Directory ID", this is needed for setting URLs later - -3. Click "App Registrations" and add a new application registration: - - Name: Grafana - - Application type: Web app / API - - Sign-on URL: `https:///login/generic_oauth` - -4. Click the name of the new application to open the application details page. - -5. Note down the "Application ID", this will be the OAuth client id. - -6. Click "Settings", then click "Keys" and add a new entry under Passwords - - Key Description: Grafana OAuth - - Duration: Never Expires - -7. Click Save then copy the key value, this will be the OAuth client secret. - -8. Configure Grafana as follows: - - ```bash - [auth.generic_oauth] - name = Azure AD - enabled = true - allow_sign_up = true - client_id = - client_secret = - scopes = openid email name - auth_url = https://login.microsoftonline.com//oauth2/authorize - token_url = https://login.microsoftonline.com//oauth2/token - api_url = - team_ids = - allowed_organizations = - ``` - -
- -## [auth.basic] -### enabled -When enabled is `true` (default) the http api will accept basic authentication. - -
- -## [auth.ldap] -### enabled -Set to `true` to enable LDAP integration (default: `false`) - -### config_file -Path to the LDAP specific configuration file (default: `/etc/grafana/ldap.toml`) - -### allow_sign_up - -Allow sign up should almost always be true (default) to allow new Grafana users to be created (if ldap authentication is ok). If set to -false only pre-existing Grafana users will be able to login (if ldap authentication is ok). - -> For details on LDAP Configuration, go to the [LDAP Integration]({{< relref "ldap.md" >}}) page. - -
- -## [auth.proxy] - -This feature allows you to handle authentication in a http reverse proxy. - -### enabled - -Defaults to `false` - -### header_name - -Defaults to X-WEBAUTH-USER - -#### header_property - -Defaults to username but can also be set to email - -### auto_sign_up - -Set to `true` to enable auto sign up of users who do not exist in Grafana DB. Defaults to `true`. - -### whitelist - -Limit where auth proxy requests come from by configuring a list of IP addresses. This can be used to prevent users spoofing the X-WEBAUTH-USER header. - -
+Grafana provides many ways to authenticate users. The docs for authentication has been split in to many different pages +below. + +- [Authentication Overview]({{< relref "auth/overview.md" >}}) (anonymous access options, hide login and more) +- [Google OAuth]({{< relref "auth/google.md" >}}) (auth.google) +- [GitHub OAuth]({{< relref "auth/github.md" >}}) (auth.github) +- [Gitlab OAuth]({{< relref "auth/gitlab.md" >}}) (auth.gitlab) +- [Generic OAuth]({{< relref "auth/generic-oauth.md" >}}) (auth.generic_oauth, okta2, auth0, bitbucket, azure) +- [Basic Authentication]({{< relref "auth/overview.md" >}}) (auth.basic) +- [LDAP Authentication]({{< relref "auth/ldap.md" >}}) (auth.ldap) +- [Auth Proxy]({{< relref "auth/auth-proxy.md" >}}) (auth.proxy) ## [session] @@ -669,9 +350,9 @@ session provider you have configured. - **file:** session file path, e.g. `data/sessions` - **mysql:** go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` -- **postgres:** ex: user=a password=b host=localhost port=5432 dbname=c sslmode=verify-full -- **memcache:** ex: 127.0.0.1:11211 -- **redis:** ex: `addr=127.0.0.1:6379,pool_size=100,prefix=grafana` +- **postgres:** ex: `user=a password=b host=localhost port=5432 dbname=c sslmode=verify-full` +- **memcache:** ex: `127.0.0.1:11211` +- **redis:** ex: `addr=127.0.0.1:6379,pool_size=100,prefix=grafana`. For unix socket, use for example: `network=unix,addr=/var/run/redis/redis.sock,pool_size=100,db=grafana` Postgres valid `sslmode` are `disable`, `require`, `verify-ca`, and `verify-full` (default). @@ -708,7 +389,7 @@ Analytics ID here. By default this feature is disabled. ## [dashboards] -### versions_to_keep (introduced in v5.0) +### versions_to_keep Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1. @@ -837,7 +518,7 @@ Secret key. e.g. AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA Url to where Grafana will send PUT request with images ### public_url -Optional parameter. Url to send to users in notifications, directly appended with the resulting uploaded file name. +Optional parameter. Url to send to users in notifications. If the string contains the sequence ${file}, it will be replaced with the uploaded filename. Otherwise, the file name will be appended to the path part of the url, leaving any query string unchanged. ### username basic auth username @@ -878,3 +559,21 @@ Defaults to true. Set to false to disable alerting engine and hide Alerting from ### execute_alerts Makes it possible to turn off alert rule execution. + +### error_or_timeout +> Available in 5.3 and above + +Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state) + +### nodata_or_nullvalues +> Available in 5.3 and above + +Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) + +# concurrent_render_limit + +> Available in 5.3 and above + +Alert notifications can include images, but rendering many images at the same time can overload the server. +This limit will protect the server from render overloading and make sure notifications are sent out quickly. Default +value is `5`. diff --git a/docs/sources/installation/debian.md b/docs/sources/installation/debian.md index d4d3b05343a..7ed44572533 100644 --- a/docs/sources/installation/debian.md +++ b/docs/sources/installation/debian.md @@ -15,7 +15,9 @@ weight = 1 Description | Download ------------ | ------------- -Stable for Debian-based Linux | [grafana_5.0.3_amd64.deb](https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_5.0.3_amd64.deb) +Stable for Debian-based Linux | [x86-64](https://grafana.com/grafana/download?platform=linux) +Stable for Debian-based Linux | [ARM64](https://grafana.com/grafana/download?platform=arm) +Stable for Debian-based Linux | [ARMv7](https://grafana.com/grafana/download?platform=arm) Read [Upgrading Grafana]({{< relref "installation/upgrading.md" >}}) for tips and guidance on updating an existing installation. @@ -24,9 +26,17 @@ installation. ```bash -wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_5.0.3_amd64.deb +wget sudo apt-get install -y adduser libfontconfig -sudo dpkg -i grafana_5.0.3_amd64.deb +sudo dpkg -i grafana__amd64.deb +``` + +Example: + +```bash +wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_5.1.4_amd64.deb +sudo apt-get install -y adduser libfontconfig +sudo dpkg -i grafana_5.1.4_amd64.deb ``` ## APT Repository @@ -34,7 +44,7 @@ sudo dpkg -i grafana_5.0.3_amd64.deb Add the following line to your `/etc/apt/sources.list` file. ```bash -deb https://packagecloud.io/grafana/stable/debian/ jessie main +deb https://packagecloud.io/grafana/stable/debian/ stretch main ``` Use the above line even if you are on Ubuntu or another Debian version. @@ -42,7 +52,7 @@ There is also a testing repository if you want beta or release candidates. ```bash -deb https://packagecloud.io/grafana/testing/debian/ jessie main +deb https://packagecloud.io/grafana/testing/debian/ stretch main ``` Then add the [Package Cloud](https://packagecloud.io/grafana) key. This @@ -90,6 +100,8 @@ This will start the `grafana-server` process as the `grafana` user, which was created during the package installation. The default HTTP port is `3000` and default user and group is `admin`. +Default login and password `admin`/ `admin` + To configure the Grafana server to start at boot time: ```bash @@ -156,3 +168,8 @@ To configure Grafana add a configuration file named `custom.ini` to the Start Grafana by executing `./bin/grafana-server web`. The `grafana-server` binary needs the working directory to be the root install directory (where the binary and the `public` folder is located). + +## Logging in for the first time + +To run Grafana open your browser and go to http://localhost:3000/. 3000 is the default http port that Grafana listens to if you haven't [configured a different port](/installation/configuration/#http-port). +Then follow the instructions [here](/guides/getting_started/). \ No newline at end of file diff --git a/docs/sources/installation/docker.md b/docs/sources/installation/docker.md index 3ca5ba06638..52353ede8c2 100644 --- a/docs/sources/installation/docker.md +++ b/docs/sources/installation/docker.md @@ -12,37 +12,15 @@ weight = 4 # Installing using Docker -Grafana is very easy to install and run using the offical docker container. +Grafana is very easy to install and run using the official docker container. ```bash $ docker run -d -p 3000:3000 grafana/grafana ``` -All Grafana configuration settings can be defined using environment -variables, this is especially useful when using the above container. - -## Docker volumes & ENV config - -The Docker container exposes two volumes, the sqlite3 database in the -folder `/var/lib/grafana` and configuration files is in `/etc/grafana/` -folder. You can map these volumes to host folders when you start the -container: - -```bash -$ docker run -d -p 3000:3000 \ - -v /var/lib/grafana:/var/lib/grafana \ - -e "GF_SECURITY_ADMIN_PASSWORD=secret" \ - grafana/grafana -``` - -In the above example I map the data folder and sets a configuration option via -an `ENV` instruction. - -See the [docker volumes documentation](https://docs.docker.com/engine/admin/volumes/volumes/) if you want to create a volume to use with the Grafana docker image instead of a bind mount (binding to a directory in the host system). - ## Configuration -All options defined in conf/grafana.ini can be overridden using environment +All options defined in `conf/grafana.ini` can be overridden using environment variables by using the syntax `GF__`. For example: @@ -56,15 +34,47 @@ $ docker run \ grafana/grafana ``` -You can use your own grafana.ini file by using environment variable `GF_PATHS_CONFIG`. - The back-end web server has a number of configuration options. Go to the [Configuration]({{< relref "configuration.md" >}}) page for details on all those options. +> For any changes to `conf/grafana.ini` (or corresponding environment variables) to take effect you need to restart Grafana by restarting the Docker container. + +### Default Paths + +The following settings are hard-coded when launching the Grafana Docker container and can only be overridden using environment variables, not in `conf/grafana.ini`. + +Setting | Default value +----------------------|--------------------------- +GF_PATHS_CONFIG | /etc/grafana/grafana.ini +GF_PATHS_DATA | /var/lib/grafana +GF_PATHS_HOME | /usr/share/grafana +GF_PATHS_LOGS | /var/log/grafana +GF_PATHS_PLUGINS | /var/lib/grafana/plugins +GF_PATHS_PROVISIONING | /etc/grafana/provisioning + +## Running a Specific Version of Grafana + +```bash +# specify right tag, e.g. 5.1.0 - see Docker Hub for available tags +$ docker run \ + -d \ + -p 3000:3000 \ + --name grafana \ + grafana/grafana:5.1.0 +``` + +## Running the master branch + +For every successful build of the master branch we update the `grafana/grafana:master` tag and create a new tag `grafana/grafana-dev:master-` with the hash of the git commit that was built. This means you can always get the latest version of Grafana. + +When running Grafana master in production we **strongly** recommend that you use the `grafana/grafana-dev:master-` tag as that will guarantee that you use a specific version of Grafana instead of whatever was the most recent commit at the time. + +For a list of available tags, check out [grafana/grafana](https://hub.docker.com/r/grafana/grafana/tags/) and [grafana/grafana-dev](https://hub.docker.com/r/grafana/grafana-dev/tags/). + ## Installing Plugins for Grafana -Pass the plugins you want installed to docker with the `GF_INSTALL_PLUGINS` environment variable as a comma separated list. This will pass each plugin name to `grafana-cli plugins install ${plugin}`. +Pass the plugins you want installed to docker with the `GF_INSTALL_PLUGINS` environment variable as a comma separated list. This will pass each plugin name to `grafana-cli plugins install ${plugin}` and install them when Grafana starts. ```bash docker run \ @@ -75,15 +85,37 @@ docker run \ grafana/grafana ``` -## Running a Specific Version of Grafana +## Building a custom Grafana image with pre-installed plugins +In the [grafana-docker](https://github.com/grafana/grafana/tree/master/packaging/docker) there is a folder called `custom/` which includes a `Dockerfile` that can be used to build a custom Grafana image. It accepts `GRAFANA_VERSION` and `GF_INSTALL_PLUGINS` as build arguments. + +Example of how to build and run: ```bash -# specify right tag, e.g. 4.5.2 - see Docker Hub for available tags -$ docker run \ +cd custom +docker build -t grafana:latest-with-plugins \ + --build-arg "GRAFANA_VERSION=latest" \ + --build-arg "GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource" . + +docker run \ -d \ -p 3000:3000 \ - --name grafana \ - grafana/grafana:5.0.2 + --name=grafana \ + grafana:latest-with-plugins +``` + +## Installing Plugins from other sources + +> Only available in Grafana v5.3.1+ + +It's possible to install plugins from custom url:s by specifying the url like this: `GF_INSTALL_PLUGINS=;` + +```bash +docker run \ + -d \ + -p 3000:3000 \ + --name=grafana \ + -e "GF_INSTALL_PLUGINS=http://plugin-domain.com/my-custom-plugin.zip;custom-plugin" \ + grafana/grafana ``` ## Configuring AWS Credentials for CloudWatch Support @@ -108,3 +140,113 @@ Supported variables: - `GF_AWS_${profile}_ACCESS_KEY_ID`: AWS access key ID (required). - `GF_AWS_${profile}_SECRET_ACCESS_KEY`: AWS secret access key (required). - `GF_AWS_${profile}_REGION`: AWS region (optional). + +## Grafana container with persistent storage (recommended) + +```bash +# create a persistent volume for your data in /var/lib/grafana (database and plugins) +docker volume create grafana-storage + +# start grafana +docker run \ + -d \ + -p 3000:3000 \ + --name=grafana \ + -v grafana-storage:/var/lib/grafana \ + grafana/grafana +``` + +## Grafana container using bind mounts + +You may want to run Grafana in Docker but use folders on your host for the database or configuration. When doing so it becomes important to start the container with a user that is able to access and write to the folder you map into the container. + +```bash +mkdir data # creates a folder for your data +ID=$(id -u) # saves your user id in the ID variable + +# starts grafana with your user id and using the data folder +docker run -d --user $ID --volume "$PWD/data:/var/lib/grafana" -p 3000:3000 grafana/grafana:5.1.0 +``` + +## Reading secrets from files (support for Docker Secrets) + +> Only available in Grafana v5.2+. + +It's possible to supply Grafana with configuration through files. This works well with [Docker Secrets](https://docs.docker.com/engine/swarm/secrets/) as the secrets by default gets mapped into `/run/secrets/` of the container. + +You can do this with any of the configuration options in conf/grafana.ini by setting `GF____FILE` to the path of the file holding the secret. + +Let's say you want to set the admin password this way. + +- Admin password secret: `/run/secrets/admin_password` +- Environment variable: `GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/admin_password` + + +## Migration from a previous version of the docker container to 5.1 or later + +The docker container for Grafana has seen a major rewrite for 5.1. + +**Important changes** + +* file ownership is no longer modified during startup with `chown` +* default user id `472` instead of `104` +* no more implicit volumes + - `/var/lib/grafana` + - `/etc/grafana` + - `/var/log/grafana` + +### Removal of implicit volumes + +Previously `/var/lib/grafana`, `/etc/grafana` and `/var/log/grafana` were defined as volumes in the `Dockerfile`. This led to the creation of three volumes each time a new instance of the Grafana container started, whether you wanted it or not. + +You should always be careful to define your own named volume for storage, but if you depended on these volumes you should be aware that an upgraded container will no longer have them. + +**Warning**: when migrating from an earlier version to 5.1 or later using docker compose and implicit volumes you need to use `docker inspect` to find out which volumes your container is mapped to so that you can map them to the upgraded container as well. You will also have to change file ownership (or user) as documented below. + +### User ID changes + +In 5.1 we switched the id of the grafana user. Unfortunately this means that files created prior to 5.1 won't have the correct permissions for later versions. We made this change so that it would be more likely that the grafana users id would be unique to Grafana. For example, on Ubuntu 16.04 `104` is already in use by the syslog user. + +Version | User | User ID +--------|---------|--------- +< 5.1 | grafana | 104 +>= 5.1 | grafana | 472 + +There are two possible solutions to this problem. Either you start the new container as the root user and change ownership from `104` to `472` or you start the upgraded container as user `104`. + +#### Running docker as a different user + +```bash +docker run --user 104 --volume "" grafana/grafana:5.1.0 +``` + +##### Specifying a user in docker-compose.yml +```yaml +version: "2" + +services: + grafana: + image: grafana/grafana:5.1.0 + ports: + - 3000:3000 + user: "104" +``` + +#### Modifying permissions + +The commands below will run bash inside the Grafana container with your volume mapped in. This makes it possible to modify the file ownership to match the new container. Always be careful when modifying permissions. + +```bash +$ docker run -ti --user root --volume "" --entrypoint bash grafana/grafana:5.1.0 + +# in the container you just started: +chown -R root:root /etc/grafana && \ + chmod -R a+r /etc/grafana && \ + chown -R grafana:grafana /var/lib/grafana && \ + chown -R grafana:grafana /usr/share/grafana +``` + +## Logging in for the first time + +To run Grafana open your browser and go to http://localhost:3000/. 3000 is the default http port that Grafana listens to if you haven't [configured a different port](/installation/configuration/#http-port). +Then follow the instructions [here](/guides/getting_started/). \ No newline at end of file diff --git a/docs/sources/installation/ldap.md b/docs/sources/installation/ldap.md deleted file mode 100644 index 85501e51d85..00000000000 --- a/docs/sources/installation/ldap.md +++ /dev/null @@ -1,139 +0,0 @@ -+++ -title = "LDAP Authentication" -description = "Grafana LDAP Authentication Guide " -keywords = ["grafana", "configuration", "documentation", "ldap"] -type = "docs" -[menu.docs] -name = "LDAP Authentication" -identifier = "ldap" -parent = "admin" -weight = 2 -+++ - -# LDAP Authentication - -Grafana (2.1 and newer) ships with a strong LDAP integration feature. The LDAP integration in Grafana allows your -Grafana users to login with their LDAP credentials. You can also specify mappings between LDAP -group memberships and Grafana Organization user roles. - -## Configuration -You turn on LDAP in the [main config file]({{< relref "configuration.md#auth-ldap" >}}) as well as specify the path to the LDAP -specific configuration file (default: `/etc/grafana/ldap.toml`). - -### Example config - -```toml -# Set to true to log user information returned from LDAP -verbose_logging = false - -[[servers]] -# Ldap server host (specify multiple hosts space separated) -host = "127.0.0.1" -# Default port is 389 or 636 if use_ssl = true -port = 389 -# Set to true if ldap server supports TLS -use_ssl = false -# Set to true if connect ldap server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS) -start_tls = false -# set to true if you want to skip ssl cert validation -ssl_skip_verify = false -# set to the path to your root CA certificate or leave unset to use system defaults -# root_ca_cert = "/path/to/certificate.crt" - -# Search user bind dn -bind_dn = "cn=admin,dc=grafana,dc=org" -# Search user bind password -# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" -bind_password = 'grafana' - -# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)" -search_filter = "(cn=%s)" - -# An array of base dns to search through -search_base_dns = ["dc=grafana,dc=org"] - -# In POSIX LDAP schemas, without memberOf attribute a secondary query must be made for groups. -# This is done by enabling group_search_filter below. You must also set member_of= "cn" -# in [servers.attributes] below. - -## Group search filter, to retrieve the groups of which the user is a member (only set if memberOf attribute is not available) -# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" -## An array of the base DNs to search through for groups. Typically uses ou=groups -# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] - -# Specify names of the ldap attributes your ldap uses -[servers.attributes] -name = "givenName" -surname = "sn" -username = "cn" -member_of = "memberOf" -email = "email" - -# Map ldap groups to grafana org roles -[[servers.group_mappings]] -group_dn = "cn=admins,dc=grafana,dc=org" -org_role = "Admin" -# The Grafana organization database id, optional, if left out the default org (id 1) will be used. Setting this allows for multiple group_dn's to be assigned to the same org_role provided the org_id differs -# org_id = 1 - -[[servers.group_mappings]] -group_dn = "cn=users,dc=grafana,dc=org" -org_role = "Editor" - -[[servers.group_mappings]] -# If you want to match all (or no ldap groups) then you can use wildcard -group_dn = "*" -org_role = "Viewer" - -``` - -## Bind & Bind Password - -By default the configuration expects you to specify a bind DN and bind password. This should be a read only user that can perform LDAP searches. -When the user DN is found a second bind is performed with the user provided username & password (in the normal Grafana login form). - -```bash -bind_dn = "cn=admin,dc=grafana,dc=org" -bind_password = "grafana" -``` - -### Single Bind Example - -If you can provide a single bind expression that matches all possible users, you can skip the second bind and bind against the user DN directly. -This allows you to not specify a bind_password in the configuration file. - -```bash -bind_dn = "cn=%s,o=users,dc=grafana,dc=org" -``` - -In this case you skip providing a `bind_password` and instead provide a `bind_dn` value with a `%s` somewhere. This will be replaced with the username entered in on the Grafana login page. -The search filter and search bases settings are still needed to perform the LDAP search to retrieve the other LDAP information (like LDAP groups and email). - -## POSIX schema (no memberOf attribute) -If your ldap server does not support the memberOf attribute add these options: - -```toml -## Group search filter, to retrieve the groups of which the user is a member (only set if memberOf attribute is not available) -group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" -## An array of the base DNs to search through for groups. Typically uses ou=groups -group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] -``` - -Also change set `member_of = "cn"` in the `[servers.attributes]` section. - - -## LDAP to Grafana Org Role Sync - -### Mappings -In `[[servers.group_mappings]]` you can map an LDAP group to a Grafana organization -and role. These will be synced every time the user logs in, with LDAP being -the authoritative source. So, if you change a user's role in the Grafana Org. -Users page, this change will be reset the next time the user logs in. If you -change the LDAP groups of a user, the change will take effect the next -time the user logs in. - -### Priority -The first group mapping that an LDAP user is matched to will be used for the sync. If you have LDAP users that fit multiple mappings, the topmost mapping in the TOML config will be used. - - - diff --git a/docs/sources/installation/mac.md b/docs/sources/installation/mac.md index b1d4f18f699..336e46c895d 100644 --- a/docs/sources/installation/mac.md +++ b/docs/sources/installation/mac.md @@ -11,6 +11,8 @@ weight = 4 # Installing on Mac +## Install using homebrew + Installation can be done using [homebrew](http://brew.sh/) Install latest stable: @@ -58,6 +60,8 @@ Then start Grafana using: brew services start grafana ``` +Default login and password `admin`/ `admin` + ### Configuration @@ -75,3 +79,22 @@ If you want to manually install a plugin place it here: `/usr/local/var/lib/graf The default sqlite database is located at `/usr/local/var/lib/grafana` +## Installing from binary tar file + +Download [the latest `.tar.gz` file](https://grafana.com/get) and +extract it. This will extract into a folder named after the version you +downloaded. This folder contains all files required to run Grafana. There are +no init scripts or install scripts in this package. + +To configure Grafana add a configuration file named `custom.ini` to the +`conf` folder and override any of the settings defined in +`conf/defaults.ini`. + +Start Grafana by executing `./bin/grafana-server web`. The `grafana-server` +binary needs the working directory to be the root install directory (where the +binary and the `public` folder is located). + +## Logging in for the first time + +To run Grafana open your browser and go to http://localhost:3000/. 3000 is the default http port that Grafana listens to if you haven't [configured a different port](/installation/configuration/#http-port). +Then follow the instructions [here](/guides/getting_started/). \ No newline at end of file diff --git a/docs/sources/installation/rpm.md b/docs/sources/installation/rpm.md index b0405eb6533..5bf3b7ed745 100644 --- a/docs/sources/installation/rpm.md +++ b/docs/sources/installation/rpm.md @@ -15,34 +15,49 @@ weight = 2 Description | Download ------------ | ------------- -Stable for CentOS / Fedora / OpenSuse / Redhat Linux | [5.0.3 (x86-64 rpm)](https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.0.3-1.x86_64.rpm) +Stable for CentOS / Fedora / OpenSuse / Redhat Linux | [x86-64](https://grafana.com/grafana/download?platform=linux) +Stable for CentOS / Fedora / OpenSuse / Redhat Linux | [ARM64](https://grafana.com/grafana/download?platform=arm) +Stable for CentOS / Fedora / OpenSuse / Redhat Linux | [ARMv7](https://grafana.com/grafana/download?platform=arm) - -Read [Upgrading Grafana]({{< relref "installation/upgrading.md" >}}) for tips and guidance on updating an existing -installation. +Read [Upgrading Grafana]({{< relref "installation/upgrading.md" >}}) for tips and guidance on updating an existing installation. ## Install Stable You can install Grafana using Yum directly. ```bash -$ sudo yum install https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.0.3-1.x86_64.rpm +$ sudo yum install ``` -Or install manually using `rpm`. +Example: -#### On CentOS / Fedora / Redhat: +```bash +$ sudo yum install https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.1.4-1.x86_64.rpm +``` + +Or install manually using `rpm`. First execute + +```bash +$ wget +``` + +Example: + +```bash +$ wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.1.4-1.x86_64.rpm +``` + +### On CentOS / Fedora / Redhat: ```bash -$ wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.0.3-1.x86_64.rpm $ sudo yum install initscripts fontconfig -$ sudo rpm -Uvh grafana-5.0.3-1.x86_64.rpm +$ sudo rpm -Uvh ``` -#### On OpenSuse: +### On OpenSuse: ```bash -$ sudo rpm -i --nodeps grafana-5.0.3-1.x86_64.rpm +$ sudo rpm -i --nodeps ``` ## Install via YUM Repository @@ -52,7 +67,7 @@ Add the following to a new file at `/etc/yum.repos.d/grafana.repo` ```bash [grafana] name=grafana -baseurl=https://packagecloud.io/grafana/stable/el/6/$basearch +baseurl=https://packagecloud.io/grafana/stable/el/7/$basearch repo_gpgcheck=1 enabled=1 gpgcheck=1 @@ -64,7 +79,7 @@ sslcacert=/etc/pki/tls/certs/ca-bundle.crt There is also a testing repository if you want beta or release candidates. ```bash -baseurl=https://packagecloud.io/grafana/testing/el/6/$basearch +baseurl=https://packagecloud.io/grafana/testing/el/7/$basearch ``` Then install Grafana via the `yum` command. @@ -100,6 +115,8 @@ This will start the `grafana-server` process as the `grafana` user, which is created during package installation. The default HTTP port is `3000`, and default user and group is `admin`. +Default login and password `admin`/ `admin` + To configure the Grafana server to start at boot time: ```bash @@ -178,3 +195,7 @@ Start Grafana by executing `./bin/grafana-server web`. The `grafana-server` binary needs the working directory to be the root install directory (where the binary and the `public` folder is located). +## Logging in for the first time + +To run Grafana open your browser and go to http://localhost:3000/. 3000 is the default http port that Grafana listens to if you haven't [configured a different port](/installation/configuration/#http-port). +Then follow the instructions [here](/guides/getting_started/). \ No newline at end of file diff --git a/docs/sources/installation/troubleshooting.md b/docs/sources/installation/troubleshooting.md index 12104c6e826..4b777f3248d 100644 --- a/docs/sources/installation/troubleshooting.md +++ b/docs/sources/installation/troubleshooting.md @@ -21,7 +21,7 @@ the data source response. To check this you should use Query Inspector (new in Grafana v4.5). The query Inspector shows query requests and responses. -For more on the query insector read [this guide here](https://community.grafana.com/t/using-grafanas-query-inspector-to-troubleshoot-issues/2630). For +For more on the query inspector read [this guide here](https://community.grafana.com/t/using-grafanas-query-inspector-to-troubleshoot-issues/2630). For older versions of Grafana read the [how troubleshoot metric query issue](https://community.grafana.com/t/how-to-troubleshoot-metric-query-issues/50/2) article. ## Logging diff --git a/docs/sources/installation/upgrading.md b/docs/sources/installation/upgrading.md index 5b00fd92924..a476a38c3c5 100644 --- a/docs/sources/installation/upgrading.md +++ b/docs/sources/installation/upgrading.md @@ -23,9 +23,9 @@ Before upgrading it can be a good idea to backup your Grafana database. This wil #### sqlite -If you use sqlite you only need to make a backup of you `grafana.db` file. This is usually located at `/var/lib/grafana/grafana.db` on unix system. +If you use sqlite you only need to make a backup of your `grafana.db` file. This is usually located at `/var/lib/grafana/grafana.db` on unix system. If you are unsure what database you use and where it is stored check you grafana configuration file. If you -installed grafana to custom location using a binary tar/zip it is usally in `/data`. +installed grafana to custom location using a binary tar/zip it is usually in `/data`. #### mysql @@ -109,3 +109,11 @@ positioning system when you load them in v5. Dashboards saved in v5 will not wor external panel plugins might need to be updated to work properly. For more details on the new panel positioning system, [click here]({{< relref "reference/dashboard.md#panel-size-position" >}}) + +## Upgrading to v5.2 + +One of the database migrations included in this release will update all annotation timestamps from second to millisecond precision. If you have a large amount of annotations the database migration may take a long time to complete which may cause problems if you use systemd to run Grafana. + +We've got one report where using systemd, PostgreSQL and a large amount of annotations (table size 1645mb) took 8-20 minutes for the database migration to complete. However, the grafana-server process was killed after 90 seconds by systemd. Any database migration queries in progress when systemd kills the grafana-server process continues to execute in database until finished. + +If you're using systemd and have a large amount of annotations consider temporary adjusting the systemd `TimeoutStartSec` setting to something high like `30m` before upgrading. diff --git a/docs/sources/installation/windows.md b/docs/sources/installation/windows.md index 2dac13a6322..b17d625a76e 100644 --- a/docs/sources/installation/windows.md +++ b/docs/sources/installation/windows.md @@ -8,18 +8,19 @@ parent = "installation" weight = 3 +++ - # Installing on Windows Description | Download ------------ | ------------- -Latest stable package for Windows | [grafana-5.0.3.windows-x64.zip](https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-5.0.3.windows-x64.zip) +Latest stable package for Windows | [x64](https://grafana.com/grafana/download?platform=windows) Read [Upgrading Grafana]({{< relref "installation/upgrading.md" >}}) for tips and guidance on updating an existing installation. ## Configure +**Important:** After you've downloaded the zip file and before extracting it, make sure to open properties for that file (right-click Properties) and check the `unblock` checkbox and `Ok`. + The zip file contains a folder with the current Grafana version. Extract this folder to anywhere you want Grafana to run from. Go into the `conf` directory and copy `sample.ini` to `custom.ini`. You should edit @@ -30,6 +31,9 @@ on windows. Edit `custom.ini` and uncomment the `http_port` configuration option (`;` is the comment character in ini files) and change it to something like `8080` or similar. That port should not require extra Windows privileges. +Default login and password `admin`/ `admin` + + Start Grafana by executing `grafana-server.exe`, located in the `bin` directory, preferably from the command line. If you want to run Grafana as windows service, download [NSSM](https://nssm.cc/). It is very easy to add Grafana as a Windows @@ -37,6 +41,11 @@ service using that tool. Read more about the [configuration options]({{< relref "configuration.md" >}}). +## Logging in for the first time + +To run Grafana open your browser and go to the port you configured above, e.g. http://localhost:8080/. +Then follow the instructions [here](/guides/getting_started/). + ## Building on Windows The Grafana backend includes Sqlite3 which requires GCC to compile. So diff --git a/docs/sources/permissions/dashboard_folder_permissions.md b/docs/sources/permissions/dashboard_folder_permissions.md new file mode 100644 index 00000000000..83cb0ee86a3 --- /dev/null +++ b/docs/sources/permissions/dashboard_folder_permissions.md @@ -0,0 +1,73 @@ ++++ +title = "Dashboard & Folder Permissions" +description = "Grafana Dashboard & Folder Permissions Guide " +keywords = ["grafana", "configuration", "documentation", "dashboard", "folder", "permissions", "teams"] +type = "docs" +[menu.docs] +name = "Dashboard & Folder" +identifier = "dashboard-folder-permissions" +parent = "permissions" +weight = 3 ++++ + +# Dashboard & Folder Permissions + +{{< docs-imagebox img="/img/docs/v50/folder_permissions.png" max-width="500px" class="docs-image--right" >}} + +For dashboards and dashboard folders there is a **Permissions** page that make it possible to +remove the default role based permissions for Editors and Viewers. On this page you can add and assign permissions to specific **Users** and **Teams**. + +You can assign & remove permissions for **Organization Roles**, **Users** and **Teams**. + +Permission levels: + +- **Admin**: Can edit & create dashboards and edit permissions. +- **Edit**: Can edit & create dashboards. **Cannot** edit folder/dashboard permissions. +- **View**: Can only view existing dashboards/folders. + +## Restricting Access + +The highest permission always wins so if you for example want to hide a folder or dashboard from others you need to remove the **Organization Role** based permission from the Access Control List (ACL). + +- You cannot override permissions for users with the **Org Admin Role**. Admins always have access to everything. +- A more specific permission with a lower permission level will not have any effect if a more general rule exists with higher permission level. You need to remove or lower the permission level of the more general rule. + +### How Grafana Resolves Multiple Permissions - Examples + +#### Example 1 (`user1` has the Editor Role) + +Permissions for a dashboard: + +- `Everyone with Editor Role Can Edit` +- `user1 Can View` + +Result: `user1` has Edit permission as the highest permission always wins. + +#### Example 2 (`user1` has the Viewer Role and is a member of `team1`) + +Permissions for a dashboard: + +- `Everyone with Viewer Role Can View` +- `user1 Can Edit` +- `team1 Can Admin` + +Result: `user1` has Admin permission as the highest permission always wins. + +#### Example 3 + +Permissions for a dashboard: + +- `user1 Can Admin (inherited from parent folder)` +- `user1 Can Edit` + +Result: You cannot override to a lower permission. `user1` has Admin permission as the highest permission always wins. + +## Summary + +- **View**: Can only view existing dashboards/folders. +- You cannot override permissions for users with **Org Admin Role** +- A more specific permission with lower permission level will not have any effect if a more general rule exists with higher permission level. + +For example if "Everyone with Editor Role Can Edit" exists in the ACL list then **John Doe** will still have Edit permission even after you have specifically added a permission for this user with the permission set to **View**. You need to remove or lower the permission level of the more general rule. +- You cannot override permissions for users with **Org Admin Role** +- A more specific permission with lower permission level will not have any effect if a more general rule exists with higher permission level. For example if "Everyone with Editor Role Can Edit" exists in the ACL list then **John Doe** will still have Edit permission even after you have specifically added a permission for this user with the permission set to **View**. You need to remove or lower the permission level of the more general rule. diff --git a/docs/sources/permissions/datasource_permissions.md b/docs/sources/permissions/datasource_permissions.md new file mode 100644 index 00000000000..ec54c1fbccd --- /dev/null +++ b/docs/sources/permissions/datasource_permissions.md @@ -0,0 +1,71 @@ ++++ +title = "Datasource Permissions" +description = "Grafana Datasource Permissions Guide " +keywords = ["grafana", "configuration", "documentation", "datasource", "permissions", "users", "teams", "enterprise"] +type = "docs" +[menu.docs] +name = "Datasource" +identifier = "datasource-permissions" +parent = "permissions" +weight = 4 ++++ + +# Datasource Permissions + +> Datasource Permissions is only available in Grafana Enterprise. Read more about [Grafana Enterprise]({{< relref "enterprise/index.md" >}}). + +Datasource permissions allows you to restrict access for users to query a datasource. For each datasource there is +a permission page that makes it possible to enable permissions and restrict query permissions to specific +**Users** and **Teams**. + +## Restricting Access - Enable Permissions + +{{< docs-imagebox img="/img/docs/enterprise/datasource_permissions_enable_still.png" class="docs-image--no-shadow docs-image--right" max-width= "600px" animated-gif="/img/docs/enterprise/datasource_permissions_enable.gif" >}} + +By default, permissions are disabled for datasources and a datasource in an organization can be queried by any user in +that organization. For example a user with `Viewer` role can still issue any possible query to a datasource, not just +those queries that exist on dashboards he/she has access to. + +When permissions are enabled for a datasource in an organization you will restrict admin and query access for that +datasource to [admin users](/permissions/organization_roles/#admin-role) in that organization. + +**To enable permissions for a datasource:** + +1. Navigate to Configuration / Data Sources. +2. Select the datasource you want to enable permissions for. +3. Select the Permissions tab and click on the `Enable` button. + +
+ +## Allow users and teams to query a datasource + +{{< docs-imagebox img="/img/docs/enterprise/datasource_permissions_add_still.png" class="docs-image--no-shadow docs-image--right" max-width= "600px" animated-gif="/img/docs/enterprise/datasource_permissions_add.gif" >}} + +After you have [enabled permissions](#restricting-access-enable-permissions) for a datasource you can assign query +permissions to users and teams which will allow access to query the datasource. + +**Assign query permission to users and teams:** + +1. Navigate to Configuration / Data Sources. +2. Select the datasource you want to assign query permissions for. +3. Select the Permissions tab. +4. click on the `Add Permission` button. +5. Select Team/User and find the team/user you want to allow query access and click on the `Save` button. + +
+ +## Restore Default Access - Disable Permissions + +{{< docs-imagebox img="/img/docs/enterprise/datasource_permissions_disable_still.png" class="docs-image--no-shadow docs-image--right" max-width= "600px" animated-gif="/img/docs/enterprise/datasource_permissions_disable.gif" >}} + +If you have enabled permissions for a datasource and want to return datasource permissions to the default, i.e. +datasource can be queried by any user in that organization, you can disable permissions with a click of a button. +Note that all existing permissions created for datasource will be deleted. + +**To disable permissions for a datasource:** + +1. Navigate to Configuration / Data Sources. +2. Select the datasource you want to disable permissions for. +3. Select the Permissions tab and click on the `Disable Permissions` button. + +
diff --git a/docs/sources/permissions/index.md b/docs/sources/permissions/index.md new file mode 100644 index 00000000000..42514f76baf --- /dev/null +++ b/docs/sources/permissions/index.md @@ -0,0 +1,12 @@ ++++ +title = "Permissions" +description = "Permissions" +type = "docs" +[menu.docs] +name = "Permissions" +identifier = "permissions" +parent = "admin" +weight = 3 ++++ + + diff --git a/docs/sources/permissions/organization_roles.md b/docs/sources/permissions/organization_roles.md new file mode 100644 index 00000000000..626d79fad87 --- /dev/null +++ b/docs/sources/permissions/organization_roles.md @@ -0,0 +1,38 @@ ++++ +title = "Organization Roles" +description = "Grafana Organization Roles Guide " +keywords = ["grafana", "configuration", "documentation", "organization", "roles", "permissions"] +type = "docs" +[menu.docs] +name = "Organization Roles" +identifier = "organization-roles" +parent = "permissions" +weight = 2 ++++ + +# Organization Roles + +Users can be belong to one or more organizations. A user's organization membership is tied to a role that defines what the user is allowed to do +in that organization. + +## Admin Role + +Can do everything scoped to the organization. For example: + +- Add & Edit data sources. +- Add & Edit organization users & teams. +- Configure App plugins & set org settings. + +## Editor Role + +- Can create and modify dashboards & alert rules. This can be disabled on specific folders and dashboards. +- **Cannot** create or edit data sources nor invite new users. + +## Viewer Role + +- View any dashboard. This can be disabled on specific folders and dashboards. +- **Cannot** create or edit dashboards nor data sources. + +This role can be tweaked via Grafana server setting [viewers_can_edit]({{< relref "installation/configuration.md#viewers-can-edit" >}}). If you set this to true users +with **Viewer** can also make transient dashboard edits, meaning they can modify panels & queries but not save the changes (nor create new dashboards). +Useful for public Grafana installations where you want anonymous users to be able to edit panels & queries but not save or create new dashboards. diff --git a/docs/sources/permissions/overview.md b/docs/sources/permissions/overview.md new file mode 100644 index 00000000000..cd3fc5417b6 --- /dev/null +++ b/docs/sources/permissions/overview.md @@ -0,0 +1,42 @@ ++++ +title = "Overview" +description = "Overview for permissions" +keywords = ["grafana", "configuration", "documentation", "admin", "users", "datasources", "permissions"] +type = "docs" +aliases = ["/reference/admin", "/administration/permissions/"] +[menu.docs] +name = "Overview" +identifier = "overview-permissions" +parent = "permissions" +weight = 1 ++++ + +# Permissions Overview + +Grafana users have permissions that are determined by their: + +- **Organization Role** (Admin, Editor, Viewer) +- Via **Team** memberships where the **Team** has been assigned specific permissions. +- Via permissions assigned directly to user (on folders, dashboards, datasources) +- The Grafana Admin (i.e. Super Admin) user flag. + +## Grafana Admin + +This admin flag makes a user a `Super Admin`. This means they can access the `Server Admin` views where all users and organizations can be administrated. + +## Organization Roles + +Users can be belong to one or more organizations. A user's organization membership is tied to a role that defines what the user is allowed to do +in that organization. Learn more about [Organization Roles]({{< relref "permissions/organization_roles.md" >}}). + + +## Dashboard & Folder Permissions + +Dashboard and folder permissions allows you to remove the default role based permissions for Editors and Viewers and assign permissions to specific **Users** and **Teams**. Learn more about [Dashboard & Folder Permissions]({{< relref "permissions/dashboard_folder_permissions.md" >}}). + +## Datasource Permissions + +Per default, a datasource in an organization can be queried by any user in that organization. For example a user with `Viewer` role can still +issue any possible query to a data source, not just those queries that exist on dashboards he/she has access to. + +Datasource permissions allows you to change the default permissions for datasources and restrict query permissions to specific **Users** and **Teams**. Read more about [Datasource Permissions]({{< relref "permissions/datasource_permissions.md" >}}). diff --git a/docs/sources/plugins/developing/apps.md b/docs/sources/plugins/developing/apps.md index a3fc35066f6..155f97461c9 100644 --- a/docs/sources/plugins/developing/apps.md +++ b/docs/sources/plugins/developing/apps.md @@ -5,7 +5,7 @@ type = "docs" [menu.docs] name = "Developing App Plugins" parent = "developing" -weight = 6 +weight = 4 +++ # Grafana Apps diff --git a/docs/sources/plugins/developing/auth-for-datasources.md b/docs/sources/plugins/developing/auth-for-datasources.md new file mode 100644 index 00000000000..c03793e745f --- /dev/null +++ b/docs/sources/plugins/developing/auth-for-datasources.md @@ -0,0 +1,99 @@ ++++ +title = "Authentication for Datasource Plugins" +type = "docs" +[menu.docs] +name = "Authentication for Datasource Plugins" +parent = "developing" +weight = 3 ++++ + +# Authentication for Datasource Plugins + +Grafana has a proxy feature that proxies all data requests through the Grafana backend. This is very useful when your datasource plugin calls an external/thirdy-party API. The Grafana proxy adds CORS headers and can authenticate against the external API. This means that a datasource plugin that proxies all requests via Grafana can enable token authentication and the token will be renewed automatically for the user when it expires. + +The plugin config page should save the API key/password to be encrypted (using the `secureJsonData` feature) and then when a request from the datasource is made, the Grafana Proxy will: + + 1. decrypt the API key/password on the backend. + 2. carry out authentication and generate an OAuth token that will be added as an `Authorization` HTTP header to all requests (or it will add a HTTP header with the API key). + 3. renew the token if it expires. + +This means that users that access the datasource config page cannot access the API key or password after is saved the first time and that no secret keys are sent in plain text through the browser where they can be spied on. + +For backend authentication to work, the external/third-party API must either have an OAuth endpoint or that the API accepts an API key as a HTTP header for authentication. + +## Plugin Routes + +You can specify routes in the `plugin.json` file for your datasource plugin. [Here is an example](https://github.com/grafana/azure-monitor-datasource/blob/d74c82145c0a4af07a7e96cc8dde231bfd449bd9/src/plugin.json#L30-L95) with lots of routes (though most plugins will just have one route). + +When you build your url to the third-party API in your datasource class, the url should start with the text specified in the path field for a route. The proxy will strip out the path text and replace it with the value in the url field. + +For example, if my code makes a call to url `azuremonitor/foo/bar` with this code: + +```js +this.backendSrv.datasourceRequest({ + url: url, + method: 'GET', +}) +``` + +and this route: + +```json +"routes": [{ + "path": "azuremonitor", + "method": "GET", + "url": "https://management.azure.com", + ... +}] +``` + +then the Grafana proxy will transform it into "https://management.azure.com/foo/bar" and add CORS headers. + +The `method` parameter is optional. It can be set to any HTTP verb to provide more fine-grained control. + +## Encrypting Sensitive Data + +When a user saves a password or secret with your datasource plugin's Config page, then you can save data to a column in the datasource table called `secureJsonData` that is an encrypted blob. Any data saved in the blob is encrypted by Grafana and can only be decrypted by the Grafana server on the backend. This means once a password is saved, no sensitive data is sent to the browser. If the password is saved in the `jsonData` blob or the `password` field then it is unencrypted and anyone with Admin access (with the help of Chrome Developer Tools) can read it. + +This is an example of using the `secureJsonData` blob to save a property called `password`: + +```html + +``` + +## API Key/HTTP Header Authentication + +Some third-party API's accept a HTTP Header for authentication. The [example](https://github.com/grafana/azure-monitor-datasource/blob/d74c82145c0a4af07a7e96cc8dde231bfd449bd9/src/plugin.json#L91-L93) below has a `headers` section that defines the name of the HTTP Header that the API expects and it uses the `SecureJSONData` blob to fetch an encrypted API key. The Grafana server proxy will decrypt the key, add the `X-API-Key` header to the request and forward it to the third-party API. + +```json +{ + "path": "appinsights", + "method": "GET", + "url": "https://api.applicationinsights.io", + "headers": [ + {"name": "X-API-Key", "content": "{{.SecureJsonData.appInsightsApiKey}}"} + ] +} +``` + +## How Token Authentication Works + +The token auth section in the `plugin.json` file looks like this: + +```json +"tokenAuth": { + "url": "https://login.microsoftonline.com/{{.JsonData.tenantId}}/oauth2/token", + "params": { + "grant_type": "client_credentials", + "client_id": "{{.JsonData.clientId}}", + "client_secret": "{{.SecureJsonData.clientSecret}}", + "resource": "https://management.azure.com/" + } +} +``` + +This interpolates in data from both `jsonData` and `secureJsonData` to generate the token request to the third-party API. It is common for tokens to have a short expiry period (30 minutes). The proxy in Grafana server will automatically renew the token if it has expired. + +## Always Restart the Grafana Server After Route Changes + +The plugin.json files are only loaded when the Grafana server starts so when a route is added or changed then the Grafana server has to be restarted for the changes to take effect. diff --git a/docs/sources/plugins/developing/datasources.md b/docs/sources/plugins/developing/datasources.md index 09a005ba714..f8792441bbd 100644 --- a/docs/sources/plugins/developing/datasources.md +++ b/docs/sources/plugins/developing/datasources.md @@ -5,7 +5,7 @@ type = "docs" [menu.docs] name = "Developing Datasource Plugins" parent = "developing" -weight = 6 +weight = 5 +++ # Datasources @@ -25,7 +25,6 @@ To interact with the rest of grafana the plugins module file can export 5 differ - Datasource (Required) - QueryCtrl (Required) - ConfigCtrl (Required) -- QueryOptionsCtrl - AnnotationsQueryCtrl ## Plugin json @@ -182,12 +181,6 @@ A JavaScript class that will be instantiated and treated as an Angular controlle Requires a static template or templateUrl variable which will be rendered as the view for this controller. -## QueryOptionsCtrl - -A JavaScript class that will be instantiated and treated as an Angular controller when the user edits metrics in a panel. This controller is responsible for handling panel wide settings for the datasource, such as interval, rate and aggregations if needed. - -Requires a static template or templateUrl variable which will be rendered as the view for this controller. - ## AnnotationsQueryCtrl A JavaScript class that will be instantiated and treated as an Angular controller when the user choose this type of datasource in the templating menu in the dashboard. diff --git a/docs/sources/plugins/developing/development.md b/docs/sources/plugins/developing/development.md index f2e70a50c6a..48410b06732 100644 --- a/docs/sources/plugins/developing/development.md +++ b/docs/sources/plugins/developing/development.md @@ -10,7 +10,7 @@ weight = 1 # Developer Guide -You can extend Grafana by writing your own plugins and then share then with other users in [our plugin repository](https://grafana.com/plugins). +You can extend Grafana by writing your own plugins and then share them with other users in [our plugin repository](https://grafana.com/plugins). ## Short version @@ -33,7 +33,7 @@ There are two blog posts about authoring a plugin that might also be of interest ## What languages? Since everything turns into javascript it's up to you to choose which language you want. That said it's probably a good idea to choose es6 or typescript since -we use es6 classes in Grafana. So it's easier to get inspiration from the Grafana repo is you choose one of those languages. +we use es6 classes in Grafana. So it's easier to get inspiration from the Grafana repo if you choose one of those languages. ## Buildscript @@ -60,7 +60,6 @@ and [apps]({{< relref "apps.md" >}}) plugins in the documentation. The Grafana SDK is quite small so far and can be found here: - [SDK file in Grafana](https://github.com/grafana/grafana/blob/master/public/app/plugins/sdk.ts) -- [SDK Readme](https://github.com/grafana/grafana/blob/master/public/app/plugins/plugin_api.md) The SDK contains three different plugin classes: PanelCtrl, MetricsPanelCtrl and QueryCtrl. For plugins of the panel type, the module.js file should export one of these. There are some extra classes for [data sources]({{< relref "datasources.md" >}}). diff --git a/docs/sources/plugins/developing/panels.md b/docs/sources/plugins/developing/panels.md index 26db69c7c94..8670c15e093 100644 --- a/docs/sources/plugins/developing/panels.md +++ b/docs/sources/plugins/developing/panels.md @@ -1,16 +1,11 @@ ---- -page_title: Plugin panel -page_description: Panel plugins for Grafana -page_keywords: grafana, plugins, documentation ---- - - +++ -title = "Installing Plugins" +title = "Developing Panel Plugins" +keywords = ["grafana", "plugins", "panel", "documentation"] type = "docs" [menu.docs] +name = "Developing Panel Plugins" parent = "developing" -weight = 1 +weight = 4 +++ @@ -20,7 +15,21 @@ Panels are the main building blocks of dashboards. ## Panel development -Examples + +### Scrolling +The grafana dashboard framework controls the panel height. To enable a scrollbar within the panel the PanelCtrl needs to set the scrollable static variable: + +```javascript +export class MyPanelCtrl extends PanelCtrl { + static scrollable = true; + ... +``` + +In this case, make sure the template has a single `
...
` root. The plugin loader will modify that element adding a scrollbar. + + + +### Examples - [clock-panel](https://github.com/grafana/clock-panel) - [singlestat-panel](https://github.com/grafana/grafana/blob/master/public/app/plugins/panel/singlestat/module.ts) diff --git a/docs/sources/plugins/developing/plugin-review-guidelines.md b/docs/sources/plugins/developing/plugin-review-guidelines.md new file mode 100644 index 00000000000..8efb023cf64 --- /dev/null +++ b/docs/sources/plugins/developing/plugin-review-guidelines.md @@ -0,0 +1,175 @@ ++++ +title = "Plugin Review Guidelines" +type = "docs" +[menu.docs] +name = "Plugin Review Guidelines" +parent = "developing" +weight = 2 ++++ + +# Plugin Review Guidelines + +The Grafana team reviews all plugins that are published on Grafana.com. There are two areas we review, the metadata for the plugin and the plugin functionality. + +## Metadata + +The plugin metadata consists of a `plugin.json` file and the README.md file. These `plugin.json` file is used by Grafana to load the plugin and the README.md file is shown in the plugins section of Grafana and the plugins section of Grafana.com. + +### README.md + +The README.md file is shown on the plugins page in Grafana and the plugin page on Grafana.com. There are some differences between the GitHub markdown and the markdown allowed in Grafana/Grafana.com: + +- Cannot contain inline HTML. +- Any image links should be absolute links. For example: https://raw.githubusercontent.com/grafana/azure-monitor-datasource/master/dist/img/grafana_cloud_install.png + +The README should: + +- describe the purpose of the plugin. +- contain steps on how to get started. + +### Plugin.json + +The `plugin.json` file is the same concept as the `package.json` file for an npm package. When the Grafana server starts it will scan the plugin folders (all folders in the data/plugins subfolder) and load every folder that contains a `plugin.json` file unless the folder contains a subfolder named `dist`. In that case, the Grafana server will load the `dist` folder instead. + +A minimal `plugin.json` file: + +```json +{ + "type": "panel", + "name": "Clock", + "id": "yourorg-clock-panel", + + "info": { + "description": "Clock panel for grafana", + "author": { + "name": "Author Name", + "url": "http://yourwebsite.com" + }, + "keywords": ["clock", "panel"], + "version": "1.0.0", + "updated": "2018-03-24" + }, + + "dependencies": { + "grafanaVersion": "3.x.x", + "plugins": [ ] + } +} +``` + +- The convention for the plugin id is [github username/org]-[plugin name]-[datasource|app|panel] and it has to be unique. Although if org and plugin name are the same then [plugin name]-[datasource|app|panel] is also valid. The org **cannot** be `grafana` unless it is a plugin created by the Grafana core team. + + Examples: + + - raintank-worldping-app + - ryantxu-ajax-panel + - alexanderzobnin-zabbix-app + - hawkular-datasource + +- The `type` field should be either `datasource` `app` or `panel`. +- The `version` field should be in the form: x.x.x e.g. `1.0.0` or `0.4.1`. + +The full file format for the `plugin.json` file is described [here](http://docs.grafana.org/plugins/developing/plugin.json/). + +## Plugin Language + +JavaScript, TypeScript, ES6 (or any other language) are all fine as long as the contents of the `dist` subdirectory are transpiled to JavaScript (ES5). + +## File and Directory Structure Conventions + +Here is a typical directory structure for a plugin. + +```bash +johnnyb-awesome-datasource +|-- dist +|-- src +| |-- img +| | |-- logo.svg +| |-- partials +| | |-- annotations.editor.html +| | |-- config.html +| | |-- query.editor.html +| |-- datasource.js +| |-- module.js +| |-- plugin.json +| |-- query_ctrl.js +|-- Gruntfile.js +|-- LICENSE +|-- package.json +|-- README.md +``` + +Most JavaScript projects have a build step. The generated JavaScript should be placed in the `dist` directory and the source code in the `src` directory. We recommend that the plugin.json file be placed in the src directory and then copied over to the dist directory when building. The `README.md` can be placed in the root or in the dist directory. + +Directories: + +- `src/` contains plugin source files. +- `src/partials` contains html templates. +- `src/img` contains plugin logos and other images. +- `dist/` contains built content. + +## HTML and CSS + +For the HTML on editor tabs, we recommend using the inbuilt Grafana styles rather than defining your own. This makes plugins feel like a more natural part of Grafana. If done correctly, the html will also be responsive and adapt to smaller screens. The `gf-form` css classes should be used for labels and inputs. + +Below is a minimal example of an editor row with one form group and two fields, a dropdown and a text input: + +```html +
+
+
My Plugin Options
+
+ +
+ +
+
+ + +
+
+
+
+``` + +Use the `width-x` and `max-width-x` classes to control the width of your labels and input fields. Try to get labels and input fields to line up neatly by having the same width for all the labels in a group and the same width for all inputs in a group if possible. + +## Data Sources + +A basic guide for data sources can be found [here](http://docs.grafana.org/plugins/developing/datasources/). + +### Config Page Guidelines + +- It should be as easy as possible for a user to configure a url. If the data source is using the `datasource-http-settings` component, it should use the `suggest-url` attribute to suggest the default url or a url that is similar to what it should be (especially important if the url refers to a REST endpoint that is not common knowledge for most users e.g. `https://yourserver:4000/api/custom-endpoint`). + + ```html + + + ``` + +- The `testDatasource` function should make a query to the data source that will also test that the authentication details are correct. This is so the data source is correctly configured when the user tries to write a query in a new dashboard. + +#### Password Security + +If possible, any passwords or secrets should be be saved in the `secureJsonData` blob. To encrypt sensitive data, the Grafana server's proxy feature must be used. The Grafana server has support for token authentication (OAuth) and HTTP Header authentication. If the calls have to be sent directly from the browser to a third-party API then this will not be possible and sensitive data will not be encrypted. + +Read more here about how [Authentication for Datasources]({{< relref "auth-for-datasources.md" >}}) works. + +If using the proxy feature then the Config page should use the `secureJsonData` blob like this: + + - good: `` + - bad: `` + +### Query Editor + +Each query editor is unique and can have a unique style. It should be adapted to what the users of the data source are used to. + +- Should use the Grafana CSS `gf-form` classes. +- Should be neat and tidy. Labels and fields in columns should be aligned and should be the same width if possible. +- The datasource should be able to handle when a user toggles a query (by clicking on the eye icon) and not execute the query. This is done by checking the `hide` property - an [example](https://github.com/grafana/grafana/blob/master/public/app/plugins/datasource/postgres/datasource.ts#L35-L38). +- Should not execute queries if fields in the Query Editor are empty and the query will throw an exception (defensive programming). +- Should handle errors. There are two main ways to do this: + - use the notification system in Grafana to show a toaster popup with the error message. Example [here](https://github.com/alexanderzobnin/grafana-zabbix/blob/fdbbba2fb03f5f2a4b3b0715415e09d5a4cf6cde/src/panel-triggers/triggers_panel_ctrl.js#L467-L471). + - provide an error notification in the query editor like the MySQL/Postgres data sources do. Example code in the `query_ctrl` [here](https://github.com/grafana/azure-monitor-datasource/blob/b184d077f082a69f962120ef0d1f8296a0d46f03/src/query_ctrl.ts#L36-L51) and in the [html](https://github.com/grafana/azure-monitor-datasource/blob/b184d077f082a69f962120ef0d1f8296a0d46f03/src/partials/query.editor.html#L190-L193). diff --git a/docs/sources/plugins/developing/plugin.json.md b/docs/sources/plugins/developing/plugin.json.md index 7de5e91986f..2d21a665207 100644 --- a/docs/sources/plugins/developing/plugin.json.md +++ b/docs/sources/plugins/developing/plugin.json.md @@ -5,7 +5,7 @@ type = "docs" [menu.docs] name = "plugin.json Schema" parent = "developing" -weight = 6 +weight = 8 +++ # Plugin.json diff --git a/docs/sources/project/building_from_source.md b/docs/sources/project/building_from_source.md index 13d71e8dcf4..eed05f05fa6 100644 --- a/docs/sources/project/building_from_source.md +++ b/docs/sources/project/building_from_source.md @@ -13,7 +13,7 @@ dev environment. Grafana ships with its own required backend server; also comple ## Dependencies -- [Go 1.9.2](https://golang.org/dl/) +- [Go (Latest Stable)](https://golang.org/dl/) - [Git](https://git-scm.com/downloads) - [NodeJS LTS](https://nodejs.org/download/) - node-gyp is the Node.js native addon build tool and it requires extra dependencies: python 2.7, make and GCC. These are already installed for most Linux distros and MacOS. See the Building On Windows section or the [node-gyp installation instructions](https://github.com/nodejs/node-gyp#installation) for more details. @@ -57,7 +57,7 @@ For this you need nodejs (v.6+). ```bash npm install -g yarn yarn install --pure-lockfile -npm run watch +yarn watch ``` ## Running Grafana Locally @@ -66,13 +66,13 @@ You can run a local instance of Grafana by running: ```bash ./bin/grafana-server ``` -If you built the binary with `go run build.go build`, run `./bin/grafana-server` +Or, if you built the binary with `go run build.go build`, run `./bin/-/grafana-server` If you built it with `go build .`, run `./grafana` Open grafana in your browser (default [http://localhost:3000](http://localhost:3000)) and login with admin user (default user/pass = admin/admin). -## Developing Grafana +# Developing Grafana To add features, customize your config, etc, you'll need to rebuild the backend when you change the source code. We use a tool named `bra` that does this. @@ -83,21 +83,18 @@ go get github.com/Unknwon/bra bra run ``` -You'll also need to run `npm run watch` to watch for changes to the front-end (typescript, html, sass) +You'll also need to run `yarn watch` to watch for changes to the front-end (typescript, html, sass) ### Running tests -- You can run backend Golang tests using "go test ./pkg/...". -- Execute all frontend tests with "npm run test" +- You can run backend Golang tests using `go test ./pkg/...`. +- Execute all frontend tests with `yarn test` -Writing & watching frontend tests (we have two test runners) +Writing & watching frontend tests + +- Start watcher: `yarn jest` +- Jest will run all test files that end with the name ".test.ts" -- jest for all new tests that do not require browser context (React+more) - - Start watcher: `npm run jest` - - Jest will run all test files that end with the name ".jest.ts" -- karma + mocha is used for testing angularjs components. We do want to migrate these test to jest over time (if possible). - - Start watcher: `npm run karma` - - Karma+Mocha runs all files that end with the name "_specs.ts". ## Creating optimized release packages @@ -124,7 +121,7 @@ Learn more about Grafana config options in the [Configuration section](/installa ## Create a pull requests Please contribute to the Grafana project and submit a pull request! Build new features, write or update documentation, fix bugs and generally make Grafana even more awesome. -## Troubleshooting +# Troubleshooting **Problem**: PhantomJS or node-sass errors when running grunt @@ -144,3 +141,8 @@ Please contribute to the Grafana project and submit a pull request! Build new fe **Problem**: On Windows, getting errors about a tool not being installed even though you just installed that tool. **Solution**: It is usually because it got added to the path and you have to restart your command prompt to use it. + +## Logging in for the first time + +To run Grafana open your browser and go to the default port http://localhost:3000 or the port you have configured. +Then follow the instructions [here](/guides/getting_started/). \ No newline at end of file diff --git a/docs/sources/reference/annotations.md b/docs/sources/reference/annotations.md index bfc104ef522..8732c8c709b 100644 --- a/docs/sources/reference/annotations.md +++ b/docs/sources/reference/annotations.md @@ -45,8 +45,11 @@ can still show them if you add a new **Annotation Query** and filter by tags. Bu ### Query by tag You can create new annotation queries that fetch annotations from the native annotation store via the `-- Grafana --` data source and by setting *Filter by* to `Tags`. Specify at least -one tag. For example create an annotation query name `outages` and specify a tag named `outage`. This query will show all annotations you create (from any dashboard or via API) that -have the `outage` tag. +one tag. For example create an annotation query name `outages` and specify a tag named `outage`. This query will show all annotations you create (from any dashboard or via API) that have the `outage` tag. By default, if you add multiple tags in the annotation query, Grafana will only show annotations that have all the tags you supplied. You can invert the behavior by enabling `Match any` which means that Grafana will show annotations that contains at least one of the tags you supplied. + +In Grafana v5.3+ it's possible to use template variables in the tag query. So if you have a dashboard showing stats for different services and a template variable that dictates which services to show, you can now use the same template variable in your annotation query to only show annotations for those services. + +{{< docs-imagebox img="/img/docs/v53/annotation_tag_filter_variable.png" max-width="600px" >}} ## Querying other data sources diff --git a/docs/sources/reference/dashboard.md b/docs/sources/reference/dashboard.md index dbc3ed8635c..6be12600da5 100644 --- a/docs/sources/reference/dashboard.md +++ b/docs/sources/reference/dashboard.md @@ -50,6 +50,7 @@ When a user creates a new dashboard, a new dashboard JSON object is initialized "annotations": { "list": [] }, + "refresh": "5s", "schemaVersion": 16, "version": 0, "links": [] @@ -71,13 +72,14 @@ Each field in the dashboard JSON is explained below with its usage: | **timepicker** | timepicker metadata, see [timepicker section](#timepicker) for details | | **templating** | templating metadata, see [templating section](#templating) for details | | **annotations** | annotations metadata, see [annotations section](#annotations) for details | -| **schemaVersion** | version of the JSON schema (integer), incremented each time a Grafana update brings changes to the said schema | +| **refresh** | auto-refresh interval +| **schemaVersion** | version of the JSON schema (integer), incremented each time a Grafana update brings changes to said schema | | **version** | version of the dashboard (integer), incremented each time the dashboard is updated | | **panels** | panels array, see below for detail. | ## Panels -Panels are the building blocks a dashboard. It consists of datasource queries, type of graphs, aliases, etc. Panel JSON consists of an array of JSON objects, each representing a different panel. Most of the fields are common for all panels but some fields depends on the panel type. Following is an example of panel JSON of a text panel. +Panels are the building blocks of a dashboard. It consists of datasource queries, type of graphs, aliases, etc. Panel JSON consists of an array of JSON objects, each representing a different panel. Most of the fields are common for all panels but some fields depend on the panel type. Following is an example of panel JSON of a text panel. ```json "panels": [ @@ -105,7 +107,7 @@ The gridPos property describes the panel size and position in grid coordinates. - `x` The x position, in same unit as `w`. - `y` The y position, in same unit as `h`. -The grid has a negative gravity that moves panels up if there i empty space above a panel. +The grid has a negative gravity that moves panels up if there is empty space above a panel. ### timepicker @@ -161,7 +163,7 @@ Usage of the fields is explained below: ### templating -`templating` fields contains array of template variables with their saved values along with some other metadata, for example: +The `templating` field contains an array of template variables with their saved values along with some other metadata, for example: ```json "templating": { @@ -236,7 +238,7 @@ Usage of the above mentioned fields in the templating section is explained below | Name | Usage | | ---- | ----- | | **enable** | whether templating is enabled or not | -| **list** | an array of objects representing, each representing one template variable | +| **list** | an array of objects each representing one template variable | | **allFormat** | format to use while fetching all values from datasource, eg: `wildcard`, `glob`, `regex`, `pipe`, etc. | | **current** | shows current selected variable text/value on the dashboard | | **datasource** | shows datasource for the variables | diff --git a/docs/sources/reference/playlist.md b/docs/sources/reference/playlist.md index 5a6bf921334..182e69eebd0 100644 --- a/docs/sources/reference/playlist.md +++ b/docs/sources/reference/playlist.md @@ -49,7 +49,7 @@ Click the back button to rewind to the previous Dashboard in the Playlist. In TV mode the top navbar, row & panel controls will all fade to transparent. This happens automatically after one minute of user inactivity but can also be toggled manually -with the `d v` sequence shortcut. Any mouse movement or keyboard action will +with the `d v` sequence shortcut, or by appending the parameter `?inactive` to the dashboard URL. Any mouse movement or keyboard action will restore navbar & controls. Another feature is the kiosk mode - in kiosk mode the navbar is completely hidden/removed from view. This can be enabled with the `d k` diff --git a/docs/sources/reference/scripting.md b/docs/sources/reference/scripting.md index 551805b567a..12ab91f3c3c 100644 --- a/docs/sources/reference/scripting.md +++ b/docs/sources/reference/scripting.md @@ -12,7 +12,7 @@ weight = 9 If you have lots of metric names that change (new servers etc) in a defined pattern it is irritating to constantly have to create new dashboards. -With scripted dashboards you can dynamically create your dashboards using javascript. In the folder grafana install folder +With scripted dashboards you can dynamically create your dashboards using javascript. In the grafana install folder under `public/dashboards/` there is a file named `scripted.js`. This file contains an example of a scripted dashboard. You can access it by using the url: `http://grafana_url/dashboard/script/scripted.js?rows=3&name=myName` @@ -21,42 +21,32 @@ If you open scripted.js you can see how it reads url parameters from ARGS variab ## Example ```javascript -var rows = 1; var seriesName = 'argName'; -if(!_.isUndefined(ARGS.rows)) { - rows = parseInt(ARGS.rows, 10); -} - if(!_.isUndefined(ARGS.name)) { seriesName = ARGS.name; } -for (var i = 0; i < rows; i++) { - - dashboard.rows.push({ - title: 'Scripted Graph ' + i, - height: '300px', - panels: [ - { - title: 'Events', - type: 'graph', - span: 12, - fill: 1, - linewidth: 2, - targets: [ - { - 'target': "randomWalk('" + seriesName + "')" - }, - { - 'target': "randomWalk('random walk2')" - } - ], - } - ] - }); - -} +dashboard.panels.push({ + title: 'Events', + type: 'graph', + fill: 1, + linewidth: 2, + gridPos: { + h: 10, + w: 24, + x: 0, + y: 10, + }, + targets: [ + { + 'target': "randomWalk('" + seriesName + "')" + }, + { + 'target': "randomWalk('random walk2')" + } + ] +}); return dashboard; ``` diff --git a/docs/sources/reference/templating.md b/docs/sources/reference/templating.md index 3a15b4ed7d1..403dabba8ae 100644 --- a/docs/sources/reference/templating.md +++ b/docs/sources/reference/templating.md @@ -1,6 +1,6 @@ +++ title = "Variables" -keywords = ["grafana", "templating", "documentation", "guide"] +keywords = ["grafana", "templating", "documentation", "guide", "template", "variable"] type = "docs" [menu.docs] name = "Variables" @@ -11,7 +11,7 @@ weight = 1 # Variables Variables allows for more interactive and dynamic dashboards. Instead of hard-coding things like server, application -and sensor name in you metric queries you can use variables in their place. Variables are shown as dropdown select boxes at the top of +and sensor name in your metric queries you can use variables in their place. Variables are shown as dropdown select boxes at the top of the dashboard. These dropdowns make it easy to change the data being displayed in your dashboard. {{< docs-imagebox img="/img/docs/v50/variables_dashboard.png" >}} @@ -36,6 +36,29 @@ interpolation the variable value might be **escaped** in order to conform to the For example, a variable used in a regex expression in an InfluxDB or Prometheus query will be regex escaped. Read the data source specific documentation article for details on value escaping during interpolation. +### Advanced Formatting Options + +> Only available in Grafana v5.1+. + +The formatting of the variable interpolation depends on the data source but there are some situations where you might want to change the default formatting. For example, the default for the MySql datasource is to join multiple values as comma-separated with quotes: `'server01','server02'`. In some cases you might want to have a comma-separated string without quotes: `server01,server02`. This is now possible with the advanced formatting options. + +Syntax: `${var_name:option}` + +Filter Option | Example | Raw | Interpolated | Description +------------ | ------------- | ------------- | ------------- | ------------- +`glob` | ${servers:glob} | `'test1', 'test2'` | `{test1,test2}` | (Default) Formats multi-value variable into a glob (for Graphite queries) +`regex` | ${servers:regex} | `'test.', 'test2'` | (test\.|test2) | Formats multi-value variable into a regex string +`pipe` | ${servers:pipe} | `'test.', 'test2'` | test.|test2 | Formats multi-value variable into a pipe-separated string +`csv`| ${servers:csv} | `'test1', 'test2'` | `test1,test2` | Formats multi-value variable as a comma-separated string +`distributed`| ${servers:distributed} | `'test1', 'test2'` | `test1,servers=test2` | Formats multi-value variable in custom format for OpenTSDB. +`lucene`| ${servers:lucene} | `'test', 'test2'` | `("test" OR "test2")` | Formats multi-value variable as a lucene expression. + +Test the formatting options on the [Grafana Play site](http://play.grafana.org/d/cJtIfcWiz/template-variable-formatting-options?orgId=1). + +If any invalid formatting option is specified, then `glob` is the default/fallback option. + +An alternative syntax (that might be deprecated in the future) is `[[var_name:option]]`. + ### Variable options A variable is presented as a dropdown select box at the top of the dashboard. It has a current value and a set of **options**. The **options** @@ -67,6 +90,7 @@ Type | Description *Custom* | Define the variable options manually using a comma separated list. *Constant* | Define a hidden constant. Useful for metric path prefixes for dashboards you want to share. During dashboard export, constant variables will be made into an import option. *Ad hoc filters* | Very special kind of variable that only works with some data sources, InfluxDB & Elasticsearch currently. It allows you to add key/value filters that will automatically be added to all metric queries that use the specified data source. +*Text box* | This variable type will display as a free text input field with an optional default value. ### Query options @@ -80,6 +104,73 @@ Option | Description *Regex* | Regex to filter or capture specific parts of the names return by your data source query. Optional. *Sort* | Define sort order for options in dropdown. **Disabled** means that the order of options returned by your data source query will be used. +#### Using regex to filter/modify values in the Variable dropdown + +Using the Regex Query Option, you filter the list of options returned by the Variable query or modify the options returned. + +Examples of filtering on the following list of options: + +```text +backend_01 +backend_02 +backend_03 +backend_04 +``` + +##### Filter so that only the options that end with `01` or `02` are returned: + +Regex: + +```regex +/.*[01|02]/ +``` + +Result: + +```text +backend_01 +backend_02 +``` + +##### Filter and modify the options using a regex capture group to return part of the text: + +Regex: + +```regex +/.*(01|02)/ +``` + +Result: + +```text +01 +02 +``` + +#### Filter and modify - Prometheus Example + +List of options: + +```text +up{instance="demo.robustperception.io:9090",job="prometheus"} 1 1521630638000 +up{instance="demo.robustperception.io:9093",job="alertmanager"} 1 1521630638000 +up{instance="demo.robustperception.io:9100",job="node"} 1 1521630638000 +``` + +Regex: + +```regex +/.*instance="([^"]*).*/ +``` + +Result: + +```text +demo.robustperception.io:9090 +demo.robustperception.io:9093 +demo.robustperception.io:9100 +``` + ### Query expressions The query expressions are different for each data source. @@ -99,14 +190,16 @@ Option | Description ------- | -------- *Multi-value* | If enabled, the variable will support the selection of multiple options at the same time. *Include All option* | Add a special `All` option whose value includes all options. -*Custom all value* | By default the `All` value will include all options in combined expression. This can become very long and can have performance problems. Many times it can be better to specify a custom all value, like a wildcard regex. To make it possible to have custom regex, globs or lucene syntax in the **Custom all value** option it is never escaped so you will have to think avbout what is a valid value for your data source. +*Custom all value* | By default the `All` value will include all options in combined expression. This can become very long and can have performance problems. Many times it can be better to specify a custom all value, like a wildcard regex. To make it possible to have custom regex, globs or lucene syntax in the **Custom all value** option it is never escaped so you will have to think about what is a valid value for your data source. -### Formating multiple values +### Formatting multiple values Interpolating a variable with multiple values selected is tricky as it is not straight forward how to format the multiple values to into a string that is valid in the given context where the variable is used. Grafana tries to solve this by allowing each data source plugin to inform the templating interpolation engine what format to use for multiple values. +Note that the *Custom all value* option on the variable will have to be left blank for Grafana to format all values into a single string. + **Graphite**, for example, uses glob expressions. A variable with multiple values would, in this case, be interpolated as `{host1,host2,host3}` if the current variable value was *host1*, *host2* and *host3*. @@ -117,7 +210,7 @@ break the regex expression. **Elasticsearch** uses lucene query syntax, so the same variable would, in this case, be formatted as `("host1" OR "host2" OR "host3")`. In this case every value needs to be escaped so that the value can contain lucene control words and quotation marks. -#### Formating troubles +#### Formatting troubles Automatic escaping & formatting can cause problems and it can be tricky to grasp the logic is behind it. Especially for InfluxDB and Prometheus where the use of regex syntax requires that the variable is used in regex operator context. @@ -153,7 +246,7 @@ Grafana has global built-in variables that can be used in expressions in the que ### The $__interval Variable -This $__interval variable is similar to the `auto` interval variable that is described above. It can be used as a parameter to group by time (for InfluxDB), Date histogram interval (for Elasticsearch) or as a *summarize* function parameter (for Graphite). +This $__interval variable is similar to the `auto` interval variable that is described above. It can be used as a parameter to group by time (for InfluxDB, MySQL, Postgres, MSSQL), Date histogram interval (for Elasticsearch) or as a *summarize* function parameter (for Graphite). Grafana automatically calculates an interval that can be used to group by time in queries. When there are more data points than can be shown on a graph then queries can be made more efficient by grouping by a larger interval. It is more efficient to group by 1 day than by 10s when looking at 3 months of data and the graph will look the same and the query will be faster. The `$__interval` is calculated using the time range and the width of the graph (the number of pixels). @@ -181,31 +274,50 @@ The `$__timeFilter` is used in the MySQL data source. This variable is only available in the Singlestat panel and can be used in the prefix or suffix fields on the Options tab. The variable will be replaced with the series name or alias. +### The $__range Variable + +> Only available in Grafana v5.3+ + +Currently only supported for Prometheus data sources. This variable represents the range for the current dashboard. It is calculated by `to - from`. It has a millisecond and a second representation called `$__range_ms` and `$__range_s`. + ## Repeating Panels Template variables can be very useful to dynamically change your queries across a whole dashboard. If you want Grafana to dynamically create new panels or rows based on what values you have selected you can use the *Repeat* feature. -If you have a variable with `Multi-value` or `Include all value` options enabled you can choose one panel or one row and have Grafana repeat that row -for every selected value. You find this option under the General tab in panel edit mode. Select the variable to repeat by, and a `min span`. -The `min span` controls how small Grafana will make the panels (if you have many values selected). Grafana will automatically adjust the width of -each repeated panel so that the whole row is filled. Currently, you cannot mix other panels on a row with a repeated panel. +If you have a variable with `Multi-value` or `Include all value` options enabled you can choose one panel and have Grafana repeat that panel +for every selected value. You find the *Repeat* feature under the *General tab* in panel edit mode. + +The `direction` controls how the panels will be arranged. + +By choosing `horizontal` the panels will be arranged side-by-side. Grafana will automatically adjust the width +of each repeated panel so that the whole row is filled. Currently, you cannot mix other panels on a row with a repeated +panel. Each panel will never be smaller that the provided `Min width` if you have many selected values. + +By choosing `vertical` the panels will be arranged from top to bottom in a column. The `Min width` doesn't have any effect in this case. The width of the repeated panels will be the same as of the first panel (the original template) being repeated. Only make changes to the first panel (the original template). To have the changes take effect on all panels you need to trigger a dynamic dashboard re-build. You can do this by either changing the variable value (that is the basis for the repeat) or reload the dashboard. ## Repeating Rows -This option requires you to open the row options view. Hover over the row left side to trigger the row menu, in this menu click `Row Options`. This -opens the row options view. Here you find a *Repeat* dropdown where you can select the variable to repeat by. +As seen above with the *Panels* you can also repeat *Rows* if you have variables set with `Multi-value` or +`Include all value` selection option. -### URL state +To enable this feature you need to first add a new *Row* using the *Add Panel* menu. Then by hovering the row title and +clicking on the cog button, you will access the `Row Options` configuration panel. You can then select the variable +you want to repeat the row for. + +It may be a good idea to use a variable in the row title as well. + +Example: [Repeated Rows Dashboard](http://play.grafana.org/dashboard/db/repeated-rows) + +## URL state Variable values are always synced to the URL using the syntax `var-=value`. -### Examples +## Examples - [Graphite Templated Dashboard](http://play.grafana.org/dashboard/db/graphite-templated-nested) - [Elasticsearch Templated Dashboard](http://play.grafana.org/dashboard/db/elasticsearch-templated) - [InfluxDB Templated Dashboard](http://play.grafana.org/dashboard/db/influxdb-templated-queries) - diff --git a/docs/sources/tutorials/ha_setup.md b/docs/sources/tutorials/ha_setup.md index 9ae2989f6e6..f141392e223 100644 --- a/docs/sources/tutorials/ha_setup.md +++ b/docs/sources/tutorials/ha_setup.md @@ -22,13 +22,13 @@ Setting up Grafana for high availability is fairly simple. It comes down to two First, you need to do is to setup MySQL or Postgres on another server and configure Grafana to use that database. You can find the configuration for doing that in the [[database]]({{< relref "configuration.md" >}}#database) section in the grafana config. -Grafana will now persist all long term data in the database. How to configure the database for high availability is out of scope for this guide. We recommend finding an expert on for the database your using. +Grafana will now persist all long term data in the database. How to configure the database for high availability is out of scope for this guide. We recommend finding an expert on for the database you're using. ## User sessions -The second thing to consider is how to deal with user sessions and how to configure your load balancer infront of Grafana. -Grafana support two says of storing session data locally on disk or in a database/cache-server. -If you want to store sessions on disk you can use `sticky sessions` in your load balanacer. If you prefer to store session data in a database/cache-server +The second thing to consider is how to deal with user sessions and how to configure your load balancer in front of Grafana. +Grafana supports two ways of storing session data: locally on disk or in a database/cache-server. +If you want to store sessions on disk you can use `sticky sessions` in your load balancer. If you prefer to store session data in a database/cache-server you can use any stateless routing strategy in your load balancer (ex round robin or least connections). ### Sticky sessions diff --git a/docs/sources/tutorials/iis.md b/docs/sources/tutorials/iis.md index 63a41d67c16..896181c4a9f 100644 --- a/docs/sources/tutorials/iis.md +++ b/docs/sources/tutorials/iis.md @@ -16,7 +16,7 @@ Example: - Parent site: http://localhost:8080 - Grafana: http://localhost:3000 -Grafana as a subpath: http://localhost:8080/grafana +Grafana as a subpath: http://localhost:8080/grafana ## Setup @@ -33,7 +33,7 @@ Given that the subpath should be `grafana` and the parent site is `localhost:808 ```bash [server] domain = localhost:8080 -root_url = %(protocol)s://%(domain)s:/grafana +root_url = %(protocol)s://%(domain)s/grafana/ ``` Restart the Grafana server after changing the config file. @@ -74,11 +74,11 @@ When navigating to the grafana url (`http://localhost:8080/grafana` in the examp 1. The `root_url` setting in the Grafana config file does not match the parent url with subpath. This could happen if the root_url is commented out by mistake (`;` is used for commenting out a line in .ini files): - `; root_url = %(protocol)s://%(domain)s:/grafana` + `; root_url = %(protocol)s://%(domain)s/grafana/` 2. or if the subpath in the `root_url` setting does not match the subpath used in the pattern in the Inbound Rule in IIS: - `root_url = %(protocol)s://%(domain)s:/grafana` + `root_url = %(protocol)s://%(domain)s/grafana/` pattern in Inbound Rule: `wrongsubpath(/)?(.*)` diff --git a/docs/sources/tutorials/index.md b/docs/sources/tutorials/index.md index cb11940c6dd..90410e901d3 100644 --- a/docs/sources/tutorials/index.md +++ b/docs/sources/tutorials/index.md @@ -1,5 +1,6 @@ +++ title = "Tutorials" +type = "docs" [menu.docs] identifier = "tutorials" weight = 6 @@ -11,7 +12,11 @@ This section of the docs contains a series for tutorials and stack setup guides. ## Articles -- [How to integrate Hubot with Grafana](hubot_howto.md) +- [Running Grafana behind a reverse proxy]({{< relref "behind_proxy.md" >}}) +- [API Tutorial: How To Create API Tokens And Dashboards For A Specific Organization]({{< relref "api_org_token_howto.md" >}}) +- [How to Use IIS with URL Rewrite as a Reverse Proxy for Grafana on Windows]({{< relref "iis.md" >}}) +- [How to integrate Hubot with Grafana]({{< relref "hubot_howto.md" >}}) +- [How to setup Grafana for high availability]({{< relref "ha_setup.md" >}}) ## External links diff --git a/docs/sources/tutorials/screencasts.md b/docs/sources/tutorials/screencasts.md index e92a07c51a7..882544e7318 100644 --- a/docs/sources/tutorials/screencasts.md +++ b/docs/sources/tutorials/screencasts.md @@ -94,7 +94,7 @@ weight = 10
- #3 Whats New In Grafana 2.0 + #3 What's New In Grafana 2.0
diff --git a/docs/sources/whatsnew/index.md b/docs/sources/whatsnew/index.md index df472f07093..f4159643d72 100644 --- a/docs/sources/whatsnew/index.md +++ b/docs/sources/whatsnew/index.md @@ -3,7 +3,7 @@ title = "What's New in Grafana" [menu.docs] name = "What's New In Grafana" identifier = "whatsnew" -weight = 3 +weight = 5 +++ diff --git a/docs/versions.json b/docs/versions.json index 2dcc7ebe776..48962a783ae 100644 --- a/docs/versions.json +++ b/docs/versions.json @@ -1,6 +1,8 @@ [ - { "version": "v5.1", "path": "/v5.1", "archived": false }, - { "version": "v5.0", "path": "/", "archived": false, "current": true }, + { "version": "v5.3", "path": "/", "archived": false, "current": true }, + { "version": "v5.2", "path": "/v5.2", "archived": true }, + { "version": "v5.1", "path": "/v5.1", "archived": true }, + { "version": "v5.0", "path": "/v5.0", "archived": true }, { "version": "v4.6", "path": "/v4.6", "archived": true }, { "version": "v4.5", "path": "/v4.5", "archived": true }, { "version": "v4.4", "path": "/v4.4", "archived": true }, diff --git a/emails/templates/layouts/default.html b/emails/templates/layouts/default.html index 07eb32874c7..ca54acdd206 100644 --- a/emails/templates/layouts/default.html +++ b/emails/templates/layouts/default.html @@ -143,7 +143,7 @@ td[class="stack-column-center"] {

Sent by Grafana v[[.BuildVersion]] -
© 2016 Grafana and raintank +
© 2018 Grafana Labs

diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 75f1f9a9a86..00000000000 --- a/examples/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Example plugin implementations - -datasource:[simple-json-datasource](https://github.com/grafana/simple-json-datasource) -app: [example-app](https://github.com/grafana/example-app) -panel: [grafana-piechart-panel](https://github.com/grafana/piechart-panel) diff --git a/examples/alerting-dashboard.json b/examples/alerting-dashboard.json deleted file mode 100644 index 744460d7847..00000000000 --- a/examples/alerting-dashboard.json +++ /dev/null @@ -1,800 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_GRAPHITE", - "label": "graphite", - "description": "", - "type": "datasource", - "pluginId": "graphite", - "pluginName": "Graphite" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "graphite", - "name": "Graphite", - "version": "1.0.0" - } - ], - "id": null, - "title": "Alerting example", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 355 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Critical alert panel", - "notifications": [], - "severity": "critical" - }, - "alerting": {}, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - }, - { - "refId": "B", - "target": "aliasByNode(scale(statsd.$apa.counters.session_start.*.count, 10), 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 355 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 20 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Warning panel alert", - "notifications": [], - "severity": "warning" - }, - "alerting": {}, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "fillColor": "rgba(235, 138, 14, 0.12)", - "line": true, - "lineColor": "rgba(247, 149, 32, 0.60)", - "op": "gt", - "value": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1 - ], - "type": "lt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "count" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "No datapoints", - "notifications": [], - "severity": "critical" - }, - "alerting": {}, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "value": 1, - "op": "lt", - "fill": true, - "line": true, - "colorMode": "critical" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Count datapoints", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "Row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 20 - ], - "type": "lt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Alert below value", - "notifications": [], - "severity": "critical" - }, - "alerting": {}, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 17, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "lt", - "value": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Alert below value", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 10, - 80 - ], - "type": "outside_range" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "10s", - "handler": 1, - "name": "Alert is outside range", - "notifications": [], - "severity": "critical" - }, - "alerting": {}, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 18, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "lt", - "value": 10 - }, - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 80 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Alert is outside range", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 60, - 80 - ], - "type": "within_range" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "10s", - "handler": 1, - "name": "Alert is within range", - "notifications": [], - "severity": "critical" - }, - "alerting": {}, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 19, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 60 - }, - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "lt", - "value": 80 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Alert is within range", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - } - ], - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [ - { - "current": { - "text": "fakesite", - "value": "fakesite" - }, - "datasource": null, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "apa", - "options": [ - { - "selected": true, - "text": "fakesite", - "value": "fakesite" - } - ], - "query": "fakesite", - "refresh": 0, - "type": "custom" - } - ] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 15, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/examples/alerting-multiple-alerts.json b/examples/alerting-multiple-alerts.json deleted file mode 100644 index e6e729ecc06..00000000000 --- a/examples/alerting-multiple-alerts.json +++ /dev/null @@ -1,2216 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_GRAPHITE", - "label": "graphite", - "description": "", - "type": "datasource", - "pluginId": "graphite", - "pluginName": "Graphite" - } - ], - "__requires": [ - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.0" - }, - { - "type": "datasource", - "id": "graphite", - "name": "Graphite", - "version": "1.0.0" - } - ], - "id": null, - "title": "Dashboard with many alerts", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 30 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "sum" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Critical alert panel", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 30 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 30 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "sum" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Critical alert panel", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 30 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 30 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "sum" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Critical alert panel", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 6, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 30 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 30 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "sum" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Critical alert panel", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 30 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "Row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 20 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Warning panel alert", - "notifications": [], - "severity": "warning" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "fillColor": "rgba(235, 138, 14, 0.12)", - "line": true, - "lineColor": "rgba(247, 149, 32, 0.60)", - "op": "gt", - "value": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 20 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Warning panel alert", - "notifications": [], - "severity": "warning" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "fillColor": "rgba(235, 138, 14, 0.12)", - "line": true, - "lineColor": "rgba(247, 149, 32, 0.60)", - "op": "gt", - "value": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 20 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Warning panel alert", - "notifications": [], - "severity": "warning" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "fillColor": "rgba(235, 138, 14, 0.12)", - "line": true, - "lineColor": "rgba(247, 149, 32, 0.60)", - "op": "gt", - "value": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 20 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "60s", - "handler": 1, - "name": "Warning panel alert", - "notifications": [], - "severity": "warning" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 7, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "warning", - "fill": true, - "fillColor": "rgba(235, 138, 14, 0.12)", - "line": true, - "lineColor": "rgba(247, 149, 32, 0.60)", - "op": "gt", - "value": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 50 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "10s", - "handler": 1, - "name": "Fast Critical panel alert", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 9, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 50 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 50 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "10s", - "handler": 1, - "name": "Fast Critical panel alert", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 10, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 50 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 50 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "10s", - "handler": 1, - "name": "Fast Critical panel alert", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 11, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 50 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 50 - ], - "type": "gt" - }, - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "avg" - }, - "type": "query" - } - ], - "enabled": true, - "frequency": "10s", - "handler": 1, - "name": "Fast Critical panel alert", - "notifications": [], - "severity": "critical" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 12, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.12)", - "line": true, - "lineColor": "rgba(237, 46, 24, 0.60)", - "op": "gt", - "value": 50 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Critical panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "alert": { - "enabled": true, - "conditions": [ - { - "type": "query", - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "evaluator": { - "type": "gt", - "params": [ - 10 - ] - } - } - ], - "severity": "warning", - "frequency": "1s", - "handler": 1, - "notifications": [], - "name": "Fast Warning panel alert" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 13, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "value": 10, - "op": "gt", - "fill": true, - "line": true, - "colorMode": "warning", - "fillColor": "rgba(235, 138, 14, 0.12)", - "lineColor": "rgba(247, 149, 32, 0.60)" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "enabled": true, - "conditions": [ - { - "type": "query", - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "evaluator": { - "type": "gt", - "params": [ - 10 - ] - } - } - ], - "severity": "warning", - "frequency": "1s", - "handler": 1, - "notifications": [], - "name": "Fast Warning panel alert" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 14, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "value": 10, - "op": "gt", - "fill": true, - "line": true, - "colorMode": "warning", - "fillColor": "rgba(235, 138, 14, 0.12)", - "lineColor": "rgba(247, 149, 32, 0.60)" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "enabled": true, - "conditions": [ - { - "type": "query", - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "evaluator": { - "type": "gt", - "params": [ - 10 - ] - } - } - ], - "severity": "warning", - "frequency": "1s", - "handler": 1, - "notifications": [], - "name": "Fast Warning panel alert" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 15, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "value": 10, - "op": "gt", - "fill": true, - "line": true, - "colorMode": "warning", - "fillColor": "rgba(235, 138, 14, 0.12)", - "lineColor": "rgba(247, 149, 32, 0.60)" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "alert": { - "enabled": true, - "conditions": [ - { - "type": "query", - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "evaluator": { - "type": "gt", - "params": [ - 10 - ] - } - } - ], - "severity": "warning", - "frequency": "1s", - "handler": 1, - "notifications": [], - "name": "Fast Warning panel alert" - }, - "aliasColors": {}, - "bars": false, - "datasource": "${DS_GRAPHITE}", - "editable": true, - "error": false, - "fill": 1, - "id": 16, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "refId": "A", - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)" - } - ], - "thresholds": [ - { - "value": 10, - "op": "gt", - "fill": true, - "line": true, - "colorMode": "warning", - "fillColor": "rgba(235, 138, 14, 0.12)", - "lineColor": "rgba(247, 149, 32, 0.60)" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Fast Warning panel", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "title": "New row", - "height": "250px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "Alert below value", - "error": false, - "span": 3, - "editable": true, - "type": "graph", - "isNew": true, - "id": 17, - "targets": [ - { - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)", - "refId": "A" - } - ], - "datasource": "${DS_GRAPHITE}", - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - }, - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - } - ], - "xaxis": { - "show": true - }, - "alert": { - "conditions": [ - { - "type": "query", - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "evaluator": { - "type": "lt", - "params": [ - 20 - ] - } - } - ], - "severity": "critical", - "frequency": "60s", - "handler": 1, - "notifications": [], - "name": "Alert below value", - "enabled": true - }, - "lines": true, - "fill": 1, - "linewidth": 2, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true, - "sort": 0, - "msResolution": false - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [ - { - "value": 20, - "op": "lt", - "fill": true, - "line": true, - "colorMode": "critical", - "fillColor": "rgba(234, 112, 112, 0.12)", - "lineColor": "rgba(237, 46, 24, 0.60)" - } - ], - "links": [] - }, - { - "title": "Alert is outside range", - "error": false, - "span": 3, - "editable": true, - "type": "graph", - "isNew": true, - "id": 18, - "targets": [ - { - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)", - "refId": "A" - } - ], - "datasource": "${DS_GRAPHITE}", - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - }, - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - } - ], - "xaxis": { - "show": true - }, - "alert": { - "conditions": [ - { - "type": "query", - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "evaluator": { - "type": "outside_range", - "params": [ - 10, - 80 - ] - } - } - ], - "severity": "critical", - "frequency": "10s", - "handler": 1, - "notifications": [], - "name": "Alert is outside range", - "enabled": true - }, - "lines": true, - "fill": 1, - "linewidth": 2, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true, - "sort": 0, - "msResolution": false - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [ - { - "value": 10, - "op": "lt", - "fill": true, - "line": true, - "colorMode": "critical", - "fillColor": "rgba(234, 112, 112, 0.12)", - "lineColor": "rgba(237, 46, 24, 0.60)" - }, - { - "value": 80, - "op": "gt", - "fill": true, - "line": true, - "colorMode": "critical", - "fillColor": "rgba(234, 112, 112, 0.12)", - "lineColor": "rgba(237, 46, 24, 0.60)" - } - ], - "links": [] - }, - { - "title": "Alert is within range", - "error": false, - "span": 3, - "editable": true, - "type": "graph", - "isNew": true, - "id": 19, - "targets": [ - { - "target": "aliasByNode(statsd.fakesite.counters.session_start.*.count, 4)", - "refId": "A" - } - ], - "datasource": "${DS_GRAPHITE}", - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - }, - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "short" - } - ], - "xaxis": { - "show": true - }, - "alert": { - "conditions": [ - { - "type": "query", - "query": { - "params": [ - "A", - "5m", - "now" - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "evaluator": { - "type": "within_range", - "params": [ - 60, - 80 - ] - } - } - ], - "severity": "critical", - "frequency": "10s", - "handler": 1, - "notifications": [], - "name": "Alert is within range", - "enabled": true - }, - "lines": true, - "fill": 1, - "linewidth": 2, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true, - "sort": 0, - "msResolution": false - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [ - { - "value": 60, - "op": "gt", - "fill": true, - "line": true, - "colorMode": "critical", - "fillColor": "rgba(234, 112, 112, 0.12)", - "lineColor": "rgba(237, 46, 24, 0.60)" - }, - { - "value": 80, - "op": "lt", - "fill": true, - "line": true, - "colorMode": "critical", - "fillColor": "rgba(234, 112, 112, 0.12)", - "lineColor": "rgba(237, 46, 24, 0.60)" - } - ], - "links": [] - } - ] - } - ], - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 50, - "links": [], - "gnetId": null -} \ No newline at end of file diff --git a/jest.config.js b/jest.config.js index 606465c9840..cac634fbf10 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,19 +1,14 @@ module.exports = { verbose: false, - "globals": { - "ts-jest": { - "tsConfigFile": "tsconfig.json" - } - }, "transform": { - "^.+\\.tsx?$": "/node_modules/ts-jest/preprocessor.js" + "^.+\\.(ts|tsx)$": "ts-jest" }, "moduleDirectories": ["node_modules", "public"], "roots": [ "/public" ], - "testRegex": "(\\.|/)(jest)\\.(jsx?|tsx?)$", + "testRegex": "(\\.|/)(test)\\.(jsx?|tsx?)$", "moduleFileExtensions": [ "ts", "tsx", diff --git a/karma.conf.js b/karma.conf.js deleted file mode 100644 index 3f006af08b6..00000000000 --- a/karma.conf.js +++ /dev/null @@ -1,40 +0,0 @@ -var webpack = require('webpack'); -var path = require('path'); -var webpackTestConfig = require('./scripts/webpack/webpack.test.js'); - -module.exports = function(config) { - - 'use strict'; - - config.set({ - frameworks: ['mocha', 'expect', 'sinon'], - - // list of files / patterns to load in the browser - files: [ - { pattern: 'public/test/index.ts', watched: false } - ], - - preprocessors: { - 'public/test/index.ts': ['webpack', 'sourcemap'], - }, - - webpack: webpackTestConfig, - webpackServer: { - noInfo: true, // please don't spam the console when running in karma! - }, - - // list of files to exclude - exclude: [], - reporters: ['dots'], - port: 9876, - colors: true, - logLevel: config.LOG_INFO, - autoWatch: true, - browsers: ['PhantomJS'], - captureTimeout: 20000, - singleRun: true, - // autoWatchBatchDelay: 1000, - // browserNoActivityTimeout: 60000, - }); - -}; diff --git a/latest.json b/latest.json index b476f44a00a..5e72c11c1f2 100644 --- a/latest.json +++ b/latest.json @@ -1,4 +1,4 @@ { - "stable": "5.0.0", - "testing": "5.0.0" + "stable": "5.3.4", + "testing": "5.3.4" } diff --git a/package.json b/package.json index 6dcfc16b82b..68b7647fb58 100644 --- a/package.json +++ b/package.json @@ -4,35 +4,38 @@ "company": "Grafana Labs" }, "name": "grafana", - "version": "5.1.0-pre1", + "version": "5.4.0-pre1", "repository": { "type": "git", "url": "http://github.com/grafana/grafana.git" }, "devDependencies": { "@types/d3": "^4.10.1", - "@types/enzyme": "^2.8.9", - "@types/jest": "^21.1.4", + "@types/enzyme": "^3.1.13", + "@types/jest": "^23.3.2", "@types/node": "^8.0.31", - "@types/react": "^16.0.25", - "@types/react-dom": "^16.0.3", - "angular-mocks": "^1.6.6", + "@types/react": "^16.4.14", + "@types/react-custom-scrollbars": "^4.0.5", + "@types/react-dom": "^16.0.7", + "@types/react-select": "^2.0.4", + "angular-mocks": "1.6.6", "autoprefixer": "^6.4.0", - "awesome-typescript-loader": "^3.2.3", "axios": "^0.17.1", "babel-core": "^6.26.0", - "babel-loader": "^7.1.2", + "babel-loader": "^7.1.4", + "babel-plugin-syntax-dynamic-import": "^6.18.0", "babel-preset-es2015": "^6.24.1", + "clean-webpack-plugin": "^0.1.19", "css-loader": "^0.28.7", - "enzyme": "^3.1.0", - "enzyme-adapter-react-16": "^1.0.1", - "enzyme-to-json": "^3.3.0", + "enzyme": "^3.6.0", + "enzyme-adapter-react-16": "^1.5.0", + "enzyme-to-json": "^3.3.4", "es6-promise": "^3.0.2", "es6-shim": "^0.35.3", "expect.js": "~0.2.0", "expose-loader": "^0.7.3", - "extract-text-webpack-plugin": "^3.0.0", - "file-loader": "^0.11.2", + "file-loader": "^1.1.11", + "fork-ts-checker-webpack-plugin": "^0.4.9", "gaze": "^1.1.2", "glob": "~7.0.0", "grunt": "1.0.1", @@ -43,10 +46,8 @@ "grunt-contrib-concat": "^1.0.1", "grunt-contrib-copy": "~1.0.0", "grunt-contrib-cssmin": "~1.0.2", - "grunt-contrib-jshint": "~1.1.0", "grunt-exec": "^1.0.1", - "grunt-jscs": "3.0.1", - "grunt-karma": "~2.0.0", + "grunt-newer": "^1.3.0", "grunt-notify": "^0.4.5", "grunt-postcss": "^0.8.0", "grunt-sass": "^2.0.0", @@ -54,60 +55,57 @@ "grunt-usemin": "3.1.1", "grunt-webpack": "^3.0.2", "html-loader": "^0.5.1", - "html-webpack-plugin": "^2.30.1", + "html-webpack-harddisk-plugin": "^0.2.0", + "html-webpack-plugin": "^3.2.0", "husky": "^0.14.3", - "jest": "^22.0.4", - "jshint-stylish": "~2.2.1", - "json-loader": "^0.5.7", - "karma": "1.7.0", - "karma-chrome-launcher": "~2.2.0", - "karma-expect": "~1.1.3", - "karma-mocha": "~1.3.0", - "karma-phantomjs-launcher": "1.0.4", - "karma-sinon": "^1.0.5", - "karma-sourcemap-loader": "^0.3.7", - "karma-webpack": "^2.0.4", + "jest": "^23.6.0", "lint-staged": "^6.0.0", "load-grunt-tasks": "3.5.2", - "mobx-react-devtools": "^4.2.15", + "mini-css-extract-plugin": "^0.4.0", "mocha": "^4.0.1", "ng-annotate-loader": "^0.6.1", - "ng-annotate-webpack-plugin": "^0.2.1-pre", + "ng-annotate-webpack-plugin": "^0.3.0", "ngtemplate-loader": "^2.0.1", "npm": "^5.4.2", + "optimize-css-assets-webpack-plugin": "^4.0.2", "phantomjs-prebuilt": "^2.1.15", "postcss-browser-reporter": "^0.5.0", "postcss-loader": "^2.0.6", "postcss-reporter": "^5.0.0", "prettier": "1.9.2", - "react-test-renderer": "^16.0.0", + "react-hot-loader": "^4.3.6", + "react-test-renderer": "^16.5.0", "sass-lint": "^1.10.2", - "sass-loader": "^6.0.6", + "sass-loader": "^7.0.1", "sinon": "1.17.6", + "style-loader": "^0.21.0", "systemjs": "0.20.19", "systemjs-plugin-css": "^0.1.36", - "ts-jest": "^22.0.0", - "ts-loader": "^3.2.0", + "ts-jest": "^23.10.4", + "ts-loader": "^5.1.0", + "tslib": "^1.9.3", "tslint": "^5.8.0", "tslint-loader": "^3.5.3", - "typescript": "^2.6.2", - "webpack": "^3.10.0", + "typescript": "^3.0.3", + "uglifyjs-webpack-plugin": "^1.2.7", + "webpack": "4.19.1", "webpack-bundle-analyzer": "^2.9.0", "webpack-cleanup-plugin": "^0.5.1", + "webpack-cli": "^2.1.4", + "webpack-dev-server": "^3.1.0", "webpack-merge": "^4.1.0", "zone.js": "^0.7.2" }, "scripts": { - "dev": "webpack --progress --colors --config scripts/webpack/webpack.dev.js", - "watch": "webpack --progress --colors --watch --config scripts/webpack/webpack.dev.js", + "dev": "webpack --progress --colors --mode development --config scripts/webpack/webpack.dev.js", + "start": "webpack-dev-server --progress --colors --mode development --config scripts/webpack/webpack.hot.js", + "watch": "webpack --progress --colors --watch --mode development --config scripts/webpack/webpack.dev.js", "build": "grunt build", "test": "grunt test", - "test:coverage": "grunt test --coverage=true", - "lint": "tslint -c tslint.json --project tsconfig.json --type-check", - "karma": "node ./node_modules/grunt-cli/bin/grunt karma:dev", - "jest": "node ./node_modules/jest-cli/bin/jest.js --notify --watch", - "api-tests": "node ./node_modules/jest-cli/bin/jest.js --notify --watch --config=tests/api/jest.js", - "precommit": "lint-staged && node ./node_modules/grunt-cli/bin/grunt precommit" + "lint": "tslint -c tslint.json --project tsconfig.json", + "jest": "jest --notify --watch", + "api-tests": "jest --notify --watch --config=tests/api/jest.js", + "precommit": "lint-staged && grunt precommit" }, "lint-staged": { "*.{ts,tsx}": [ @@ -130,42 +128,57 @@ }, "license": "Apache-2.0", "dependencies": { - "angular": "^1.6.6", - "angular-bindonce": "^0.3.1", - "angular-native-dragdrop": "^1.2.2", - "angular-route": "^1.6.6", - "angular-sanitize": "^1.6.6", + "angular": "1.6.6", + "angular-bindonce": "0.3.1", + "angular-native-dragdrop": "1.2.2", + "angular-route": "1.6.6", + "angular-sanitize": "1.6.6", + "babel-jest": "^23.6.0", "babel-polyfill": "^6.26.0", + "baron": "^3.0.3", "brace": "^0.10.0", "classnames": "^2.2.5", "clipboard": "^1.7.1", "d3": "^4.11.0", - "d3-scale-chromatic": "^1.1.1", + "d3-scale-chromatic": "^1.3.0", "eventemitter3": "^2.0.3", "file-saver": "^1.3.3", + "immutable": "^3.8.2", "jquery": "^3.2.1", - "lodash": "^4.17.4", - "mobx": "^3.4.1", - "mobx-react": "^4.3.5", - "mobx-state-tree": "^1.3.1", - "moment": "^2.18.1", + "lodash": "^4.17.10", + "moment": "^2.22.2", "mousetrap": "^1.6.0", "mousetrap-global-bind": "^1.1.0", - "perfect-scrollbar": "^1.2.0", - "prop-types": "^15.6.0", - "react": "^16.2.0", - "react-dom": "^16.2.0", - "react-grid-layout-grafana": "0.16.0", + "prismjs": "^1.6.0", + "prop-types": "^15.6.2", + "rc-cascader": "^0.14.0", + "react": "^16.5.0", + "react-custom-scrollbars": "^4.2.1", + "react-dom": "^16.5.0", + "react-grid-layout": "0.16.6", "react-highlight-words": "^0.10.0", "react-popper": "^0.7.5", - "react-select": "^1.1.0", + "react-redux": "^5.0.7", + "react-select": "2.1.0", "react-sizeme": "^2.3.6", + "react-table": "^6.8.6", "react-transition-group": "^2.2.1", + "redux": "^4.0.0", + "redux-logger": "^3.0.6", + "redux-thunk": "^2.3.0", "remarkable": "^1.7.1", "rst2html": "github:thoward/rst2html#990cb89", "rxjs": "^5.4.3", + "slate": "^0.33.4", + "slate-plain-serializer": "^0.5.10", + "slate-prism": "^0.5.0", + "slate-react": "^0.12.4", "tether": "^1.4.0", "tether-drop": "https://github.com/torkelo/drop/tarball/master", - "tinycolor2": "^1.4.1" + "tinycolor2": "^1.4.1", + "tslint-react": "^3.6.0" + }, + "resolutions": { + "caniuse-db": "1.0.30000772" } } diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 00000000000..dc8972b0ba0 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,52 @@ +FROM debian:stretch-slim + +ARG GRAFANA_TGZ="grafana-latest.linux-x64.tar.gz" + +RUN apt-get update && apt-get install -qq -y tar && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +COPY ${GRAFANA_TGZ} /tmp/grafana.tar.gz + +RUN mkdir /tmp/grafana && tar xfvz /tmp/grafana.tar.gz --strip-components=1 -C /tmp/grafana + +FROM debian:stretch-slim + +ARG GF_UID="472" +ARG GF_GID="472" + +ENV PATH=/usr/share/grafana/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + GF_PATHS_CONFIG="/etc/grafana/grafana.ini" \ + GF_PATHS_DATA="/var/lib/grafana" \ + GF_PATHS_HOME="/usr/share/grafana" \ + GF_PATHS_LOGS="/var/log/grafana" \ + GF_PATHS_PLUGINS="/var/lib/grafana/plugins" \ + GF_PATHS_PROVISIONING="/etc/grafana/provisioning" + +WORKDIR $GF_PATHS_HOME + +RUN apt-get update && apt-get install -qq -y libfontconfig ca-certificates curl && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=0 /tmp/grafana "$GF_PATHS_HOME" + +RUN mkdir -p "$GF_PATHS_HOME/.aws" && \ + groupadd -r -g $GF_GID grafana && \ + useradd -r -u $GF_UID -g grafana grafana && \ + mkdir -p "$GF_PATHS_PROVISIONING/datasources" \ + "$GF_PATHS_PROVISIONING/dashboards" \ + "$GF_PATHS_LOGS" \ + "$GF_PATHS_PLUGINS" \ + "$GF_PATHS_DATA" && \ + cp "$GF_PATHS_HOME/conf/sample.ini" "$GF_PATHS_CONFIG" && \ + cp "$GF_PATHS_HOME/conf/ldap.toml" /etc/grafana/ldap.toml && \ + chown -R grafana:grafana "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS" && \ + chmod 777 "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS" + +EXPOSE 3000 + +COPY ./run.sh /run.sh + +USER grafana +ENTRYPOINT [ "/run.sh" ] diff --git a/packaging/docker/README.md b/packaging/docker/README.md new file mode 100644 index 00000000000..cfb3c7248ef --- /dev/null +++ b/packaging/docker/README.md @@ -0,0 +1,43 @@ +# Grafana Docker image + +## Running your Grafana container + +Start your container binding the external port `3000`. + +```bash +docker run -d --name=grafana -p 3000:3000 grafana/grafana +``` + +Try it out, default admin user is admin/admin. + +## How to use the container + +Further documentation can be found at http://docs.grafana.org/installation/docker/ + +## Changelog + +### v5.1.5, v5.2.0-beta2 +* Fix: config keys ending with _FILE are not respected [#170](https://github.com/grafana/grafana-docker/issues/170) + +### v5.2.0-beta1 +* Support for Docker Secrets + +### v5.1.0 +* Major restructuring of the container +* Usage of `chown` removed +* File permissions incompatibility with previous versions + * user id changed from 104 to 472 + * group id changed from 107 to 472 +* Runs as the grafana user by default (instead of root) +* All default volumes removed + +### v4.2.0 +* Plugins are now installed into ${GF_PATHS_PLUGINS} +* Building the container now requires a full url to the deb package instead of just version +* Fixes bug caused by installing multiple plugins + +### v4.0.0-beta2 +* Plugins dir (`/var/lib/grafana/plugins`) is no longer a separate volume + +### v3.1.1 +* Make it possible to install specific plugin version https://github.com/grafana/grafana-docker/issues/59#issuecomment-260584026 diff --git a/packaging/docker/build-deploy.sh b/packaging/docker/build-deploy.sh new file mode 100755 index 00000000000..ac3226a4a61 --- /dev/null +++ b/packaging/docker/build-deploy.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e + +_grafana_version=$1 +./build.sh "$_grafana_version" +docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" + +./push_to_docker_hub.sh "$_grafana_version" + +if echo "$_grafana_version" | grep -q "^master-"; then + apk add --no-cache curl + ./deploy_to_k8s.sh "grafana/grafana-dev:$_grafana_version" +fi diff --git a/packaging/docker/build-enterprise.sh b/packaging/docker/build-enterprise.sh new file mode 100755 index 00000000000..2f59e436d95 --- /dev/null +++ b/packaging/docker/build-enterprise.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +_grafana_tag=$1 +_docker_repo=${2:-grafana/grafana-enterprise} + +docker build \ + --tag "${_docker_repo}:${_grafana_tag}"\ + --no-cache=true \ + . + +docker push "${_docker_repo}:${_grafana_tag}" diff --git a/packaging/docker/build.sh b/packaging/docker/build.sh new file mode 100755 index 00000000000..c303c71cd5f --- /dev/null +++ b/packaging/docker/build.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +_grafana_tag=$1 + +# If the tag starts with v, treat this as a official release +if echo "$_grafana_tag" | grep -q "^v"; then + _grafana_version=$(echo "${_grafana_tag}" | cut -d "v" -f 2) + _docker_repo=${2:-grafana/grafana} +else + _grafana_version=$_grafana_tag + _docker_repo=${2:-grafana/grafana-dev} +fi + +echo "Building ${_docker_repo}:${_grafana_version}" + +docker build \ + --tag "${_docker_repo}:${_grafana_version}" \ + --no-cache=true . + +# Tag as 'latest' for official release; otherwise tag as grafana/grafana:master +if echo "$_grafana_tag" | grep -q "^v"; then + docker tag "${_docker_repo}:${_grafana_version}" "${_docker_repo}:latest" +else + docker tag "${_docker_repo}:${_grafana_version}" "grafana/grafana:master" +fi diff --git a/packaging/docker/custom/Dockerfile b/packaging/docker/custom/Dockerfile new file mode 100644 index 00000000000..79eba5f29e9 --- /dev/null +++ b/packaging/docker/custom/Dockerfile @@ -0,0 +1,16 @@ +ARG GRAFANA_VERSION="latest" + +FROM grafana/grafana:${GRAFANA_VERSION} + +USER grafana + +ARG GF_INSTALL_PLUGINS="" + +RUN if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then \ + OLDIFS=$IFS; \ + IFS=','; \ + for plugin in ${GF_INSTALL_PLUGINS}; do \ + IFS=$OLDIFS; \ + grafana-cli --pluginsDir "$GF_PATHS_PLUGINS" plugins install ${plugin}; \ + done; \ +fi diff --git a/packaging/docker/deploy_to_k8s.sh b/packaging/docker/deploy_to_k8s.sh new file mode 100755 index 00000000000..26cf88ef688 --- /dev/null +++ b/packaging/docker/deploy_to_k8s.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +curl -s --header "Content-Type: application/json" \ + --data "{\"build_parameters\": {\"CIRCLE_JOB\": \"deploy\", \"IMAGE_NAMES\": \"$1\"}}" \ + --request POST \ + https://circleci.com/api/v1.1/project/github/raintank/deployment_tools/tree/master?circle-token=$CIRCLE_TOKEN diff --git a/packaging/docker/push_to_docker_hub.sh b/packaging/docker/push_to_docker_hub.sh new file mode 100755 index 00000000000..526c216f8fa --- /dev/null +++ b/packaging/docker/push_to_docker_hub.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +_grafana_tag=$1 + +# If the tag starts with v, treat this as a official release +if echo "$_grafana_tag" | grep -q "^v"; then + _grafana_version=$(echo "${_grafana_tag}" | cut -d "v" -f 2) + _docker_repo=${2:-grafana/grafana} +else + _grafana_version=$_grafana_tag + _docker_repo=${2:-grafana/grafana-dev} +fi + +echo "pushing ${_docker_repo}:${_grafana_version}" +docker push "${_docker_repo}:${_grafana_version}" + +if echo "$_grafana_tag" | grep -q "^v" && echo "$_grafana_tag" | grep -vq "beta"; then + echo "pushing ${_docker_repo}:latest" + docker push "${_docker_repo}:latest" +elif echo "$_grafana_tag" | grep -q "master"; then + echo "pushing grafana/grafana:master" + docker push grafana/grafana:master +fi diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh new file mode 100755 index 00000000000..bc001bdf90a --- /dev/null +++ b/packaging/docker/run.sh @@ -0,0 +1,88 @@ +#!/bin/bash -e + +PERMISSIONS_OK=0 + +if [ ! -r "$GF_PATHS_CONFIG" ]; then + echo "GF_PATHS_CONFIG='$GF_PATHS_CONFIG' is not readable." + PERMISSIONS_OK=1 +fi + +if [ ! -w "$GF_PATHS_DATA" ]; then + echo "GF_PATHS_DATA='$GF_PATHS_DATA' is not writable." + PERMISSIONS_OK=1 +fi + +if [ ! -r "$GF_PATHS_HOME" ]; then + echo "GF_PATHS_HOME='$GF_PATHS_HOME' is not readable." + PERMISSIONS_OK=1 +fi + +if [ $PERMISSIONS_OK -eq 1 ]; then + echo "You may have issues with file permissions, more information here: http://docs.grafana.org/installation/docker/#migration-from-a-previous-version-of-the-docker-container-to-5-1-or-later" +fi + +if [ ! -d "$GF_PATHS_PLUGINS" ]; then + mkdir "$GF_PATHS_PLUGINS" +fi + +if [ ! -z ${GF_AWS_PROFILES+x} ]; then + > "$GF_PATHS_HOME/.aws/credentials" + + for profile in ${GF_AWS_PROFILES}; do + access_key_varname="GF_AWS_${profile}_ACCESS_KEY_ID" + secret_key_varname="GF_AWS_${profile}_SECRET_ACCESS_KEY" + region_varname="GF_AWS_${profile}_REGION" + + if [ ! -z "${!access_key_varname}" -a ! -z "${!secret_key_varname}" ]; then + echo "[${profile}]" >> "$GF_PATHS_HOME/.aws/credentials" + echo "aws_access_key_id = ${!access_key_varname}" >> "$GF_PATHS_HOME/.aws/credentials" + echo "aws_secret_access_key = ${!secret_key_varname}" >> "$GF_PATHS_HOME/.aws/credentials" + if [ ! -z "${!region_varname}" ]; then + echo "region = ${!region_varname}" >> "$GF_PATHS_HOME/.aws/credentials" + fi + fi + done + + chmod 600 "$GF_PATHS_HOME/.aws/credentials" +fi + +# Convert all environment variables with names ending in __FILE into the content of +# the file that they point at and use the name without the trailing __FILE. +# This can be used to carry in Docker secrets. +for VAR_NAME in $(env | grep '^GF_[^=]\+__FILE=.\+' | sed -r "s/([^=]*)__FILE=.*/\1/g"); do + VAR_NAME_FILE="$VAR_NAME"__FILE + if [ "${!VAR_NAME}" ]; then + echo >&2 "ERROR: Both $VAR_NAME and $VAR_NAME_FILE are set (but are exclusive)" + exit 1 + fi + echo "Getting secret $VAR_NAME from ${!VAR_NAME_FILE}" + export "$VAR_NAME"="$(< "${!VAR_NAME_FILE}")" + unset "$VAR_NAME_FILE" +done + +export HOME="$GF_PATHS_HOME" + +if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then + OLDIFS=$IFS + IFS=',' + for plugin in ${GF_INSTALL_PLUGINS}; do + IFS=$OLDIFS + if [[ $plugin =~ .*\;.* ]]; then + pluginUrl=$(echo "$plugin" | cut -d';' -f 1) + pluginWithoutUrl=$(echo "$plugin" | cut -d';' -f 2) + grafana-cli --pluginUrl "${pluginUrl}" --pluginsDir "${GF_PATHS_PLUGINS}" plugins install ${pluginWithoutUrl} + else + grafana-cli --pluginsDir "${GF_PATHS_PLUGINS}" plugins install ${plugin} + fi + done +fi + +exec grafana-server \ + --homepath="$GF_PATHS_HOME" \ + --config="$GF_PATHS_CONFIG" \ + "$@" \ + cfg:default.log.mode="console" \ + cfg:default.paths.data="$GF_PATHS_DATA" \ + cfg:default.paths.logs="$GF_PATHS_LOGS" \ + cfg:default.paths.plugins="$GF_PATHS_PLUGINS" \ + cfg:default.paths.provisioning="$GF_PATHS_PROVISIONING" diff --git a/packaging/publish/publish_testing.sh b/packaging/publish/publish_testing.sh index 08ba2a89dd9..9fd4e1f93b9 100755 --- a/packaging/publish/publish_testing.sh +++ b/packaging/publish/publish_testing.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash -deb_ver=5.0.0-beta5 -rpm_ver=5.0.0-beta5 +deb_ver=5.1.0-beta1 +rpm_ver=5.1.0-beta1 wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_${deb_ver}_amd64.deb diff --git a/packaging/release_process.md b/packaging/release_process.md deleted file mode 100644 index 6037a9c499c..00000000000 --- a/packaging/release_process.md +++ /dev/null @@ -1,29 +0,0 @@ -# New Grafana Release Processes - -## Building release packages - -1) Update package.json so that it has the right version. -2) Create a git tag for the release: `git tag -a v3.0.4 -m "3.0.4 release"` -3) Push branch & tag to github! -2) Packages from master a built automatically by circle CI for this repo [grafana/grafana-packer](https://github.com/grafana/grafana-packer) - -### Non master branch - -When building from non master branch create a new branch in repo [grafana/grafana-packer](https://github.com/grafana/grafana-packer) -and configure circle.yml to deploy that branch as well, https://github.com/grafana/grafana-packer/blob/master/circle.yml#L25, -you also need to update https://github.com/grafana/grafana-packer/blob/v3.1.x/deploy.sh#L7. - -### Windows build - -Sign into ci.appveyor.com and the Grafana project's build history page. Builds for windows take a long time (around 20min) -and fail quite often for random reasons so I usually continue with the release process without a windows build already built. - -1) Click on the green build that has the correct version and tag -2) Click on `DEPLOYMENTS` -3) Click on `NEW DEPLOYMENT` -4) Select GrafanaBuildS3 -4) Select the build you want to deploy. - -The deployment should be quick (just uploads the release zip file to S3) - - diff --git a/pkg/api/admin.go b/pkg/api/admin.go index 52d271ce69b..54a86724f0c 100644 --- a/pkg/api/admin.go +++ b/pkg/api/admin.go @@ -12,7 +12,7 @@ import ( func AdminGetSettings(c *m.ReqContext) { settings := make(map[string]interface{}) - for _, section := range setting.Cfg.Sections() { + for _, section := range setting.Raw.Sections() { jsonSec := make(map[string]interface{}) settings[section.Name()] = jsonSec diff --git a/pkg/api/admin_users.go b/pkg/api/admin_users.go index 4cf7f4db4ec..dc3d390dda9 100644 --- a/pkg/api/admin_users.go +++ b/pkg/api/admin_users.go @@ -47,14 +47,14 @@ func AdminCreateUser(c *m.ReqContext, form dtos.AdminCreateUserForm) { } func AdminUpdateUserPassword(c *m.ReqContext, form dtos.AdminUpdateUserPasswordForm) { - userId := c.ParamsInt64(":id") + userID := c.ParamsInt64(":id") if len(form.Password) < 4 { c.JsonApiErr(400, "New password too short", nil) return } - userQuery := m.GetUserByIdQuery{Id: userId} + userQuery := m.GetUserByIdQuery{Id: userID} if err := bus.Dispatch(&userQuery); err != nil { c.JsonApiErr(500, "Could not read user from database", err) @@ -64,7 +64,7 @@ func AdminUpdateUserPassword(c *m.ReqContext, form dtos.AdminUpdateUserPasswordF passwordHashed := util.EncodePassword(form.Password, userQuery.Result.Salt) cmd := m.ChangeUserPasswordCommand{ - UserId: userId, + UserId: userID, NewPassword: passwordHashed, } @@ -77,10 +77,10 @@ func AdminUpdateUserPassword(c *m.ReqContext, form dtos.AdminUpdateUserPasswordF } func AdminUpdateUserPermissions(c *m.ReqContext, form dtos.AdminUpdateUserPermissionsForm) { - userId := c.ParamsInt64(":id") + userID := c.ParamsInt64(":id") cmd := m.UpdateUserPermissionsCommand{ - UserId: userId, + UserId: userID, IsGrafanaAdmin: form.IsGrafanaAdmin, } @@ -93,9 +93,9 @@ func AdminUpdateUserPermissions(c *m.ReqContext, form dtos.AdminUpdateUserPermis } func AdminDeleteUser(c *m.ReqContext) { - userId := c.ParamsInt64(":id") + userID := c.ParamsInt64(":id") - cmd := m.DeleteUserCommand{UserId: userId} + cmd := m.DeleteUserCommand{UserId: userID} if err := bus.Dispatch(&cmd); err != nil { c.JsonApiErr(500, "Failed to delete user", err) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index eea4ef90c05..c68cee50948 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -2,12 +2,14 @@ package api import ( "fmt" + "strconv" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/guardian" + "github.com/grafana/grafana/pkg/services/search" ) func ValidateOrgAlert(c *m.ReqContext) { @@ -26,10 +28,10 @@ func ValidateOrgAlert(c *m.ReqContext) { } func GetAlertStatesForDashboard(c *m.ReqContext) Response { - dashboardId := c.QueryInt64("dashboardId") + dashboardID := c.QueryInt64("dashboardId") - if dashboardId == 0 { - return ApiError(400, "Missing query parameter dashboardId", nil) + if dashboardID == 0 { + return Error(400, "Missing query parameter dashboardId", nil) } query := m.GetAlertStatesForDashboardQuery{ @@ -38,20 +40,72 @@ func GetAlertStatesForDashboard(c *m.ReqContext) Response { } if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to fetch alert states", err) + return Error(500, "Failed to fetch alert states", err) } - return Json(200, query.Result) + return JSON(200, query.Result) } // GET /api/alerts func GetAlerts(c *m.ReqContext) Response { + dashboardQuery := c.Query("dashboardQuery") + dashboardTags := c.QueryStrings("dashboardTag") + stringDashboardIDs := c.QueryStrings("dashboardId") + stringFolderIDs := c.QueryStrings("folderId") + + dashboardIDs := make([]int64, 0) + for _, id := range stringDashboardIDs { + dashboardID, err := strconv.ParseInt(id, 10, 64) + if err == nil { + dashboardIDs = append(dashboardIDs, dashboardID) + } + } + + if dashboardQuery != "" || len(dashboardTags) > 0 || len(stringFolderIDs) > 0 { + folderIDs := make([]int64, 0) + for _, id := range stringFolderIDs { + folderID, err := strconv.ParseInt(id, 10, 64) + if err == nil { + folderIDs = append(folderIDs, folderID) + } + } + + searchQuery := search.Query{ + Title: dashboardQuery, + Tags: dashboardTags, + SignedInUser: c.SignedInUser, + Limit: 1000, + OrgId: c.OrgId, + DashboardIds: dashboardIDs, + Type: string(search.DashHitDB), + FolderIds: folderIDs, + Permission: m.PERMISSION_VIEW, + } + + err := bus.Dispatch(&searchQuery) + if err != nil { + return Error(500, "List alerts failed", err) + } + + for _, d := range searchQuery.Result { + if d.Type == search.DashHitDB && d.Id > 0 { + dashboardIDs = append(dashboardIDs, d.Id) + } + } + + // if we didn't find any dashboards, return empty result + if len(dashboardIDs) == 0 { + return JSON(200, []*m.AlertListItemDTO{}) + } + } + query := m.GetAlertsQuery{ - OrgId: c.OrgId, - DashboardId: c.QueryInt64("dashboardId"), - PanelId: c.QueryInt64("panelId"), - Limit: c.QueryInt64("limit"), - User: c.SignedInUser, + OrgId: c.OrgId, + DashboardIDs: dashboardIDs, + PanelId: c.QueryInt64("panelId"), + Limit: c.QueryInt64("limit"), + User: c.SignedInUser, + Query: c.Query("query"), } states := c.QueryStrings("state") @@ -60,33 +114,37 @@ func GetAlerts(c *m.ReqContext) Response { } if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "List alerts failed", err) + return Error(500, "List alerts failed", err) } for _, alert := range query.Result { alert.Url = m.GetDashboardUrl(alert.DashboardUid, alert.DashboardSlug) } - return Json(200, query.Result) + return JSON(200, query.Result) } // POST /api/alerts/test func AlertTest(c *m.ReqContext, dto dtos.AlertTestCommand) Response { if _, idErr := dto.Dashboard.Get("id").Int64(); idErr != nil { - return ApiError(400, "The dashboard needs to be saved at least once before you can test an alert rule", nil) + return Error(400, "The dashboard needs to be saved at least once before you can test an alert rule", nil) } backendCmd := alerting.AlertTestCommand{ OrgId: c.OrgId, Dashboard: dto.Dashboard, PanelId: dto.PanelId, + User: c.SignedInUser, } if err := bus.Dispatch(&backendCmd); err != nil { if validationErr, ok := err.(alerting.ValidationError); ok { - return ApiError(422, validationErr.Error(), nil) + return Error(422, validationErr.Error(), nil) } - return ApiError(500, "Failed to test rule", err) + if err == m.ErrDataSourceAccessDenied { + return Error(403, "Access denied to datasource", err) + } + return Error(500, "Failed to test rule", err) } res := backendCmd.Result @@ -109,7 +167,7 @@ func AlertTest(c *m.ReqContext, dto dtos.AlertTestCommand) Response { dtoRes.TimeMs = fmt.Sprintf("%1.3fms", res.GetDurationMs()) - return Json(200, dtoRes) + return JSON(200, dtoRes) } // GET /api/alerts/:id @@ -118,70 +176,63 @@ func GetAlert(c *m.ReqContext) Response { query := m.GetAlertByIdQuery{Id: id} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "List alerts failed", err) + return Error(500, "List alerts failed", err) } - return Json(200, &query.Result) + return JSON(200, &query.Result) } func GetAlertNotifiers(c *m.ReqContext) Response { - return Json(200, alerting.GetNotifiers()) + return JSON(200, alerting.GetNotifiers()) } func GetAlertNotifications(c *m.ReqContext) Response { query := &m.GetAllAlertNotificationsQuery{OrgId: c.OrgId} if err := bus.Dispatch(query); err != nil { - return ApiError(500, "Failed to get alert notifications", err) + return Error(500, "Failed to get alert notifications", err) } result := make([]*dtos.AlertNotification, 0) for _, notification := range query.Result { - result = append(result, &dtos.AlertNotification{ - Id: notification.Id, - Name: notification.Name, - Type: notification.Type, - IsDefault: notification.IsDefault, - Created: notification.Created, - Updated: notification.Updated, - }) + result = append(result, dtos.NewAlertNotification(notification)) } - return Json(200, result) + return JSON(200, result) } -func GetAlertNotificationById(c *m.ReqContext) Response { +func GetAlertNotificationByID(c *m.ReqContext) Response { query := &m.GetAlertNotificationsQuery{ OrgId: c.OrgId, Id: c.ParamsInt64("notificationId"), } if err := bus.Dispatch(query); err != nil { - return ApiError(500, "Failed to get alert notifications", err) + return Error(500, "Failed to get alert notifications", err) } - return Json(200, query.Result) + return JSON(200, dtos.NewAlertNotification(query.Result)) } func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationCommand) Response { cmd.OrgId = c.OrgId if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to create alert notification", err) + return Error(500, "Failed to create alert notification", err) } - return Json(200, cmd.Result) + return JSON(200, dtos.NewAlertNotification(cmd.Result)) } func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationCommand) Response { cmd.OrgId = c.OrgId if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update alert notification", err) + return Error(500, "Failed to update alert notification", err) } - return Json(200, cmd.Result) + return JSON(200, dtos.NewAlertNotification(cmd.Result)) } func DeleteAlertNotification(c *m.ReqContext) Response { @@ -191,10 +242,10 @@ func DeleteAlertNotification(c *m.ReqContext) Response { } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to delete alert notification", err) + return Error(500, "Failed to delete alert notification", err) } - return ApiSuccess("Notification deleted") + return Success("Notification deleted") } //POST /api/alert-notifications/test @@ -207,41 +258,41 @@ func NotificationTest(c *m.ReqContext, dto dtos.NotificationTestCommand) Respons if err := bus.Dispatch(cmd); err != nil { if err == m.ErrSmtpNotEnabled { - return ApiError(412, err.Error(), err) + return Error(412, err.Error(), err) } - return ApiError(500, "Failed to send alert notifications", err) + return Error(500, "Failed to send alert notifications", err) } - return ApiSuccess("Test notification sent") + return Success("Test notification sent") } //POST /api/alerts/:alertId/pause func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response { - alertId := c.ParamsInt64("alertId") + alertID := c.ParamsInt64("alertId") - query := m.GetAlertByIdQuery{Id: alertId} + query := m.GetAlertByIdQuery{Id: alertID} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Get Alert failed", err) + return Error(500, "Get Alert failed", err) } guardian := guardian.New(query.Result.DashboardId, c.OrgId, c.SignedInUser) if canEdit, err := guardian.CanEdit(); err != nil || !canEdit { if err != nil { - return ApiError(500, "Error while checking permissions for Alert", err) + return Error(500, "Error while checking permissions for Alert", err) } - return ApiError(403, "Access denied to this dashboard and alert", nil) + return Error(403, "Access denied to this dashboard and alert", nil) } cmd := m.PauseAlertCommand{ OrgId: c.OrgId, - AlertIds: []int64{alertId}, + AlertIds: []int64{alertID}, Paused: dto.Paused, } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "", err) + return Error(500, "", err) } var response m.AlertStateType = m.AlertStatePending @@ -252,12 +303,12 @@ func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response { } result := map[string]interface{}{ - "alertId": alertId, + "alertId": alertID, "state": response, "message": "Alert " + pausedState, } - return Json(200, result) + return JSON(200, result) } //POST /api/admin/pause-all-alerts @@ -267,7 +318,7 @@ func PauseAllAlerts(c *m.ReqContext, dto dtos.PauseAllAlertsCommand) Response { } if err := bus.Dispatch(&updateCmd); err != nil { - return ApiError(500, "Failed to pause alerts", err) + return Error(500, "Failed to pause alerts", err) } var response m.AlertStateType = m.AlertStatePending @@ -283,5 +334,5 @@ func PauseAllAlerts(c *m.ReqContext, dto dtos.PauseAllAlertsCommand) Response { "alertsAffected": updateCmd.ResultCount, } - return Json(200, result) + return JSON(200, result) } diff --git a/pkg/api/alerting_test.go b/pkg/api/alerting_test.go index 9302ef7beca..331beeef5e4 100644 --- a/pkg/api/alerting_test.go +++ b/pkg/api/alerting_test.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/search" . "github.com/smartystreets/goconvey/convey" ) @@ -30,7 +31,7 @@ func TestAlertingApiEndpoint(t *testing.T) { }) bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error { - query.Result = []*m.Team{} + query.Result = []*m.TeamDTO{} return nil }) @@ -64,6 +65,60 @@ func TestAlertingApiEndpoint(t *testing.T) { }) }) }) + + loggedInUserScenarioWithRole("When calling GET on", "GET", "/api/alerts?dashboardId=1", "/api/alerts", m.ROLE_EDITOR, func(sc *scenarioContext) { + var searchQuery *search.Query + bus.AddHandler("test", func(query *search.Query) error { + searchQuery = query + return nil + }) + + var getAlertsQuery *m.GetAlertsQuery + bus.AddHandler("test", func(query *m.GetAlertsQuery) error { + getAlertsQuery = query + return nil + }) + + sc.handlerFunc = GetAlerts + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + + So(searchQuery, ShouldBeNil) + So(getAlertsQuery, ShouldNotBeNil) + }) + + loggedInUserScenarioWithRole("When calling GET on", "GET", "/api/alerts?dashboardId=1&dashboardId=2&folderId=3&dashboardTag=abc&dashboardQuery=dbQuery&limit=5&query=alertQuery", "/api/alerts", m.ROLE_EDITOR, func(sc *scenarioContext) { + var searchQuery *search.Query + bus.AddHandler("test", func(query *search.Query) error { + searchQuery = query + query.Result = search.HitList{ + &search.Hit{Id: 1}, + &search.Hit{Id: 2}, + } + return nil + }) + + var getAlertsQuery *m.GetAlertsQuery + bus.AddHandler("test", func(query *m.GetAlertsQuery) error { + getAlertsQuery = query + return nil + }) + + sc.handlerFunc = GetAlerts + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + + So(searchQuery, ShouldNotBeNil) + So(searchQuery.DashboardIds[0], ShouldEqual, 1) + So(searchQuery.DashboardIds[1], ShouldEqual, 2) + So(searchQuery.FolderIds[0], ShouldEqual, 3) + So(searchQuery.Tags[0], ShouldEqual, "abc") + So(searchQuery.Title, ShouldEqual, "dbQuery") + + So(getAlertsQuery, ShouldNotBeNil) + So(getAlertsQuery.DashboardIDs[0], ShouldEqual, 1) + So(getAlertsQuery.DashboardIDs[1], ShouldEqual, 2) + So(getAlertsQuery.Limit, ShouldEqual, 5) + So(getAlertsQuery.Query, ShouldEqual, "alertQuery") + }) }) } @@ -80,7 +135,7 @@ func postAlertScenario(desc string, url string, routePattern string, role m.Role defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.UserId = TestUserID sc.context.OrgId = TestOrgID diff --git a/pkg/api/annotations.go b/pkg/api/annotations.go index fb75e0bf129..242b5531f51 100644 --- a/pkg/api/annotations.go +++ b/pkg/api/annotations.go @@ -2,7 +2,6 @@ package api import ( "strings" - "time" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/components/simplejson" @@ -15,32 +14,33 @@ import ( func GetAnnotations(c *m.ReqContext) Response { query := &annotations.ItemQuery{ - From: c.QueryInt64("from") / 1000, - To: c.QueryInt64("to") / 1000, + From: c.QueryInt64("from"), + To: c.QueryInt64("to"), OrgId: c.OrgId, + UserId: c.QueryInt64("userId"), AlertId: c.QueryInt64("alertId"), DashboardId: c.QueryInt64("dashboardId"), PanelId: c.QueryInt64("panelId"), Limit: c.QueryInt64("limit"), Tags: c.QueryStrings("tags"), Type: c.Query("type"), + MatchAny: c.QueryBool("matchAny"), } repo := annotations.GetRepository() items, err := repo.Find(query) if err != nil { - return ApiError(500, "Failed to get annotations", err) + return Error(500, "Failed to get annotations", err) } for _, item := range items { if item.Email != "" { item.AvatarUrl = dtos.GetGravatarUrl(item.Email) } - item.Time = item.Time * 1000 } - return Json(200, items) + return JSON(200, items) } type CreateAnnotationError struct { @@ -52,7 +52,7 @@ func (e *CreateAnnotationError) Error() string { } func PostAnnotation(c *m.ReqContext, cmd dtos.PostAnnotationsCmd) Response { - if canSave, err := canSaveByDashboardId(c, cmd.DashboardId); err != nil || !canSave { + if canSave, err := canSaveByDashboardID(c, cmd.DashboardId); err != nil || !canSave { return dashboardGuardianResponse(err) } @@ -60,7 +60,7 @@ func PostAnnotation(c *m.ReqContext, cmd dtos.PostAnnotationsCmd) Response { if cmd.Text == "" { err := &CreateAnnotationError{"text field should not be empty"} - return ApiError(500, "Failed to save annotation", err) + return Error(500, "Failed to save annotation", err) } item := annotations.Item{ @@ -68,18 +68,14 @@ func PostAnnotation(c *m.ReqContext, cmd dtos.PostAnnotationsCmd) Response { UserId: c.UserId, DashboardId: cmd.DashboardId, PanelId: cmd.PanelId, - Epoch: cmd.Time / 1000, + Epoch: cmd.Time, Text: cmd.Text, Data: cmd.Data, Tags: cmd.Tags, } - if item.Epoch == 0 { - item.Epoch = time.Now().Unix() - } - if err := repo.Save(&item); err != nil { - return ApiError(500, "Failed to save annotation", err) + return Error(500, "Failed to save annotation", err) } startID := item.Id @@ -93,24 +89,24 @@ func PostAnnotation(c *m.ReqContext, cmd dtos.PostAnnotationsCmd) Response { } if err := repo.Update(&item); err != nil { - return ApiError(500, "Failed set regionId on annotation", err) + return Error(500, "Failed set regionId on annotation", err) } item.Id = 0 - item.Epoch = cmd.TimeEnd / 1000 + item.Epoch = cmd.TimeEnd if err := repo.Save(&item); err != nil { - return ApiError(500, "Failed save annotation for region end time", err) + return Error(500, "Failed save annotation for region end time", err) } - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "message": "Annotation added", "id": startID, "endId": item.Id, }) } - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "message": "Annotation added", "id": startID, }) @@ -129,12 +125,9 @@ func PostGraphiteAnnotation(c *m.ReqContext, cmd dtos.PostGraphiteAnnotationsCmd if cmd.What == "" { err := &CreateAnnotationError{"what field should not be empty"} - return ApiError(500, "Failed to save Graphite annotation", err) + return Error(500, "Failed to save Graphite annotation", err) } - if cmd.When == 0 { - cmd.When = time.Now().Unix() - } text := formatGraphiteAnnotation(cmd.What, cmd.Data) // Support tags in prior to Graphite 0.10.0 format (string of tags separated by space) @@ -152,133 +145,137 @@ func PostGraphiteAnnotation(c *m.ReqContext, cmd dtos.PostGraphiteAnnotationsCmd tagsArray = append(tagsArray, tagStr) } else { err := &CreateAnnotationError{"tag should be a string"} - return ApiError(500, "Failed to save Graphite annotation", err) + return Error(500, "Failed to save Graphite annotation", err) } } default: err := &CreateAnnotationError{"unsupported tags format"} - return ApiError(500, "Failed to save Graphite annotation", err) + return Error(500, "Failed to save Graphite annotation", err) } item := annotations.Item{ OrgId: c.OrgId, UserId: c.UserId, - Epoch: cmd.When, + Epoch: cmd.When * 1000, Text: text, Tags: tagsArray, } if err := repo.Save(&item); err != nil { - return ApiError(500, "Failed to save Graphite annotation", err) + return Error(500, "Failed to save Graphite annotation", err) } - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "message": "Graphite annotation added", "id": item.Id, }) } func UpdateAnnotation(c *m.ReqContext, cmd dtos.UpdateAnnotationsCmd) Response { - annotationId := c.ParamsInt64(":annotationId") + annotationID := c.ParamsInt64(":annotationId") repo := annotations.GetRepository() - if resp := canSave(c, repo, annotationId); resp != nil { + if resp := canSave(c, repo, annotationID); resp != nil { return resp } item := annotations.Item{ OrgId: c.OrgId, UserId: c.UserId, - Id: annotationId, - Epoch: cmd.Time / 1000, + Id: annotationID, + Epoch: cmd.Time, Text: cmd.Text, Tags: cmd.Tags, } if err := repo.Update(&item); err != nil { - return ApiError(500, "Failed to update annotation", err) + return Error(500, "Failed to update annotation", err) } if cmd.IsRegion { itemRight := item itemRight.RegionId = item.Id - itemRight.Epoch = cmd.TimeEnd / 1000 + itemRight.Epoch = cmd.TimeEnd // We don't know id of region right event, so set it to 0 and find then using query like // ... WHERE region_id = AND id != ... itemRight.Id = 0 if err := repo.Update(&itemRight); err != nil { - return ApiError(500, "Failed to update annotation for region end time", err) + return Error(500, "Failed to update annotation for region end time", err) } } - return ApiSuccess("Annotation updated") + return Success("Annotation updated") } func DeleteAnnotations(c *m.ReqContext, cmd dtos.DeleteAnnotationsCmd) Response { repo := annotations.GetRepository() err := repo.Delete(&annotations.DeleteParams{ - AlertId: cmd.PanelId, + OrgId: c.OrgId, + Id: cmd.AnnotationId, + RegionId: cmd.RegionId, DashboardId: cmd.DashboardId, PanelId: cmd.PanelId, }) if err != nil { - return ApiError(500, "Failed to delete annotations", err) + return Error(500, "Failed to delete annotations", err) } - return ApiSuccess("Annotations deleted") + return Success("Annotations deleted") } -func DeleteAnnotationById(c *m.ReqContext) Response { +func DeleteAnnotationByID(c *m.ReqContext) Response { repo := annotations.GetRepository() - annotationId := c.ParamsInt64(":annotationId") + annotationID := c.ParamsInt64(":annotationId") - if resp := canSave(c, repo, annotationId); resp != nil { + if resp := canSave(c, repo, annotationID); resp != nil { return resp } err := repo.Delete(&annotations.DeleteParams{ - Id: annotationId, + OrgId: c.OrgId, + Id: annotationID, }) if err != nil { - return ApiError(500, "Failed to delete annotation", err) + return Error(500, "Failed to delete annotation", err) } - return ApiSuccess("Annotation deleted") + return Success("Annotation deleted") } func DeleteAnnotationRegion(c *m.ReqContext) Response { repo := annotations.GetRepository() - regionId := c.ParamsInt64(":regionId") + regionID := c.ParamsInt64(":regionId") - if resp := canSave(c, repo, regionId); resp != nil { + if resp := canSave(c, repo, regionID); resp != nil { return resp } err := repo.Delete(&annotations.DeleteParams{ - RegionId: regionId, + OrgId: c.OrgId, + RegionId: regionID, }) if err != nil { - return ApiError(500, "Failed to delete annotation region", err) + return Error(500, "Failed to delete annotation region", err) } - return ApiSuccess("Annotation region deleted") + return Success("Annotation region deleted") } -func canSaveByDashboardId(c *m.ReqContext, dashboardId int64) (bool, error) { - if dashboardId == 0 && !c.SignedInUser.HasRole(m.ROLE_EDITOR) { +func canSaveByDashboardID(c *m.ReqContext, dashboardID int64) (bool, error) { + if dashboardID == 0 && !c.SignedInUser.HasRole(m.ROLE_EDITOR) { return false, nil } - if dashboardId > 0 { - guardian := guardian.New(dashboardId, c.OrgId, c.SignedInUser) - if canEdit, err := guardian.CanEdit(); err != nil || !canEdit { + if dashboardID != 0 { + guard := guardian.New(dashboardID, c.OrgId, c.SignedInUser) + if canEdit, err := guard.CanEdit(); err != nil || !canEdit { return false, err } } @@ -286,32 +283,16 @@ func canSaveByDashboardId(c *m.ReqContext, dashboardId int64) (bool, error) { return true, nil } -func canSave(c *m.ReqContext, repo annotations.Repository, annotationId int64) Response { - items, err := repo.Find(&annotations.ItemQuery{AnnotationId: annotationId, OrgId: c.OrgId}) +func canSave(c *m.ReqContext, repo annotations.Repository, annotationID int64) Response { + items, err := repo.Find(&annotations.ItemQuery{AnnotationId: annotationID, OrgId: c.OrgId}) if err != nil || len(items) == 0 { - return ApiError(500, "Could not find annotation to update", err) + return Error(500, "Could not find annotation to update", err) } - dashboardId := items[0].DashboardId + dashboardID := items[0].DashboardId - if canSave, err := canSaveByDashboardId(c, dashboardId); err != nil || !canSave { - return dashboardGuardianResponse(err) - } - - return nil -} - -func canSaveByRegionId(c *m.ReqContext, repo annotations.Repository, regionId int64) Response { - items, err := repo.Find(&annotations.ItemQuery{RegionId: regionId, OrgId: c.OrgId}) - - if err != nil || len(items) == 0 { - return ApiError(500, "Could not find annotation to update", err) - } - - dashboardId := items[0].DashboardId - - if canSave, err := canSaveByDashboardId(c, dashboardId); err != nil || !canSave { + if canSave, err := canSaveByDashboardID(c, dashboardID); err != nil || !canSave { return dashboardGuardianResponse(err) } diff --git a/pkg/api/annotations_test.go b/pkg/api/annotations_test.go index 7c298550673..08f3018c694 100644 --- a/pkg/api/annotations_test.go +++ b/pkg/api/annotations_test.go @@ -41,7 +41,7 @@ func TestAnnotationsApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) { - sc.handlerFunc = DeleteAnnotationById + sc.handlerFunc = DeleteAnnotationByID sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() So(sc.resp.Code, ShouldEqual, 403) }) @@ -68,7 +68,7 @@ func TestAnnotationsApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) { - sc.handlerFunc = DeleteAnnotationById + sc.handlerFunc = DeleteAnnotationByID sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() So(sc.resp.Code, ShouldEqual, 200) }) @@ -100,6 +100,11 @@ func TestAnnotationsApiEndpoint(t *testing.T) { Id: 1, } + deleteCmd := dtos.DeleteAnnotationsCmd{ + DashboardId: 1, + PanelId: 1, + } + viewerRole := m.ROLE_VIEWER editorRole := m.ROLE_EDITOR @@ -114,7 +119,7 @@ func TestAnnotationsApiEndpoint(t *testing.T) { }) bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error { - query.Result = []*m.Team{} + query.Result = []*m.TeamDTO{} return nil }) @@ -132,7 +137,7 @@ func TestAnnotationsApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) { - sc.handlerFunc = DeleteAnnotationById + sc.handlerFunc = DeleteAnnotationByID sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() So(sc.resp.Code, ShouldEqual, 403) }) @@ -159,7 +164,7 @@ func TestAnnotationsApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/annotations/1", "/api/annotations/:annotationId", role, func(sc *scenarioContext) { - sc.handlerFunc = DeleteAnnotationById + sc.handlerFunc = DeleteAnnotationByID sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() So(sc.resp.Code, ShouldEqual, 200) }) @@ -171,6 +176,25 @@ func TestAnnotationsApiEndpoint(t *testing.T) { }) }) }) + + Convey("When user is an Admin", func() { + role := m.ROLE_ADMIN + Convey("Should be able to do anything", func() { + postAnnotationScenario("When calling POST on", "/api/annotations", "/api/annotations", role, cmd, func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + + putAnnotationScenario("When calling PUT on", "/api/annotations/1", "/api/annotations/:annotationId", role, updateCmd, func(sc *scenarioContext) { + sc.fakeReqWithParams("PUT", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + deleteAnnotationsScenario("When calling POST on", "/api/annotations/mass-delete", "/api/annotations/mass-delete", role, deleteCmd, func(sc *scenarioContext) { + sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + }) + }) }) } @@ -199,7 +223,7 @@ func postAnnotationScenario(desc string, url string, routePattern string, role m defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.UserId = TestUserID sc.context.OrgId = TestOrgID @@ -222,7 +246,7 @@ func putAnnotationScenario(desc string, url string, routePattern string, role m. defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.UserId = TestUserID sc.context.OrgId = TestOrgID @@ -239,3 +263,26 @@ func putAnnotationScenario(desc string, url string, routePattern string, role m. fn(sc) }) } + +func deleteAnnotationsScenario(desc string, url string, routePattern string, role m.RoleType, cmd dtos.DeleteAnnotationsCmd, fn scenarioFunc) { + Convey(desc+" "+url, func() { + defer bus.ClearBusHandlers() + + sc := setupScenarioContext(url) + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + sc.context.UserId = TestUserID + sc.context.OrgId = TestOrgID + sc.context.OrgRole = role + + return DeleteAnnotations(c, cmd) + }) + + fakeAnnoRepo = &fakeAnnotationsRepo{} + annotations.SetRepository(fakeAnnoRepo) + + sc.m.Post(routePattern, sc.defaultHandler) + + fn(sc) + }) +} diff --git a/pkg/api/api.go b/pkg/api/api.go index 5b3cde09fd5..c372debdb72 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -4,369 +4,379 @@ import ( "github.com/go-macaron/binding" "github.com/grafana/grafana/pkg/api/avatar" "github.com/grafana/grafana/pkg/api/dtos" + "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" ) -// Register adds http routes -func (hs *HttpServer) registerRoutes() { - macaronR := hs.macaron - reqSignedIn := middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true}) - reqGrafanaAdmin := middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true, ReqGrafanaAdmin: true}) - reqEditorRole := middleware.RoleAuth(m.ROLE_EDITOR, m.ROLE_ADMIN) - reqOrgAdmin := middleware.RoleAuth(m.ROLE_ADMIN) - redirectFromLegacyDashboardUrl := middleware.RedirectFromLegacyDashboardUrl() - redirectFromLegacyDashboardSoloUrl := middleware.RedirectFromLegacyDashboardSoloUrl() +func (hs *HTTPServer) registerRoutes() { + reqSignedIn := middleware.ReqSignedIn + reqGrafanaAdmin := middleware.ReqGrafanaAdmin + reqEditorRole := middleware.ReqEditorRole + reqOrgAdmin := middleware.ReqOrgAdmin + redirectFromLegacyDashboardURL := middleware.RedirectFromLegacyDashboardURL() + redirectFromLegacyDashboardSoloURL := middleware.RedirectFromLegacyDashboardSoloURL() quota := middleware.Quota bind := binding.Bind - // automatically set HEAD for every GET - macaronR.SetAutoHead(true) - - r := newRouteRegister(middleware.RequestMetrics, middleware.RequestTracing) + r := hs.RouteRegister // not logged in views - r.Get("/", reqSignedIn, Index) + r.Get("/", reqSignedIn, hs.Index) r.Get("/logout", Logout) - r.Post("/login", quota("session"), bind(dtos.LoginCommand{}), wrap(LoginPost)) + r.Post("/login", quota("session"), bind(dtos.LoginCommand{}), Wrap(LoginPost)) r.Get("/login/:name", quota("session"), OAuthLogin) - r.Get("/login", LoginView) - r.Get("/invite/:code", Index) + r.Get("/login", hs.LoginView) + r.Get("/invite/:code", hs.Index) // authed views - r.Get("/profile/", reqSignedIn, Index) - r.Get("/profile/password", reqSignedIn, Index) - r.Get("/profile/switch-org/:id", reqSignedIn, ChangeActiveOrgAndRedirectToHome) - r.Get("/org/", reqSignedIn, Index) - r.Get("/org/new", reqSignedIn, Index) - r.Get("/datasources/", reqSignedIn, Index) - r.Get("/datasources/new", reqSignedIn, Index) - r.Get("/datasources/edit/*", reqSignedIn, Index) - r.Get("/org/users", reqSignedIn, Index) - r.Get("/org/users/new", reqSignedIn, Index) - r.Get("/org/users/invite", reqSignedIn, Index) - r.Get("/org/teams", reqSignedIn, Index) - r.Get("/org/teams/*", reqSignedIn, Index) - r.Get("/org/apikeys/", reqSignedIn, Index) - r.Get("/dashboard/import/", reqSignedIn, Index) - r.Get("/configuration", reqGrafanaAdmin, Index) - r.Get("/admin", reqGrafanaAdmin, Index) - r.Get("/admin/settings", reqGrafanaAdmin, Index) - r.Get("/admin/users", reqGrafanaAdmin, Index) - r.Get("/admin/users/create", reqGrafanaAdmin, Index) - r.Get("/admin/users/edit/:id", reqGrafanaAdmin, Index) - r.Get("/admin/orgs", reqGrafanaAdmin, Index) - r.Get("/admin/orgs/edit/:id", reqGrafanaAdmin, Index) - r.Get("/admin/stats", reqGrafanaAdmin, Index) + r.Get("/profile/", reqSignedIn, hs.Index) + r.Get("/profile/password", reqSignedIn, hs.Index) + r.Get("/profile/switch-org/:id", reqSignedIn, hs.ChangeActiveOrgAndRedirectToHome) + r.Get("/org/", reqSignedIn, hs.Index) + r.Get("/org/new", reqSignedIn, hs.Index) + r.Get("/datasources/", reqSignedIn, hs.Index) + r.Get("/datasources/new", reqSignedIn, hs.Index) + r.Get("/datasources/edit/*", reqSignedIn, hs.Index) + r.Get("/org/users", reqSignedIn, hs.Index) + r.Get("/org/users/new", reqSignedIn, hs.Index) + r.Get("/org/users/invite", reqSignedIn, hs.Index) + r.Get("/org/teams", reqSignedIn, hs.Index) + r.Get("/org/teams/*", reqSignedIn, hs.Index) + r.Get("/org/apikeys/", reqSignedIn, hs.Index) + r.Get("/dashboard/import/", reqSignedIn, hs.Index) + r.Get("/configuration", reqGrafanaAdmin, hs.Index) + r.Get("/admin", reqGrafanaAdmin, hs.Index) + r.Get("/admin/settings", reqGrafanaAdmin, hs.Index) + r.Get("/admin/users", reqGrafanaAdmin, hs.Index) + r.Get("/admin/users/create", reqGrafanaAdmin, hs.Index) + r.Get("/admin/users/edit/:id", reqGrafanaAdmin, hs.Index) + r.Get("/admin/orgs", reqGrafanaAdmin, hs.Index) + r.Get("/admin/orgs/edit/:id", reqGrafanaAdmin, hs.Index) + r.Get("/admin/stats", reqGrafanaAdmin, hs.Index) - r.Get("/styleguide", reqSignedIn, Index) + r.Get("/styleguide", reqSignedIn, hs.Index) - r.Get("/plugins", reqSignedIn, Index) - r.Get("/plugins/:id/edit", reqSignedIn, Index) - r.Get("/plugins/:id/page/:page", reqSignedIn, Index) + r.Get("/plugins", reqSignedIn, hs.Index) + r.Get("/plugins/:id/edit", reqSignedIn, hs.Index) + r.Get("/plugins/:id/page/:page", reqSignedIn, hs.Index) - r.Get("/d/:uid/:slug", reqSignedIn, Index) - r.Get("/d/:uid", reqSignedIn, Index) - r.Get("/dashboard/db/:slug", reqSignedIn, redirectFromLegacyDashboardUrl, Index) - r.Get("/dashboard/script/*", reqSignedIn, Index) - r.Get("/dashboard-solo/snapshot/*", Index) - r.Get("/d-solo/:uid/:slug", reqSignedIn, Index) - r.Get("/dashboard-solo/db/:slug", reqSignedIn, redirectFromLegacyDashboardSoloUrl, Index) - r.Get("/dashboard-solo/script/*", reqSignedIn, Index) - r.Get("/import/dashboard", reqSignedIn, Index) - r.Get("/dashboards/", reqSignedIn, Index) - r.Get("/dashboards/*", reqSignedIn, Index) + r.Get("/d/:uid/:slug", reqSignedIn, hs.Index) + r.Get("/d/:uid", reqSignedIn, hs.Index) + r.Get("/dashboard/db/:slug", reqSignedIn, redirectFromLegacyDashboardURL, hs.Index) + r.Get("/dashboard/script/*", reqSignedIn, hs.Index) + r.Get("/dashboard-solo/snapshot/*", hs.Index) + r.Get("/d-solo/:uid/:slug", reqSignedIn, hs.Index) + r.Get("/dashboard-solo/db/:slug", reqSignedIn, redirectFromLegacyDashboardSoloURL, hs.Index) + r.Get("/dashboard-solo/script/*", reqSignedIn, hs.Index) + r.Get("/import/dashboard", reqSignedIn, hs.Index) + r.Get("/dashboards/", reqSignedIn, hs.Index) + r.Get("/dashboards/*", reqSignedIn, hs.Index) - r.Get("/playlists/", reqSignedIn, Index) - r.Get("/playlists/*", reqSignedIn, Index) - r.Get("/alerting/", reqSignedIn, Index) - r.Get("/alerting/*", reqSignedIn, Index) + r.Get("/explore", reqEditorRole, hs.Index) + + r.Get("/playlists/", reqSignedIn, hs.Index) + r.Get("/playlists/*", reqSignedIn, hs.Index) + r.Get("/alerting/", reqSignedIn, hs.Index) + r.Get("/alerting/*", reqSignedIn, hs.Index) // sign up - r.Get("/signup", Index) - r.Get("/api/user/signup/options", wrap(GetSignUpOptions)) - r.Post("/api/user/signup", quota("user"), bind(dtos.SignUpForm{}), wrap(SignUp)) - r.Post("/api/user/signup/step2", bind(dtos.SignUpStep2Form{}), wrap(SignUpStep2)) + r.Get("/signup", hs.Index) + r.Get("/api/user/signup/options", Wrap(GetSignUpOptions)) + r.Post("/api/user/signup", quota("user"), bind(dtos.SignUpForm{}), Wrap(SignUp)) + r.Post("/api/user/signup/step2", bind(dtos.SignUpStep2Form{}), Wrap(SignUpStep2)) // invited - r.Get("/api/user/invite/:code", wrap(GetInviteInfoByCode)) - r.Post("/api/user/invite/complete", bind(dtos.CompleteInviteForm{}), wrap(CompleteInvite)) + r.Get("/api/user/invite/:code", Wrap(GetInviteInfoByCode)) + r.Post("/api/user/invite/complete", bind(dtos.CompleteInviteForm{}), Wrap(CompleteInvite)) // reset password - r.Get("/user/password/send-reset-email", Index) - r.Get("/user/password/reset", Index) + r.Get("/user/password/send-reset-email", hs.Index) + r.Get("/user/password/reset", hs.Index) - r.Post("/api/user/password/send-reset-email", bind(dtos.SendResetPasswordEmailForm{}), wrap(SendResetPasswordEmail)) - r.Post("/api/user/password/reset", bind(dtos.ResetUserPasswordForm{}), wrap(ResetPassword)) + r.Post("/api/user/password/send-reset-email", bind(dtos.SendResetPasswordEmailForm{}), Wrap(SendResetPasswordEmail)) + r.Post("/api/user/password/reset", bind(dtos.ResetUserPasswordForm{}), Wrap(ResetPassword)) // dashboard snapshots - r.Get("/dashboard/snapshot/*", Index) - r.Get("/dashboard/snapshots/", reqSignedIn, Index) + r.Get("/dashboard/snapshot/*", hs.Index) + r.Get("/dashboard/snapshots/", reqSignedIn, hs.Index) // api for dashboard snapshots r.Post("/api/snapshots/", bind(m.CreateDashboardSnapshotCommand{}), CreateDashboardSnapshot) r.Get("/api/snapshot/shared-options/", GetSharingOptions) r.Get("/api/snapshots/:key", GetDashboardSnapshot) - r.Get("/api/snapshots-delete/:key", reqEditorRole, wrap(DeleteDashboardSnapshot)) + r.Get("/api/snapshots-delete/:deleteKey", Wrap(DeleteDashboardSnapshotByDeleteKey)) + r.Delete("/api/snapshots/:key", reqEditorRole, Wrap(DeleteDashboardSnapshot)) // api renew session based on remember cookie - r.Get("/api/login/ping", quota("session"), LoginApiPing) + r.Get("/api/login/ping", quota("session"), LoginAPIPing) // authed api - r.Group("/api", func(apiRoute RouteRegister) { + r.Group("/api", func(apiRoute routing.RouteRegister) { // user (signed in) - apiRoute.Group("/user", func(userRoute RouteRegister) { - userRoute.Get("/", wrap(GetSignedInUser)) - userRoute.Put("/", bind(m.UpdateUserCommand{}), wrap(UpdateSignedInUser)) - userRoute.Post("/using/:id", wrap(UserSetUsingOrg)) - userRoute.Get("/orgs", wrap(GetSignedInUserOrgList)) + apiRoute.Group("/user", func(userRoute routing.RouteRegister) { + userRoute.Get("/", Wrap(GetSignedInUser)) + userRoute.Put("/", bind(m.UpdateUserCommand{}), Wrap(UpdateSignedInUser)) + userRoute.Post("/using/:id", Wrap(UserSetUsingOrg)) + userRoute.Get("/orgs", Wrap(GetSignedInUserOrgList)) + userRoute.Get("/teams", Wrap(GetSignedInUserTeamList)) - userRoute.Post("/stars/dashboard/:id", wrap(StarDashboard)) - userRoute.Delete("/stars/dashboard/:id", wrap(UnstarDashboard)) + userRoute.Post("/stars/dashboard/:id", Wrap(StarDashboard)) + userRoute.Delete("/stars/dashboard/:id", Wrap(UnstarDashboard)) - userRoute.Put("/password", bind(m.ChangeUserPasswordCommand{}), wrap(ChangeUserPassword)) - userRoute.Get("/quotas", wrap(GetUserQuotas)) - userRoute.Put("/helpflags/:id", wrap(SetHelpFlag)) + userRoute.Put("/password", bind(m.ChangeUserPasswordCommand{}), Wrap(ChangeUserPassword)) + userRoute.Get("/quotas", Wrap(GetUserQuotas)) + userRoute.Put("/helpflags/:id", Wrap(SetHelpFlag)) // For dev purpose - userRoute.Get("/helpflags/clear", wrap(ClearHelpFlags)) + userRoute.Get("/helpflags/clear", Wrap(ClearHelpFlags)) - userRoute.Get("/preferences", wrap(GetUserPreferences)) - userRoute.Put("/preferences", bind(dtos.UpdatePrefsCmd{}), wrap(UpdateUserPreferences)) + userRoute.Get("/preferences", Wrap(GetUserPreferences)) + userRoute.Put("/preferences", bind(dtos.UpdatePrefsCmd{}), Wrap(UpdateUserPreferences)) }) // users (admin permission required) - apiRoute.Group("/users", func(usersRoute RouteRegister) { - usersRoute.Get("/", wrap(SearchUsers)) - usersRoute.Get("/search", wrap(SearchUsersWithPaging)) - usersRoute.Get("/:id", wrap(GetUserById)) - usersRoute.Get("/:id/orgs", wrap(GetUserOrgList)) + apiRoute.Group("/users", func(usersRoute routing.RouteRegister) { + usersRoute.Get("/", Wrap(SearchUsers)) + usersRoute.Get("/search", Wrap(SearchUsersWithPaging)) + usersRoute.Get("/:id", Wrap(GetUserByID)) + usersRoute.Get("/:id/orgs", Wrap(GetUserOrgList)) // query parameters /users/lookup?loginOrEmail=admin@example.com - usersRoute.Get("/lookup", wrap(GetUserByLoginOrEmail)) - usersRoute.Put("/:id", bind(m.UpdateUserCommand{}), wrap(UpdateUser)) - usersRoute.Post("/:id/using/:orgId", wrap(UpdateUserActiveOrg)) + usersRoute.Get("/lookup", Wrap(GetUserByLoginOrEmail)) + usersRoute.Put("/:id", bind(m.UpdateUserCommand{}), Wrap(UpdateUser)) + usersRoute.Post("/:id/using/:orgId", Wrap(UpdateUserActiveOrg)) }, reqGrafanaAdmin) // team (admin permission required) - apiRoute.Group("/teams", func(teamsRoute RouteRegister) { - teamsRoute.Get("/:teamId", wrap(GetTeamById)) - teamsRoute.Get("/search", wrap(SearchTeams)) - teamsRoute.Post("/", bind(m.CreateTeamCommand{}), wrap(CreateTeam)) - teamsRoute.Put("/:teamId", bind(m.UpdateTeamCommand{}), wrap(UpdateTeam)) - teamsRoute.Delete("/:teamId", wrap(DeleteTeamById)) - teamsRoute.Get("/:teamId/members", wrap(GetTeamMembers)) - teamsRoute.Post("/:teamId/members", bind(m.AddTeamMemberCommand{}), wrap(AddTeamMember)) - teamsRoute.Delete("/:teamId/members/:userId", wrap(RemoveTeamMember)) + apiRoute.Group("/teams", func(teamsRoute routing.RouteRegister) { + teamsRoute.Post("/", bind(m.CreateTeamCommand{}), Wrap(CreateTeam)) + teamsRoute.Put("/:teamId", bind(m.UpdateTeamCommand{}), Wrap(UpdateTeam)) + teamsRoute.Delete("/:teamId", Wrap(DeleteTeamByID)) + teamsRoute.Get("/:teamId/members", Wrap(GetTeamMembers)) + teamsRoute.Post("/:teamId/members", bind(m.AddTeamMemberCommand{}), Wrap(AddTeamMember)) + teamsRoute.Delete("/:teamId/members/:userId", Wrap(RemoveTeamMember)) + teamsRoute.Get("/:teamId/preferences", Wrap(GetTeamPreferences)) + teamsRoute.Put("/:teamId/preferences", bind(dtos.UpdatePrefsCmd{}), Wrap(UpdateTeamPreferences)) }, reqOrgAdmin) + // team without requirement of user to be org admin + apiRoute.Group("/teams", func(teamsRoute routing.RouteRegister) { + teamsRoute.Get("/:teamId", Wrap(GetTeamByID)) + teamsRoute.Get("/search", Wrap(SearchTeams)) + }) + // org information available to all users. - apiRoute.Group("/org", func(orgRoute RouteRegister) { - orgRoute.Get("/", wrap(GetOrgCurrent)) - orgRoute.Get("/quotas", wrap(GetOrgQuotas)) + apiRoute.Group("/org", func(orgRoute routing.RouteRegister) { + orgRoute.Get("/", Wrap(GetOrgCurrent)) + orgRoute.Get("/quotas", Wrap(GetOrgQuotas)) }) // current org - apiRoute.Group("/org", func(orgRoute RouteRegister) { - orgRoute.Put("/", bind(dtos.UpdateOrgForm{}), wrap(UpdateOrgCurrent)) - orgRoute.Put("/address", bind(dtos.UpdateOrgAddressForm{}), wrap(UpdateOrgAddressCurrent)) - orgRoute.Post("/users", quota("user"), bind(m.AddOrgUserCommand{}), wrap(AddOrgUserToCurrentOrg)) - orgRoute.Get("/users", wrap(GetOrgUsersForCurrentOrg)) - orgRoute.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), wrap(UpdateOrgUserForCurrentOrg)) - orgRoute.Delete("/users/:userId", wrap(RemoveOrgUserForCurrentOrg)) + apiRoute.Group("/org", func(orgRoute routing.RouteRegister) { + orgRoute.Put("/", bind(dtos.UpdateOrgForm{}), Wrap(UpdateOrgCurrent)) + orgRoute.Put("/address", bind(dtos.UpdateOrgAddressForm{}), Wrap(UpdateOrgAddressCurrent)) + orgRoute.Post("/users", quota("user"), bind(m.AddOrgUserCommand{}), Wrap(AddOrgUserToCurrentOrg)) + orgRoute.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), Wrap(UpdateOrgUserForCurrentOrg)) + orgRoute.Delete("/users/:userId", Wrap(RemoveOrgUserForCurrentOrg)) // invites - orgRoute.Get("/invites", wrap(GetPendingOrgInvites)) - orgRoute.Post("/invites", quota("user"), bind(dtos.AddInviteForm{}), wrap(AddOrgInvite)) - orgRoute.Patch("/invites/:code/revoke", wrap(RevokeInvite)) + orgRoute.Get("/invites", Wrap(GetPendingOrgInvites)) + orgRoute.Post("/invites", quota("user"), bind(dtos.AddInviteForm{}), Wrap(AddOrgInvite)) + orgRoute.Patch("/invites/:code/revoke", Wrap(RevokeInvite)) // prefs - orgRoute.Get("/preferences", wrap(GetOrgPreferences)) - orgRoute.Put("/preferences", bind(dtos.UpdatePrefsCmd{}), wrap(UpdateOrgPreferences)) + orgRoute.Get("/preferences", Wrap(GetOrgPreferences)) + orgRoute.Put("/preferences", bind(dtos.UpdatePrefsCmd{}), Wrap(UpdateOrgPreferences)) }, reqOrgAdmin) + // current org without requirement of user to be org admin + apiRoute.Group("/org", func(orgRoute routing.RouteRegister) { + orgRoute.Get("/users", Wrap(GetOrgUsersForCurrentOrg)) + }) + // create new org - apiRoute.Post("/orgs", quota("org"), bind(m.CreateOrgCommand{}), wrap(CreateOrg)) + apiRoute.Post("/orgs", quota("org"), bind(m.CreateOrgCommand{}), Wrap(CreateOrg)) // search all orgs - apiRoute.Get("/orgs", reqGrafanaAdmin, wrap(SearchOrgs)) + apiRoute.Get("/orgs", reqGrafanaAdmin, Wrap(SearchOrgs)) // orgs (admin routes) - apiRoute.Group("/orgs/:orgId", func(orgsRoute RouteRegister) { - orgsRoute.Get("/", wrap(GetOrgById)) - orgsRoute.Put("/", bind(dtos.UpdateOrgForm{}), wrap(UpdateOrg)) - orgsRoute.Put("/address", bind(dtos.UpdateOrgAddressForm{}), wrap(UpdateOrgAddress)) - orgsRoute.Delete("/", wrap(DeleteOrgById)) - orgsRoute.Get("/users", wrap(GetOrgUsers)) - orgsRoute.Post("/users", bind(m.AddOrgUserCommand{}), wrap(AddOrgUser)) - orgsRoute.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), wrap(UpdateOrgUser)) - orgsRoute.Delete("/users/:userId", wrap(RemoveOrgUser)) - orgsRoute.Get("/quotas", wrap(GetOrgQuotas)) - orgsRoute.Put("/quotas/:target", bind(m.UpdateOrgQuotaCmd{}), wrap(UpdateOrgQuota)) + apiRoute.Group("/orgs/:orgId", func(orgsRoute routing.RouteRegister) { + orgsRoute.Get("/", Wrap(GetOrgByID)) + orgsRoute.Put("/", bind(dtos.UpdateOrgForm{}), Wrap(UpdateOrg)) + orgsRoute.Put("/address", bind(dtos.UpdateOrgAddressForm{}), Wrap(UpdateOrgAddress)) + orgsRoute.Delete("/", Wrap(DeleteOrgByID)) + orgsRoute.Get("/users", Wrap(GetOrgUsers)) + orgsRoute.Post("/users", bind(m.AddOrgUserCommand{}), Wrap(AddOrgUser)) + orgsRoute.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), Wrap(UpdateOrgUser)) + orgsRoute.Delete("/users/:userId", Wrap(RemoveOrgUser)) + orgsRoute.Get("/quotas", Wrap(GetOrgQuotas)) + orgsRoute.Put("/quotas/:target", bind(m.UpdateOrgQuotaCmd{}), Wrap(UpdateOrgQuota)) }, reqGrafanaAdmin) // orgs (admin routes) - apiRoute.Group("/orgs/name/:name", func(orgsRoute RouteRegister) { - orgsRoute.Get("/", wrap(GetOrgByName)) + apiRoute.Group("/orgs/name/:name", func(orgsRoute routing.RouteRegister) { + orgsRoute.Get("/", Wrap(GetOrgByName)) }, reqGrafanaAdmin) // auth api keys - apiRoute.Group("/auth/keys", func(keysRoute RouteRegister) { - keysRoute.Get("/", wrap(GetApiKeys)) - keysRoute.Post("/", quota("api_key"), bind(m.AddApiKeyCommand{}), wrap(AddApiKey)) - keysRoute.Delete("/:id", wrap(DeleteApiKey)) + apiRoute.Group("/auth/keys", func(keysRoute routing.RouteRegister) { + keysRoute.Get("/", Wrap(GetAPIKeys)) + keysRoute.Post("/", quota("api_key"), bind(m.AddApiKeyCommand{}), Wrap(AddAPIKey)) + keysRoute.Delete("/:id", Wrap(DeleteAPIKey)) }, reqOrgAdmin) // Preferences - apiRoute.Group("/preferences", func(prefRoute RouteRegister) { - prefRoute.Post("/set-home-dash", bind(m.SavePreferencesCommand{}), wrap(SetHomeDashboard)) + apiRoute.Group("/preferences", func(prefRoute routing.RouteRegister) { + prefRoute.Post("/set-home-dash", bind(m.SavePreferencesCommand{}), Wrap(SetHomeDashboard)) }) // Data sources - apiRoute.Group("/datasources", func(datasourceRoute RouteRegister) { - datasourceRoute.Get("/", wrap(GetDataSources)) - datasourceRoute.Post("/", quota("data_source"), bind(m.AddDataSourceCommand{}), wrap(AddDataSource)) - datasourceRoute.Put("/:id", bind(m.UpdateDataSourceCommand{}), wrap(UpdateDataSource)) - datasourceRoute.Delete("/:id", wrap(DeleteDataSourceById)) - datasourceRoute.Delete("/name/:name", wrap(DeleteDataSourceByName)) - datasourceRoute.Get("/:id", wrap(GetDataSourceById)) - datasourceRoute.Get("/name/:name", wrap(GetDataSourceByName)) + apiRoute.Group("/datasources", func(datasourceRoute routing.RouteRegister) { + datasourceRoute.Get("/", Wrap(GetDataSources)) + datasourceRoute.Post("/", quota("data_source"), bind(m.AddDataSourceCommand{}), Wrap(AddDataSource)) + datasourceRoute.Put("/:id", bind(m.UpdateDataSourceCommand{}), Wrap(UpdateDataSource)) + datasourceRoute.Delete("/:id", Wrap(DeleteDataSourceById)) + datasourceRoute.Delete("/name/:name", Wrap(DeleteDataSourceByName)) + datasourceRoute.Get("/:id", Wrap(GetDataSourceById)) + datasourceRoute.Get("/name/:name", Wrap(GetDataSourceByName)) }, reqOrgAdmin) - apiRoute.Get("/datasources/id/:name", wrap(GetDataSourceIdByName), reqSignedIn) + apiRoute.Get("/datasources/id/:name", Wrap(GetDataSourceIdByName), reqSignedIn) - apiRoute.Get("/plugins", wrap(GetPluginList)) - apiRoute.Get("/plugins/:pluginId/settings", wrap(GetPluginSettingById)) - apiRoute.Get("/plugins/:pluginId/markdown/:name", wrap(GetPluginMarkdown)) + apiRoute.Get("/plugins", Wrap(hs.GetPluginList)) + apiRoute.Get("/plugins/:pluginId/settings", Wrap(GetPluginSettingByID)) + apiRoute.Get("/plugins/:pluginId/markdown/:name", Wrap(GetPluginMarkdown)) - apiRoute.Group("/plugins", func(pluginRoute RouteRegister) { - pluginRoute.Get("/:pluginId/dashboards/", wrap(GetPluginDashboards)) - pluginRoute.Post("/:pluginId/settings", bind(m.UpdatePluginSettingCmd{}), wrap(UpdatePluginSetting)) + apiRoute.Group("/plugins", func(pluginRoute routing.RouteRegister) { + pluginRoute.Get("/:pluginId/dashboards/", Wrap(GetPluginDashboards)) + pluginRoute.Post("/:pluginId/settings", bind(m.UpdatePluginSettingCmd{}), Wrap(UpdatePluginSetting)) }, reqOrgAdmin) - apiRoute.Get("/frontend/settings/", GetFrontendSettings) + apiRoute.Get("/frontend/settings/", hs.GetFrontendSettings) apiRoute.Any("/datasources/proxy/:id/*", reqSignedIn, hs.ProxyDataSourceRequest) apiRoute.Any("/datasources/proxy/:id", reqSignedIn, hs.ProxyDataSourceRequest) // Folders - apiRoute.Group("/folders", func(folderRoute RouteRegister) { - folderRoute.Get("/", wrap(GetFolders)) - folderRoute.Get("/id/:id", wrap(GetFolderById)) - folderRoute.Post("/", bind(m.CreateFolderCommand{}), wrap(CreateFolder)) + apiRoute.Group("/folders", func(folderRoute routing.RouteRegister) { + folderRoute.Get("/", Wrap(GetFolders)) + folderRoute.Get("/id/:id", Wrap(GetFolderByID)) + folderRoute.Post("/", bind(m.CreateFolderCommand{}), Wrap(CreateFolder)) - folderRoute.Group("/:uid", func(folderUidRoute RouteRegister) { - folderUidRoute.Get("/", wrap(GetFolderByUid)) - folderUidRoute.Put("/", bind(m.UpdateFolderCommand{}), wrap(UpdateFolder)) - folderUidRoute.Delete("/", wrap(DeleteFolder)) + folderRoute.Group("/:uid", func(folderUidRoute routing.RouteRegister) { + folderUidRoute.Get("/", Wrap(GetFolderByUID)) + folderUidRoute.Put("/", bind(m.UpdateFolderCommand{}), Wrap(UpdateFolder)) + folderUidRoute.Delete("/", Wrap(DeleteFolder)) - folderUidRoute.Group("/permissions", func(folderPermissionRoute RouteRegister) { - folderPermissionRoute.Get("/", wrap(GetFolderPermissionList)) - folderPermissionRoute.Post("/", bind(dtos.UpdateDashboardAclCommand{}), wrap(UpdateFolderPermissions)) + folderUidRoute.Group("/permissions", func(folderPermissionRoute routing.RouteRegister) { + folderPermissionRoute.Get("/", Wrap(GetFolderPermissionList)) + folderPermissionRoute.Post("/", bind(dtos.UpdateDashboardAclCommand{}), Wrap(UpdateFolderPermissions)) }) }) }) // Dashboard - apiRoute.Group("/dashboards", func(dashboardRoute RouteRegister) { - dashboardRoute.Get("/uid/:uid", wrap(GetDashboard)) - dashboardRoute.Delete("/uid/:uid", wrap(DeleteDashboardByUid)) + apiRoute.Group("/dashboards", func(dashboardRoute routing.RouteRegister) { + dashboardRoute.Get("/uid/:uid", Wrap(GetDashboard)) + dashboardRoute.Delete("/uid/:uid", Wrap(DeleteDashboardByUID)) - dashboardRoute.Get("/db/:slug", wrap(GetDashboard)) - dashboardRoute.Delete("/db/:slug", wrap(DeleteDashboard)) + dashboardRoute.Get("/db/:slug", Wrap(GetDashboard)) + dashboardRoute.Delete("/db/:slug", Wrap(DeleteDashboard)) - dashboardRoute.Post("/calculate-diff", bind(dtos.CalculateDiffOptions{}), wrap(CalculateDashboardDiff)) + dashboardRoute.Post("/calculate-diff", bind(dtos.CalculateDiffOptions{}), Wrap(CalculateDashboardDiff)) - dashboardRoute.Post("/db", bind(m.SaveDashboardCommand{}), wrap(PostDashboard)) - dashboardRoute.Get("/home", wrap(GetHomeDashboard)) + dashboardRoute.Post("/db", bind(m.SaveDashboardCommand{}), Wrap(PostDashboard)) + dashboardRoute.Get("/home", Wrap(GetHomeDashboard)) dashboardRoute.Get("/tags", GetDashboardTags) - dashboardRoute.Post("/import", bind(dtos.ImportDashboardCommand{}), wrap(ImportDashboard)) + dashboardRoute.Post("/import", bind(dtos.ImportDashboardCommand{}), Wrap(ImportDashboard)) - dashboardRoute.Group("/id/:dashboardId", func(dashIdRoute RouteRegister) { - dashIdRoute.Get("/versions", wrap(GetDashboardVersions)) - dashIdRoute.Get("/versions/:id", wrap(GetDashboardVersion)) - dashIdRoute.Post("/restore", bind(dtos.RestoreDashboardVersionCommand{}), wrap(RestoreDashboardVersion)) + dashboardRoute.Group("/id/:dashboardId", func(dashIdRoute routing.RouteRegister) { + dashIdRoute.Get("/versions", Wrap(GetDashboardVersions)) + dashIdRoute.Get("/versions/:id", Wrap(GetDashboardVersion)) + dashIdRoute.Post("/restore", bind(dtos.RestoreDashboardVersionCommand{}), Wrap(RestoreDashboardVersion)) - dashIdRoute.Group("/permissions", func(dashboardPermissionRoute RouteRegister) { - dashboardPermissionRoute.Get("/", wrap(GetDashboardPermissionList)) - dashboardPermissionRoute.Post("/", bind(dtos.UpdateDashboardAclCommand{}), wrap(UpdateDashboardPermissions)) + dashIdRoute.Group("/permissions", func(dashboardPermissionRoute routing.RouteRegister) { + dashboardPermissionRoute.Get("/", Wrap(GetDashboardPermissionList)) + dashboardPermissionRoute.Post("/", bind(dtos.UpdateDashboardAclCommand{}), Wrap(UpdateDashboardPermissions)) }) }) }) // Dashboard snapshots - apiRoute.Group("/dashboard/snapshots", func(dashboardRoute RouteRegister) { - dashboardRoute.Get("/", wrap(SearchDashboardSnapshots)) + apiRoute.Group("/dashboard/snapshots", func(dashboardRoute routing.RouteRegister) { + dashboardRoute.Get("/", Wrap(SearchDashboardSnapshots)) }) // Playlist - apiRoute.Group("/playlists", func(playlistRoute RouteRegister) { - playlistRoute.Get("/", wrap(SearchPlaylists)) - playlistRoute.Get("/:id", ValidateOrgPlaylist, wrap(GetPlaylist)) - playlistRoute.Get("/:id/items", ValidateOrgPlaylist, wrap(GetPlaylistItems)) - playlistRoute.Get("/:id/dashboards", ValidateOrgPlaylist, wrap(GetPlaylistDashboards)) - playlistRoute.Delete("/:id", reqEditorRole, ValidateOrgPlaylist, wrap(DeletePlaylist)) - playlistRoute.Put("/:id", reqEditorRole, bind(m.UpdatePlaylistCommand{}), ValidateOrgPlaylist, wrap(UpdatePlaylist)) - playlistRoute.Post("/", reqEditorRole, bind(m.CreatePlaylistCommand{}), wrap(CreatePlaylist)) + apiRoute.Group("/playlists", func(playlistRoute routing.RouteRegister) { + playlistRoute.Get("/", Wrap(SearchPlaylists)) + playlistRoute.Get("/:id", ValidateOrgPlaylist, Wrap(GetPlaylist)) + playlistRoute.Get("/:id/items", ValidateOrgPlaylist, Wrap(GetPlaylistItems)) + playlistRoute.Get("/:id/dashboards", ValidateOrgPlaylist, Wrap(GetPlaylistDashboards)) + playlistRoute.Delete("/:id", reqEditorRole, ValidateOrgPlaylist, Wrap(DeletePlaylist)) + playlistRoute.Put("/:id", reqEditorRole, bind(m.UpdatePlaylistCommand{}), ValidateOrgPlaylist, Wrap(UpdatePlaylist)) + playlistRoute.Post("/", reqEditorRole, bind(m.CreatePlaylistCommand{}), Wrap(CreatePlaylist)) }) // Search apiRoute.Get("/search/", Search) // metrics - apiRoute.Post("/tsdb/query", bind(dtos.MetricRequest{}), wrap(QueryMetrics)) - apiRoute.Get("/tsdb/testdata/scenarios", wrap(GetTestDataScenarios)) - apiRoute.Get("/tsdb/testdata/gensql", reqGrafanaAdmin, wrap(GenerateSqlTestData)) - apiRoute.Get("/tsdb/testdata/random-walk", wrap(GetTestDataRandomWalk)) + apiRoute.Post("/tsdb/query", bind(dtos.MetricRequest{}), Wrap(hs.QueryMetrics)) + apiRoute.Get("/tsdb/testdata/scenarios", Wrap(GetTestDataScenarios)) + apiRoute.Get("/tsdb/testdata/gensql", reqGrafanaAdmin, Wrap(GenerateSQLTestData)) + apiRoute.Get("/tsdb/testdata/random-walk", Wrap(GetTestDataRandomWalk)) - apiRoute.Group("/alerts", func(alertsRoute RouteRegister) { - alertsRoute.Post("/test", bind(dtos.AlertTestCommand{}), wrap(AlertTest)) - alertsRoute.Post("/:alertId/pause", reqEditorRole, bind(dtos.PauseAlertCommand{}), wrap(PauseAlert)) - alertsRoute.Get("/:alertId", ValidateOrgAlert, wrap(GetAlert)) - alertsRoute.Get("/", wrap(GetAlerts)) - alertsRoute.Get("/states-for-dashboard", wrap(GetAlertStatesForDashboard)) + apiRoute.Group("/alerts", func(alertsRoute routing.RouteRegister) { + alertsRoute.Post("/test", bind(dtos.AlertTestCommand{}), Wrap(AlertTest)) + alertsRoute.Post("/:alertId/pause", reqEditorRole, bind(dtos.PauseAlertCommand{}), Wrap(PauseAlert)) + alertsRoute.Get("/:alertId", ValidateOrgAlert, Wrap(GetAlert)) + alertsRoute.Get("/", Wrap(GetAlerts)) + alertsRoute.Get("/states-for-dashboard", Wrap(GetAlertStatesForDashboard)) }) - apiRoute.Get("/alert-notifications", wrap(GetAlertNotifications)) - apiRoute.Get("/alert-notifiers", wrap(GetAlertNotifiers)) + apiRoute.Get("/alert-notifications", Wrap(GetAlertNotifications)) + apiRoute.Get("/alert-notifiers", Wrap(GetAlertNotifiers)) - apiRoute.Group("/alert-notifications", func(alertNotifications RouteRegister) { - alertNotifications.Post("/test", bind(dtos.NotificationTestCommand{}), wrap(NotificationTest)) - alertNotifications.Post("/", bind(m.CreateAlertNotificationCommand{}), wrap(CreateAlertNotification)) - alertNotifications.Put("/:notificationId", bind(m.UpdateAlertNotificationCommand{}), wrap(UpdateAlertNotification)) - alertNotifications.Get("/:notificationId", wrap(GetAlertNotificationById)) - alertNotifications.Delete("/:notificationId", wrap(DeleteAlertNotification)) + apiRoute.Group("/alert-notifications", func(alertNotifications routing.RouteRegister) { + alertNotifications.Post("/test", bind(dtos.NotificationTestCommand{}), Wrap(NotificationTest)) + alertNotifications.Post("/", bind(m.CreateAlertNotificationCommand{}), Wrap(CreateAlertNotification)) + alertNotifications.Put("/:notificationId", bind(m.UpdateAlertNotificationCommand{}), Wrap(UpdateAlertNotification)) + alertNotifications.Get("/:notificationId", Wrap(GetAlertNotificationByID)) + alertNotifications.Delete("/:notificationId", Wrap(DeleteAlertNotification)) }, reqEditorRole) - apiRoute.Get("/annotations", wrap(GetAnnotations)) - apiRoute.Post("/annotations/mass-delete", reqOrgAdmin, bind(dtos.DeleteAnnotationsCmd{}), wrap(DeleteAnnotations)) + apiRoute.Get("/annotations", Wrap(GetAnnotations)) + apiRoute.Post("/annotations/mass-delete", reqOrgAdmin, bind(dtos.DeleteAnnotationsCmd{}), Wrap(DeleteAnnotations)) - apiRoute.Group("/annotations", func(annotationsRoute RouteRegister) { - annotationsRoute.Post("/", bind(dtos.PostAnnotationsCmd{}), wrap(PostAnnotation)) - annotationsRoute.Delete("/:annotationId", wrap(DeleteAnnotationById)) - annotationsRoute.Put("/:annotationId", bind(dtos.UpdateAnnotationsCmd{}), wrap(UpdateAnnotation)) - annotationsRoute.Delete("/region/:regionId", wrap(DeleteAnnotationRegion)) - annotationsRoute.Post("/graphite", reqEditorRole, bind(dtos.PostGraphiteAnnotationsCmd{}), wrap(PostGraphiteAnnotation)) + apiRoute.Group("/annotations", func(annotationsRoute routing.RouteRegister) { + annotationsRoute.Post("/", bind(dtos.PostAnnotationsCmd{}), Wrap(PostAnnotation)) + annotationsRoute.Delete("/:annotationId", Wrap(DeleteAnnotationByID)) + annotationsRoute.Put("/:annotationId", bind(dtos.UpdateAnnotationsCmd{}), Wrap(UpdateAnnotation)) + annotationsRoute.Delete("/region/:regionId", Wrap(DeleteAnnotationRegion)) + annotationsRoute.Post("/graphite", reqEditorRole, bind(dtos.PostGraphiteAnnotationsCmd{}), Wrap(PostGraphiteAnnotation)) }) // error test - r.Get("/metrics/error", wrap(GenerateError)) + r.Get("/metrics/error", Wrap(GenerateError)) }, reqSignedIn) // admin api - r.Group("/api/admin", func(adminRoute RouteRegister) { + r.Group("/api/admin", func(adminRoute routing.RouteRegister) { adminRoute.Get("/settings", AdminGetSettings) adminRoute.Post("/users", bind(dtos.AdminCreateUserForm{}), AdminCreateUser) adminRoute.Put("/users/:id/password", bind(dtos.AdminUpdateUserPasswordForm{}), AdminUpdateUserPassword) adminRoute.Put("/users/:id/permissions", bind(dtos.AdminUpdateUserPermissionsForm{}), AdminUpdateUserPermissions) adminRoute.Delete("/users/:id", AdminDeleteUser) - adminRoute.Get("/users/:id/quotas", wrap(GetUserQuotas)) - adminRoute.Put("/users/:id/quotas/:target", bind(m.UpdateUserQuotaCmd{}), wrap(UpdateUserQuota)) + adminRoute.Get("/users/:id/quotas", Wrap(GetUserQuotas)) + adminRoute.Put("/users/:id/quotas/:target", bind(m.UpdateUserQuotaCmd{}), Wrap(UpdateUserQuota)) adminRoute.Get("/stats", AdminGetStats) - adminRoute.Post("/pause-all-alerts", bind(dtos.PauseAllAlertsCommand{}), wrap(PauseAllAlerts)) + adminRoute.Post("/pause-all-alerts", bind(dtos.PauseAllAlertsCommand{}), Wrap(PauseAllAlerts)) }, reqGrafanaAdmin) // rendering - r.Get("/render/*", reqSignedIn, RenderToPng) + r.Get("/render/*", reqSignedIn, hs.RenderToPng) // grafana.net proxy r.Any("/api/gnet/*", reqSignedIn, ProxyGnetRequest) @@ -380,10 +390,4 @@ func (hs *HttpServer) registerRoutes() { // streams //r.Post("/api/streams/push", reqSignedIn, bind(dtos.StreamMessage{}), liveConn.PushToStream) - - r.Register(macaronR) - - InitAppPluginRoutes(macaronR) - - macaronR.NotFound(NotFoundHandler) } diff --git a/pkg/api/apikey.go b/pkg/api/apikey.go index 24ed69ec691..7fda738f1cd 100644 --- a/pkg/api/apikey.go +++ b/pkg/api/apikey.go @@ -7,11 +7,11 @@ import ( m "github.com/grafana/grafana/pkg/models" ) -func GetApiKeys(c *m.ReqContext) Response { +func GetAPIKeys(c *m.ReqContext) Response { query := m.GetApiKeysQuery{OrgId: c.OrgId} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to list api keys", err) + return Error(500, "Failed to list api keys", err) } result := make([]*m.ApiKeyDTO, len(query.Result)) @@ -23,25 +23,25 @@ func GetApiKeys(c *m.ReqContext) Response { } } - return Json(200, result) + return JSON(200, result) } -func DeleteApiKey(c *m.ReqContext) Response { +func DeleteAPIKey(c *m.ReqContext) Response { id := c.ParamsInt64(":id") cmd := &m.DeleteApiKeyCommand{Id: id, OrgId: c.OrgId} err := bus.Dispatch(cmd) if err != nil { - return ApiError(500, "Failed to delete API key", err) + return Error(500, "Failed to delete API key", err) } - return ApiSuccess("API key deleted") + return Success("API key deleted") } -func AddApiKey(c *m.ReqContext, cmd m.AddApiKeyCommand) Response { +func AddAPIKey(c *m.ReqContext, cmd m.AddApiKeyCommand) Response { if !cmd.Role.IsValid() { - return ApiError(400, "Invalid role specified", nil) + return Error(400, "Invalid role specified", nil) } cmd.OrgId = c.OrgId @@ -50,12 +50,12 @@ func AddApiKey(c *m.ReqContext, cmd m.AddApiKeyCommand) Response { cmd.Key = newKeyInfo.HashedKey if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to add API key", err) + return Error(500, "Failed to add API key", err) } result := &dtos.NewApiKeyResult{ Name: cmd.Result.Name, Key: newKeyInfo.ClientSecret} - return Json(200, result) + return JSON(200, result) } diff --git a/pkg/api/app_routes.go b/pkg/api/app_routes.go index 8d74d96396b..a2137089fc6 100644 --- a/pkg/api/app_routes.go +++ b/pkg/api/app_routes.go @@ -18,7 +18,7 @@ import ( var pluginProxyTransport *http.Transport -func InitAppPluginRoutes(r *macaron.Macaron) { +func (hs *HTTPServer) initAppPluginRoutes(r *macaron.Macaron) { pluginProxyTransport = &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: setting.PluginAppsSkipVerifyTLS, @@ -55,11 +55,11 @@ func InitAppPluginRoutes(r *macaron.Macaron) { } } -func AppPluginRoute(route *plugins.AppPluginRoute, appId string) macaron.Handler { +func AppPluginRoute(route *plugins.AppPluginRoute, appID string) macaron.Handler { return func(c *m.ReqContext) { path := c.Params("*") - proxy := pluginproxy.NewApiPluginProxy(c, path, route, appId) + proxy := pluginproxy.NewApiPluginProxy(c, path, route, appID) proxy.Transport = pluginProxyTransport proxy.ServeHTTP(c.Resp, c.Req.Request) } diff --git a/pkg/api/avatar/avatar.go b/pkg/api/avatar/avatar.go index ce9da1e8790..6cf164285bf 100644 --- a/pkg/api/avatar/avatar.go +++ b/pkg/api/avatar/avatar.go @@ -97,15 +97,6 @@ type CacheServer struct { cache *gocache.Cache } -func (this *CacheServer) mustInt(r *http.Request, defaultValue int, keys ...string) (v int) { - for _, k := range keys { - if _, err := fmt.Sscanf(r.FormValue(k), "%d", &v); err == nil { - defaultValue = v - } - } - return defaultValue -} - func (this *CacheServer) Handler(ctx *macaron.Context) { urlPath := ctx.Req.URL.Path hash := urlPath[strings.LastIndex(urlPath, "/")+1:] @@ -226,7 +217,7 @@ func (this *thunderTask) Fetch() { this.Done() } -var client *http.Client = &http.Client{ +var client = &http.Client{ Timeout: time.Second * 2, Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}, } @@ -258,9 +249,6 @@ func (this *thunderTask) fetch() error { this.Avatar.data = &bytes.Buffer{} writer := bufio.NewWriter(this.Avatar.data) - if _, err = io.Copy(writer, resp.Body); err != nil { - return err - } - - return nil + _, err = io.Copy(writer, resp.Body) + return err } diff --git a/pkg/api/common.go b/pkg/api/common.go index 370f78f8b1d..7973c72c8fa 100644 --- a/pkg/api/common.go +++ b/pkg/api/common.go @@ -11,10 +11,10 @@ import ( var ( NotFound = func() Response { - return ApiError(404, "Not found", nil) + return Error(404, "Not found", nil) } ServerError = func(err error) Response { - return ApiError(500, "Server error", err) + return Error(500, "Server error", err) } ) @@ -30,7 +30,7 @@ type NormalResponse struct { err error } -func wrap(action interface{}) macaron.Handler { +func Wrap(action interface{}) macaron.Handler { return func(c *m.ReqContext) { var res Response @@ -67,22 +67,25 @@ func (r *NormalResponse) Header(key, value string) *NormalResponse { return r } -// functions to create responses +// Empty create an empty response func Empty(status int) *NormalResponse { return Respond(status, nil) } -func Json(status int, body interface{}) *NormalResponse { +// JSON create a JSON response +func JSON(status int, body interface{}) *NormalResponse { return Respond(status, body).Header("Content-Type", "application/json") } -func ApiSuccess(message string) *NormalResponse { +// Success create a successful response +func Success(message string) *NormalResponse { resp := make(map[string]interface{}) resp["message"] = message - return Json(200, resp) + return JSON(200, resp) } -func ApiError(status int, message string, err error) *NormalResponse { +// Error create a erroneous response +func Error(status int, message string, err error) *NormalResponse { data := make(map[string]interface{}) switch status { @@ -102,7 +105,7 @@ func ApiError(status int, message string, err error) *NormalResponse { } } - resp := Json(status, data) + resp := JSON(status, data) if err != nil { resp.errMessage = message @@ -112,6 +115,7 @@ func ApiError(status int, message string, err error) *NormalResponse { return resp } +// Respond create a response func Respond(status int, body interface{}) *NormalResponse { var b []byte var err error @@ -122,7 +126,7 @@ func Respond(status int, body interface{}) *NormalResponse { b = []byte(t) default: if b, err = json.Marshal(body); err != nil { - return ApiError(500, "body json marshal", err) + return Error(500, "body json marshal", err) } } return &NormalResponse{ diff --git a/pkg/api/common_test.go b/pkg/api/common_test.go index e1cbd20edb3..8b66a7a468b 100644 --- a/pkg/api/common_test.go +++ b/pkg/api/common_test.go @@ -23,7 +23,7 @@ func loggedInUserScenarioWithRole(desc string, method string, url string, routeP defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.UserId = TestUserID sc.context.OrgId = TestOrgID @@ -46,6 +46,31 @@ func loggedInUserScenarioWithRole(desc string, method string, url string, routeP }) } +func anonymousUserScenario(desc string, method string, url string, routePattern string, fn scenarioFunc) { + Convey(desc+" "+url, func() { + defer bus.ClearBusHandlers() + + sc := setupScenarioContext(url) + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { + sc.context = c + if sc.handlerFunc != nil { + return sc.handlerFunc(sc.context) + } + + return nil + }) + + switch method { + case "GET": + sc.m.Get(routePattern, sc.defaultHandler) + case "DELETE": + sc.m.Delete(routePattern, sc.defaultHandler) + } + + fn(sc) + }) +} + func (sc *scenarioContext) fakeReq(method, url string) *scenarioContext { sc.resp = httptest.NewRecorder() req, err := http.NewRequest(method, url, nil) @@ -99,7 +124,7 @@ func setupScenarioContext(url string) *scenarioContext { })) sc.m.Use(middleware.GetContextHandler()) - sc.m.Use(middleware.Sessioner(&session.Options{})) + sc.m.Use(middleware.Sessioner(&session.Options{}, 0)) return sc } diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 877524ad5dd..6abb72f1559 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -6,6 +6,7 @@ import ( "os" "path" + "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/api/dtos" @@ -22,12 +23,16 @@ import ( "github.com/grafana/grafana/pkg/util" ) -func isDashboardStarredByUser(c *m.ReqContext, dashId int64) (bool, error) { +const ( + anonString = "Anonymous" +) + +func isDashboardStarredByUser(c *m.ReqContext, dashID int64) (bool, error) { if !c.IsSignedIn { return false, nil } - query := m.IsStarredByUserQuery{UserId: c.UserId, DashboardId: dashId} + query := m.IsStarredByUserQuery{UserId: c.UserId, DashboardId: dashID} if err := bus.Dispatch(&query); err != nil { return false, err } @@ -37,10 +42,10 @@ func isDashboardStarredByUser(c *m.ReqContext, dashId int64) (bool, error) { func dashboardGuardianResponse(err error) Response { if err != nil { - return ApiError(500, "Error while checking dashboard permissions", err) + return Error(500, "Error while checking dashboard permissions", err) } - return ApiError(403, "Access denied to this dashboard", nil) + return Error(403, "Access denied to this dashboard", nil) } func GetDashboard(c *m.ReqContext) Response { @@ -60,11 +65,11 @@ func GetDashboard(c *m.ReqContext) Response { isStarred, err := isDashboardStarredByUser(c, dash.Id) if err != nil { - return ApiError(500, "Error while checking if dashboard was starred by user", err) + return Error(500, "Error while checking if dashboard was starred by user", err) } // Finding creator and last updater of the dashboard - updater, creator := "Anonymous", "Anonymous" + updater, creator := anonString, anonString if dash.UpdatedBy > 0 { updater = getUserLogin(dash.UpdatedBy) } @@ -96,12 +101,22 @@ func GetDashboard(c *m.ReqContext) Response { if dash.FolderId > 0 { query := m.GetDashboardQuery{Id: dash.FolderId, OrgId: c.OrgId} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Dashboard folder could not be read", err) + return Error(500, "Dashboard folder could not be read", err) } meta.FolderTitle = query.Result.Title meta.FolderUrl = query.Result.GetUrl() } + isDashboardProvisioned := &m.IsDashboardProvisionedQuery{DashboardId: dash.Id} + err = bus.Dispatch(isDashboardProvisioned) + if err != nil { + return Error(500, "Error while checking if dashboard is provisioned", err) + } + + if isDashboardProvisioned.Result { + meta.Provisioned = true + } + // make sure db version is in sync with json model version dash.Data.Set("version", dash.Version) @@ -111,31 +126,29 @@ func GetDashboard(c *m.ReqContext) Response { } c.TimeRequest(metrics.M_Api_Dashboard_Get) - return Json(200, dto) + return JSON(200, dto) } -func getUserLogin(userId int64) string { - query := m.GetUserByIdQuery{Id: userId} +func getUserLogin(userID int64) string { + query := m.GetUserByIdQuery{Id: userID} err := bus.Dispatch(&query) if err != nil { - return "Anonymous" - } else { - user := query.Result - return user.Login + return anonString } + return query.Result.Login } -func getDashboardHelper(orgId int64, slug string, id int64, uid string) (*m.Dashboard, Response) { +func getDashboardHelper(orgID int64, slug string, id int64, uid string) (*m.Dashboard, Response) { var query m.GetDashboardQuery if len(uid) > 0 { - query = m.GetDashboardQuery{Uid: uid, Id: id, OrgId: orgId} + query = m.GetDashboardQuery{Uid: uid, Id: id, OrgId: orgID} } else { - query = m.GetDashboardQuery{Slug: slug, Id: id, OrgId: orgId} + query = m.GetDashboardQuery{Slug: slug, Id: id, OrgId: orgID} } if err := bus.Dispatch(&query); err != nil { - return nil, ApiError(404, "Dashboard not found", err) + return nil, Error(404, "Dashboard not found", err) } return query.Result, nil @@ -145,11 +158,11 @@ func DeleteDashboard(c *m.ReqContext) Response { query := m.GetDashboardsBySlugQuery{OrgId: c.OrgId, Slug: c.Params(":slug")} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to retrieve dashboards by slug", err) + return Error(500, "Failed to retrieve dashboards by slug", err) } if len(query.Result) > 1 { - return Json(412, util.DynMap{"status": "multiple-slugs-exists", "message": m.ErrDashboardsWithSameSlugExists.Error()}) + return JSON(412, util.DynMap{"status": "multiple-slugs-exists", "message": m.ErrDashboardsWithSameSlugExists.Error()}) } dash, rsp := getDashboardHelper(c.OrgId, c.Params(":slug"), 0, "") @@ -164,16 +177,16 @@ func DeleteDashboard(c *m.ReqContext) Response { cmd := m.DeleteDashboardCommand{OrgId: c.OrgId, Id: dash.Id} if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to delete dashboard", err) + return Error(500, "Failed to delete dashboard", err) } - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "title": dash.Title, "message": fmt.Sprintf("Dashboard %s deleted", dash.Title), }) } -func DeleteDashboardByUid(c *m.ReqContext) Response { +func DeleteDashboardByUID(c *m.ReqContext) Response { dash, rsp := getDashboardHelper(c.OrgId, "", 0, c.Params(":uid")) if rsp != nil { return rsp @@ -186,10 +199,10 @@ func DeleteDashboardByUid(c *m.ReqContext) Response { cmd := m.DeleteDashboardCommand{OrgId: c.OrgId, Id: dash.Id} if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to delete dashboard", err) + return Error(500, "Failed to delete dashboard", err) } - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "title": dash.Title, "message": fmt.Sprintf("Dashboard %s deleted", dash.Title), }) @@ -204,10 +217,10 @@ func PostDashboard(c *m.ReqContext, cmd m.SaveDashboardCommand) Response { if dash.Id == 0 && dash.Uid == "" { limitReached, err := quota.QuotaReached(c, "dashboard") if err != nil { - return ApiError(500, "failed to get quota", err) + return Error(500, "failed to get quota", err) } if limitReached { - return ApiError(403, "Quota reached", nil) + return Error(403, "Quota reached", nil) } } @@ -230,24 +243,25 @@ func PostDashboard(c *m.ReqContext, cmd m.SaveDashboardCommand) Response { err == m.ErrDashboardWithSameUIDExists || err == m.ErrFolderNotFound || err == m.ErrDashboardFolderCannotHaveParent || - err == m.ErrDashboardFolderNameExists { - return ApiError(400, err.Error(), nil) + err == m.ErrDashboardFolderNameExists || + err == m.ErrDashboardCannotSaveProvisionedDashboard { + return Error(400, err.Error(), nil) } if err == m.ErrDashboardUpdateAccessDenied { - return ApiError(403, err.Error(), err) + return Error(403, err.Error(), err) } - if err == m.ErrDashboardContainsInvalidAlertData { - return ApiError(500, "Invalid alert data. Cannot save dashboard", err) + if validationErr, ok := err.(alerting.ValidationError); ok { + return Error(422, validationErr.Error(), nil) } if err != nil { if err == m.ErrDashboardWithSameNameInFolderExists { - return Json(412, util.DynMap{"status": "name-exists", "message": err.Error()}) + return JSON(412, util.DynMap{"status": "name-exists", "message": err.Error()}) } if err == m.ErrDashboardVersionMismatch { - return Json(412, util.DynMap{"status": "version-mismatch", "message": err.Error()}) + return JSON(412, util.DynMap{"status": "version-mismatch", "message": err.Error()}) } if pluginErr, ok := err.(m.UpdatePluginDashboardError); ok { message := "The dashboard belongs to plugin " + pluginErr.PluginId + "." @@ -255,20 +269,20 @@ func PostDashboard(c *m.ReqContext, cmd m.SaveDashboardCommand) Response { if pluginDef, exist := plugins.Plugins[pluginErr.PluginId]; exist { message = "The dashboard belongs to plugin " + pluginDef.Name + "." } - return Json(412, util.DynMap{"status": "plugin-dashboard", "message": message}) + return JSON(412, util.DynMap{"status": "plugin-dashboard", "message": message}) } if err == m.ErrDashboardNotFound { - return Json(404, util.DynMap{"status": "not-found", "message": err.Error()}) + return JSON(404, util.DynMap{"status": "not-found", "message": err.Error()}) } - return ApiError(500, "Failed to save dashboard", err) + return Error(500, "Failed to save dashboard", err) } if err == m.ErrDashboardFailedToUpdateAlertData { - return ApiError(500, "Invalid alert data. Cannot save dashboard", err) + return Error(500, "Invalid alert data. Cannot save dashboard", err) } c.TimeRequest(metrics.M_Api_Dashboard_Save) - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "status": "success", "slug": dashboard.Slug, "version": dashboard.Version, @@ -279,9 +293,9 @@ func PostDashboard(c *m.ReqContext, cmd m.SaveDashboardCommand) Response { } func GetHomeDashboard(c *m.ReqContext) Response { - prefsQuery := m.GetPreferencesWithDefaultsQuery{OrgId: c.OrgId, UserId: c.UserId} + prefsQuery := m.GetPreferencesWithDefaultsQuery{User: c.SignedInUser} if err := bus.Dispatch(&prefsQuery); err != nil { - return ApiError(500, "Failed to get preferences", err) + return Error(500, "Failed to get preferences", err) } if prefsQuery.Result.HomeDashboardId != 0 { @@ -290,16 +304,15 @@ func GetHomeDashboard(c *m.ReqContext) Response { if err == nil { url := m.GetDashboardUrl(slugQuery.Result.Uid, slugQuery.Result.Slug) dashRedirect := dtos.DashboardRedirect{RedirectUri: url} - return Json(200, &dashRedirect) - } else { - log.Warn("Failed to get slug from database, %s", err.Error()) + return JSON(200, &dashRedirect) } + log.Warn("Failed to get slug from database, %s", err.Error()) } filePath := path.Join(setting.StaticRootPath, "dashboards/home.json") file, err := os.Open(filePath) if err != nil { - return ApiError(500, "Failed to load home dashboard", err) + return Error(500, "Failed to load home dashboard", err) } dash := dtos.DashboardFullWithMeta{} @@ -309,14 +322,14 @@ func GetHomeDashboard(c *m.ReqContext) Response { jsonParser := json.NewDecoder(file) if err := jsonParser.Decode(&dash.Dashboard); err != nil { - return ApiError(500, "Failed to load home dashboard", err) + return Error(500, "Failed to load home dashboard", err) } if c.HasUserRole(m.ROLE_ADMIN) && !c.HasHelpFlag(m.HelpFlagGettingStartedPanelDismissed) { addGettingStartedPanelToHomeDashboard(dash.Dashboard) } - return Json(200, &dash) + return JSON(200, &dash) } func addGettingStartedPanelToHomeDashboard(dash *simplejson.Json) { @@ -339,22 +352,22 @@ func addGettingStartedPanelToHomeDashboard(dash *simplejson.Json) { // GetDashboardVersions returns all dashboard versions as JSON func GetDashboardVersions(c *m.ReqContext) Response { - dashId := c.ParamsInt64(":dashboardId") + dashID := c.ParamsInt64(":dashboardId") - guardian := guardian.New(dashId, c.OrgId, c.SignedInUser) + guardian := guardian.New(dashID, c.OrgId, c.SignedInUser) if canSave, err := guardian.CanSave(); err != nil || !canSave { return dashboardGuardianResponse(err) } query := m.GetDashboardVersionsQuery{ OrgId: c.OrgId, - DashboardId: dashId, + DashboardId: dashID, Limit: c.QueryInt("limit"), Start: c.QueryInt("start"), } if err := bus.Dispatch(&query); err != nil { - return ApiError(404, fmt.Sprintf("No versions found for dashboardId %d", dashId), err) + return Error(404, fmt.Sprintf("No versions found for dashboardId %d", dashID), err) } for _, version := range query.Result { @@ -373,29 +386,29 @@ func GetDashboardVersions(c *m.ReqContext) Response { } } - return Json(200, query.Result) + return JSON(200, query.Result) } // GetDashboardVersion returns the dashboard version with the given ID. func GetDashboardVersion(c *m.ReqContext) Response { - dashId := c.ParamsInt64(":dashboardId") + dashID := c.ParamsInt64(":dashboardId") - guardian := guardian.New(dashId, c.OrgId, c.SignedInUser) + guardian := guardian.New(dashID, c.OrgId, c.SignedInUser) if canSave, err := guardian.CanSave(); err != nil || !canSave { return dashboardGuardianResponse(err) } query := m.GetDashboardVersionQuery{ OrgId: c.OrgId, - DashboardId: dashId, + DashboardId: dashID, Version: c.ParamsInt(":id"), } if err := bus.Dispatch(&query); err != nil { - return ApiError(500, fmt.Sprintf("Dashboard version %d not found for dashboardId %d", query.Version, dashId), err) + return Error(500, fmt.Sprintf("Dashboard version %d not found for dashboardId %d", query.Version, dashID), err) } - creator := "Anonymous" + creator := anonString if query.Result.CreatedBy > 0 { creator = getUserLogin(query.Result.CreatedBy) } @@ -405,7 +418,7 @@ func GetDashboardVersion(c *m.ReqContext) Response { CreatedBy: creator, } - return Json(200, dashVersionMeta) + return JSON(200, dashVersionMeta) } // POST /api/dashboards/calculate-diff performs diffs on two dashboards @@ -441,9 +454,9 @@ func CalculateDashboardDiff(c *m.ReqContext, apiOptions dtos.CalculateDiffOption result, err := dashdiffs.CalculateDiff(&options) if err != nil { if err == m.ErrDashboardVersionNotFound { - return ApiError(404, "Dashboard version not found", err) + return Error(404, "Dashboard version not found", err) } - return ApiError(500, "Unable to compute diff", err) + return Error(500, "Unable to compute diff", err) } if options.DiffType == dashdiffs.DiffDelta { @@ -467,7 +480,7 @@ func RestoreDashboardVersion(c *m.ReqContext, apiCmd dtos.RestoreDashboardVersio versionQuery := m.GetDashboardVersionQuery{DashboardId: dash.Id, Version: apiCmd.Version, OrgId: c.OrgId} if err := bus.Dispatch(&versionQuery); err != nil { - return ApiError(404, "Dashboard version not found", nil) + return Error(404, "Dashboard version not found", nil) } version := versionQuery.Result diff --git a/pkg/api/dashboard_permission.go b/pkg/api/dashboard_permission.go index a62c27ab320..342eaf556c6 100644 --- a/pkg/api/dashboard_permission.go +++ b/pkg/api/dashboard_permission.go @@ -10,14 +10,14 @@ import ( ) func GetDashboardPermissionList(c *m.ReqContext) Response { - dashId := c.ParamsInt64(":dashboardId") + dashID := c.ParamsInt64(":dashboardId") - _, rsp := getDashboardHelper(c.OrgId, "", dashId, "") + _, rsp := getDashboardHelper(c.OrgId, "", dashID, "") if rsp != nil { return rsp } - g := guardian.New(dashId, c.OrgId, c.SignedInUser) + g := guardian.New(dashID, c.OrgId, c.SignedInUser) if canAdmin, err := g.CanAdmin(); err != nil || !canAdmin { return dashboardGuardianResponse(err) @@ -25,38 +25,43 @@ func GetDashboardPermissionList(c *m.ReqContext) Response { acl, err := g.GetAcl() if err != nil { - return ApiError(500, "Failed to get dashboard permissions", err) + return Error(500, "Failed to get dashboard permissions", err) } for _, perm := range acl { + perm.UserAvatarUrl = dtos.GetGravatarUrl(perm.UserEmail) + + if perm.TeamId > 0 { + perm.TeamAvatarUrl = dtos.GetGravatarUrlWithDefault(perm.TeamEmail, perm.Team) + } if perm.Slug != "" { perm.Url = m.GetDashboardFolderUrl(perm.IsFolder, perm.Uid, perm.Slug) } } - return Json(200, acl) + return JSON(200, acl) } func UpdateDashboardPermissions(c *m.ReqContext, apiCmd dtos.UpdateDashboardAclCommand) Response { - dashId := c.ParamsInt64(":dashboardId") + dashID := c.ParamsInt64(":dashboardId") - _, rsp := getDashboardHelper(c.OrgId, "", dashId, "") + _, rsp := getDashboardHelper(c.OrgId, "", dashID, "") if rsp != nil { return rsp } - g := guardian.New(dashId, c.OrgId, c.SignedInUser) + g := guardian.New(dashID, c.OrgId, c.SignedInUser) if canAdmin, err := g.CanAdmin(); err != nil || !canAdmin { return dashboardGuardianResponse(err) } cmd := m.UpdateDashboardAclCommand{} - cmd.DashboardId = dashId + cmd.DashboardId = dashID for _, item := range apiCmd.Items { cmd.Items = append(cmd.Items, &m.DashboardAcl{ OrgId: c.OrgId, - DashboardId: dashId, + DashboardId: dashID, UserId: item.UserId, TeamId: item.TeamId, Role: item.Role, @@ -70,21 +75,21 @@ func UpdateDashboardPermissions(c *m.ReqContext, apiCmd dtos.UpdateDashboardAclC if err != nil { if err == guardian.ErrGuardianPermissionExists || err == guardian.ErrGuardianOverride { - return ApiError(400, err.Error(), err) + return Error(400, err.Error(), err) } - return ApiError(500, "Error while checking dashboard permissions", err) + return Error(500, "Error while checking dashboard permissions", err) } - return ApiError(403, "Cannot remove own admin permission for a folder", nil) + return Error(403, "Cannot remove own admin permission for a folder", nil) } if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrDashboardAclInfoMissing || err == m.ErrDashboardPermissionDashboardEmpty { - return ApiError(409, err.Error(), err) + return Error(409, err.Error(), err) } - return ApiError(500, "Failed to create permission", err) + return Error(500, "Failed to create permission", err) } - return ApiSuccess("Dashboard permissions updated") + return Success("Dashboard permissions updated") } diff --git a/pkg/api/dashboard_permission_test.go b/pkg/api/dashboard_permission_test.go index bdf80ef5241..f65c5f1f5fa 100644 --- a/pkg/api/dashboard_permission_test.go +++ b/pkg/api/dashboard_permission_test.go @@ -143,7 +143,7 @@ func TestDashboardPermissionApiEndpoint(t *testing.T) { }) }) - Convey("When trying to override inherited permissions with lower presedence", func() { + Convey("When trying to override inherited permissions with lower precedence", func() { origNewGuardian := guardian.New guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{ CanAdminValue: true, @@ -194,7 +194,7 @@ func updateDashboardPermissionScenario(desc string, url string, routePattern str sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.OrgId = TestOrgID sc.context.UserId = TestUserID diff --git a/pkg/api/dashboard_snapshot.go b/pkg/api/dashboard_snapshot.go index 4656940d2bb..e4e9c9d040f 100644 --- a/pkg/api/dashboard_snapshot.go +++ b/pkg/api/dashboard_snapshot.go @@ -91,40 +91,60 @@ func GetDashboardSnapshot(c *m.ReqContext) { c.JSON(200, dto) } -// GET /api/snapshots-delete/:key -func DeleteDashboardSnapshot(c *m.ReqContext) Response { - key := c.Params(":key") +// GET /api/snapshots-delete/:deleteKey +func DeleteDashboardSnapshotByDeleteKey(c *m.ReqContext) Response { + key := c.Params(":deleteKey") query := &m.GetDashboardSnapshotQuery{DeleteKey: key} err := bus.Dispatch(query) if err != nil { - return ApiError(500, "Failed to get dashboard snapshot", err) + return Error(500, "Failed to get dashboard snapshot", err) + } + + cmd := &m.DeleteDashboardSnapshotCommand{DeleteKey: query.Result.DeleteKey} + + if err := bus.Dispatch(cmd); err != nil { + return Error(500, "Failed to delete dashboard snapshot", err) + } + + return JSON(200, util.DynMap{"message": "Snapshot deleted. It might take an hour before it's cleared from any CDN caches."}) +} + +// DELETE /api/snapshots/:key +func DeleteDashboardSnapshot(c *m.ReqContext) Response { + key := c.Params(":key") + + query := &m.GetDashboardSnapshotQuery{Key: key} + + err := bus.Dispatch(query) + if err != nil { + return Error(500, "Failed to get dashboard snapshot", err) } if query.Result == nil { - return ApiError(404, "Failed to get dashboard snapshot", nil) + return Error(404, "Failed to get dashboard snapshot", nil) } dashboard := query.Result.Dashboard - dashboardId := dashboard.Get("id").MustInt64() + dashboardID := dashboard.Get("id").MustInt64() - guardian := guardian.New(dashboardId, c.OrgId, c.SignedInUser) + guardian := guardian.New(dashboardID, c.OrgId, c.SignedInUser) canEdit, err := guardian.CanEdit() if err != nil { - return ApiError(500, "Error while checking permissions for snapshot", err) + return Error(500, "Error while checking permissions for snapshot", err) } if !canEdit && query.Result.UserId != c.SignedInUser.UserId { - return ApiError(403, "Access denied to this snapshot", nil) + return Error(403, "Access denied to this snapshot", nil) } - cmd := &m.DeleteDashboardSnapshotCommand{DeleteKey: key} + cmd := &m.DeleteDashboardSnapshotCommand{DeleteKey: query.Result.DeleteKey} if err := bus.Dispatch(cmd); err != nil { - return ApiError(500, "Failed to delete dashboard snapshot", err) + return Error(500, "Failed to delete dashboard snapshot", err) } - return Json(200, util.DynMap{"message": "Snapshot deleted. It might take an hour before it's cleared from a CDN cache."}) + return JSON(200, util.DynMap{"message": "Snapshot deleted. It might take an hour before it's cleared from any CDN caches."}) } // GET /api/dashboard/snapshots @@ -145,7 +165,7 @@ func SearchDashboardSnapshots(c *m.ReqContext) Response { err := bus.Dispatch(&searchQuery) if err != nil { - return ApiError(500, "Search failed", err) + return Error(500, "Search failed", err) } dtos := make([]*m.DashboardSnapshotDTO, len(searchQuery.Result)) @@ -154,7 +174,6 @@ func SearchDashboardSnapshots(c *m.ReqContext) Response { Id: snapshot.Id, Name: snapshot.Name, Key: snapshot.Key, - DeleteKey: snapshot.DeleteKey, OrgId: snapshot.OrgId, UserId: snapshot.UserId, External: snapshot.External, @@ -165,5 +184,5 @@ func SearchDashboardSnapshots(c *m.ReqContext) Response { } } - return Json(200, dtos) + return JSON(200, dtos) } diff --git a/pkg/api/dashboard_snapshot_test.go b/pkg/api/dashboard_snapshot_test.go index 87c2b9e99d4..e58f2c4712d 100644 --- a/pkg/api/dashboard_snapshot_test.go +++ b/pkg/api/dashboard_snapshot_test.go @@ -39,7 +39,7 @@ func TestDashboardSnapshotApiEndpoint(t *testing.T) { return nil }) - teamResp := []*m.Team{} + teamResp := []*m.TeamDTO{} bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error { query.Result = teamResp return nil @@ -47,15 +47,30 @@ func TestDashboardSnapshotApiEndpoint(t *testing.T) { Convey("When user has editor role and is not in the ACL", func() { Convey("Should not be able to delete snapshot", func() { - loggedInUserScenarioWithRole("When calling GET on", "GET", "/api/snapshots-delete/12345", "/api/snapshots-delete/:key", m.ROLE_EDITOR, func(sc *scenarioContext) { + loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/snapshots/12345", "/api/snapshots/:key", m.ROLE_EDITOR, func(sc *scenarioContext) { sc.handlerFunc = DeleteDashboardSnapshot - sc.fakeReqWithParams("GET", sc.url, map[string]string{"key": "12345"}).exec() + sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"key": "12345"}).exec() So(sc.resp.Code, ShouldEqual, 403) }) }) }) + Convey("When user is anonymous", func() { + Convey("Should be able to delete snapshot by deleteKey", func() { + anonymousUserScenario("When calling GET on", "GET", "/api/snapshots-delete/12345", "/api/snapshots-delete/:deleteKey", func(sc *scenarioContext) { + sc.handlerFunc = DeleteDashboardSnapshotByDeleteKey + sc.fakeReqWithParams("GET", sc.url, map[string]string{"deleteKey": "12345"}).exec() + + So(sc.resp.Code, ShouldEqual, 200) + respJSON, err := simplejson.NewJson(sc.resp.Body.Bytes()) + So(err, ShouldBeNil) + + So(respJSON.Get("message").MustString(), ShouldStartWith, "Snapshot deleted") + }) + }) + }) + Convey("When user is editor and dashboard has default ACL", func() { aclMockResp = []*m.DashboardAclInfoDTO{ {Role: &viewerRole, Permission: m.PERMISSION_VIEW}, @@ -63,9 +78,9 @@ func TestDashboardSnapshotApiEndpoint(t *testing.T) { } Convey("Should be able to delete a snapshot", func() { - loggedInUserScenarioWithRole("When calling GET on", "GET", "/api/snapshots-delete/12345", "/api/snapshots-delete/:key", m.ROLE_EDITOR, func(sc *scenarioContext) { + loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/snapshots/12345", "/api/snapshots/:key", m.ROLE_EDITOR, func(sc *scenarioContext) { sc.handlerFunc = DeleteDashboardSnapshot - sc.fakeReqWithParams("GET", sc.url, map[string]string{"key": "12345"}).exec() + sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"key": "12345"}).exec() So(sc.resp.Code, ShouldEqual, 200) respJSON, err := simplejson.NewJson(sc.resp.Body.Bytes()) @@ -81,9 +96,9 @@ func TestDashboardSnapshotApiEndpoint(t *testing.T) { mockSnapshotResult.UserId = TestUserID Convey("Should be able to delete a snapshot", func() { - loggedInUserScenarioWithRole("When calling GET on", "GET", "/api/snapshots-delete/12345", "/api/snapshots-delete/:key", m.ROLE_EDITOR, func(sc *scenarioContext) { + loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/snapshots/12345", "/api/snapshots/:key", m.ROLE_EDITOR, func(sc *scenarioContext) { sc.handlerFunc = DeleteDashboardSnapshot - sc.fakeReqWithParams("GET", sc.url, map[string]string{"key": "12345"}).exec() + sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"key": "12345"}).exec() So(sc.resp.Code, ShouldEqual, 200) respJSON, err := simplejson.NewJson(sc.resp.Body.Bytes()) diff --git a/pkg/api/dashboard_test.go b/pkg/api/dashboard_test.go index 6c5b4e4c102..2726623c242 100644 --- a/pkg/api/dashboard_test.go +++ b/pkg/api/dashboard_test.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/setting" @@ -42,6 +43,11 @@ func TestDashboardApiEndpoint(t *testing.T) { return nil }) + bus.AddHandler("test", func(query *m.IsDashboardProvisionedQuery) error { + query.Result = false + return nil + }) + viewerRole := m.ROLE_VIEWER editorRole := m.ROLE_EDITOR @@ -56,7 +62,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error { - query.Result = []*m.Team{} + query.Result = []*m.TeamDTO{} return nil }) @@ -105,7 +111,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 403) Convey("Should lookup dashboard by uid", func() { @@ -165,7 +171,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 200) Convey("Should lookup dashboard by uid", func() { @@ -192,6 +198,11 @@ func TestDashboardApiEndpoint(t *testing.T) { fakeDash.HasAcl = true setting.ViewersCanEdit = false + bus.AddHandler("test", func(query *m.IsDashboardProvisionedQuery) error { + query.Result = false + return nil + }) + bus.AddHandler("test", func(query *m.GetDashboardsBySlugQuery) error { dashboards := []*m.Dashboard{fakeDash} query.Result = dashboards @@ -220,7 +231,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error { - query.Result = []*m.Team{} + query.Result = []*m.TeamDTO{} return nil }) @@ -271,7 +282,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 403) Convey("Should lookup dashboard by uid", func() { @@ -329,7 +340,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 403) Convey("Should lookup dashboard by uid", func() { @@ -398,7 +409,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 200) Convey("Should lookup dashboard by uid", func() { @@ -468,7 +479,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 403) Convey("Should lookup dashboard by uid", func() { @@ -527,7 +538,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 200) Convey("Should lookup dashboard by uid", func() { @@ -594,7 +605,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) loggedInUserScenarioWithRole("When calling DELETE on", "DELETE", "/api/dashboards/uid/abcdefghi", "/api/dashboards/uid/:uid", role, func(sc *scenarioContext) { - CallDeleteDashboardByUid(sc) + CallDeleteDashboardByUID(sc) So(sc.resp.Code, ShouldEqual, 403) Convey("Should lookup dashboard by uid", func() { @@ -625,6 +636,11 @@ func TestDashboardApiEndpoint(t *testing.T) { dashTwo.FolderId = 3 dashTwo.HasAcl = false + bus.AddHandler("test", func(query *m.IsDashboardProvisionedQuery) error { + query.Result = false + return nil + }) + bus.AddHandler("test", func(query *m.GetDashboardsBySlugQuery) error { dashboards := []*m.Dashboard{dashOne, dashTwo} query.Result = dashboards @@ -638,7 +654,7 @@ func TestDashboardApiEndpoint(t *testing.T) { Convey("Should result in 412 Precondition failed", func() { So(sc.resp.Code, ShouldEqual, 412) - result := sc.ToJson() + result := sc.ToJSON() So(result.Get("status").MustString(), ShouldEqual, "multiple-slugs-exists") So(result.Get("message").MustString(), ShouldEqual, m.ErrDashboardsWithSameSlugExists.Error()) }) @@ -686,7 +702,7 @@ func TestDashboardApiEndpoint(t *testing.T) { }) Convey("It should return correct response data", func() { - result := sc.ToJson() + result := sc.ToJSON() So(result.Get("status").MustString(), ShouldEqual, "success") So(result.Get("id").MustInt64(), ShouldEqual, 2) So(result.Get("uid").MustString(), ShouldEqual, "uid") @@ -710,7 +726,7 @@ func TestDashboardApiEndpoint(t *testing.T) { {SaveError: m.ErrDashboardVersionMismatch, ExpectedStatusCode: 412}, {SaveError: m.ErrDashboardTitleEmpty, ExpectedStatusCode: 400}, {SaveError: m.ErrDashboardFolderCannotHaveParent, ExpectedStatusCode: 400}, - {SaveError: m.ErrDashboardContainsInvalidAlertData, ExpectedStatusCode: 500}, + {SaveError: alerting.ValidationError{Reason: "Mu"}, ExpectedStatusCode: 422}, {SaveError: m.ErrDashboardFailedToUpdateAlertData, ExpectedStatusCode: 500}, {SaveError: m.ErrDashboardFailedGenerateUniqueUid, ExpectedStatusCode: 500}, {SaveError: m.ErrDashboardTypeMismatch, ExpectedStatusCode: 400}, @@ -720,6 +736,7 @@ func TestDashboardApiEndpoint(t *testing.T) { {SaveError: m.ErrDashboardUpdateAccessDenied, ExpectedStatusCode: 403}, {SaveError: m.ErrDashboardInvalidUid, ExpectedStatusCode: 400}, {SaveError: m.ErrDashboardUidToLong, ExpectedStatusCode: 400}, + {SaveError: m.ErrDashboardCannotSaveProvisionedDashboard, ExpectedStatusCode: 400}, {SaveError: m.UpdatePluginDashboardError{PluginId: "plug"}, ExpectedStatusCode: 412}, } @@ -750,6 +767,11 @@ func TestDashboardApiEndpoint(t *testing.T) { return nil }) + bus.AddHandler("test", func(query *m.IsDashboardProvisionedQuery) error { + query.Result = false + return nil + }) + bus.AddHandler("test", func(query *m.GetDashboardVersionQuery) error { query.Result = &m.DashboardVersion{ Data: simplejson.NewFromAny(map[string]interface{}{ @@ -837,12 +859,12 @@ func CallDeleteDashboard(sc *scenarioContext) { sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() } -func CallDeleteDashboardByUid(sc *scenarioContext) { +func CallDeleteDashboardByUID(sc *scenarioContext) { bus.AddHandler("test", func(cmd *m.DeleteDashboardCommand) error { return nil }) - sc.handlerFunc = DeleteDashboardByUid + sc.handlerFunc = DeleteDashboardByUID sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() } @@ -861,7 +883,7 @@ func postDashboardScenario(desc string, url string, routePattern string, mock *d defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.SignedInUser = &m.SignedInUser{OrgId: cmd.OrgId, UserId: cmd.UserId} @@ -886,7 +908,7 @@ func postDiffScenario(desc string, url string, routePattern string, cmd dtos.Cal defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.SignedInUser = &m.SignedInUser{ OrgId: TestOrgID, @@ -903,7 +925,7 @@ func postDiffScenario(desc string, url string, routePattern string, cmd dtos.Cal }) } -func (sc *scenarioContext) ToJson() *simplejson.Json { +func (sc *scenarioContext) ToJSON() *simplejson.Json { var result *simplejson.Json err := json.NewDecoder(sc.resp.Body).Decode(&result) So(err, ShouldBeNil) diff --git a/pkg/api/dataproxy.go b/pkg/api/dataproxy.go index c6fe8b6cd8c..5cde0efd0b4 100644 --- a/pkg/api/dataproxy.go +++ b/pkg/api/dataproxy.go @@ -1,47 +1,22 @@ package api import ( - "fmt" - "time" - "github.com/grafana/grafana/pkg/api/pluginproxy" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/metrics" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" ) -const HeaderNameNoBackendCache = "X-Grafana-NoCache" - -func (hs *HttpServer) getDatasourceById(id int64, orgId int64, nocache bool) (*m.DataSource, error) { - cacheKey := fmt.Sprintf("ds-%d", id) - - if !nocache { - if cached, found := hs.cache.Get(cacheKey); found { - ds := cached.(*m.DataSource) - if ds.OrgId == orgId { - return ds, nil - } - } - } - - query := m.GetDataSourceByIdQuery{Id: id, OrgId: orgId} - if err := bus.Dispatch(&query); err != nil { - return nil, err - } - - hs.cache.Set(cacheKey, query.Result, time.Second*5) - return query.Result, nil -} - -func (hs *HttpServer) ProxyDataSourceRequest(c *m.ReqContext) { +func (hs *HTTPServer) ProxyDataSourceRequest(c *m.ReqContext) { c.TimeRequest(metrics.M_DataSource_ProxyReq_Timer) - nocache := c.Req.Header.Get(HeaderNameNoBackendCache) == "true" - - ds, err := hs.getDatasourceById(c.ParamsInt64(":id"), c.OrgId, nocache) - + dsId := c.ParamsInt64(":id") + ds, err := hs.DatasourceCache.GetDatasource(dsId, c.SignedInUser, c.SkipCache) if err != nil { + if err == m.ErrDataSourceAccessDenied { + c.JsonApiErr(403, "Access denied to datasource", err) + return + } c.JsonApiErr(500, "Unable to load datasource meta data", err) return } @@ -53,7 +28,21 @@ func (hs *HttpServer) ProxyDataSourceRequest(c *m.ReqContext) { return } - proxyPath := c.Params("*") + // macaron does not include trailing slashes when resolving a wildcard path + proxyPath := ensureProxyPathTrailingSlash(c.Req.URL.Path, c.Params("*")) + proxy := pluginproxy.NewDataSourceProxy(ds, plugin, c, proxyPath) proxy.HandleRequest() } + +// ensureProxyPathTrailingSlash Check for a trailing slash in original path and makes +// sure that a trailing slash is added to proxy path, if not already exists. +func ensureProxyPathTrailingSlash(originalPath, proxyPath string) string { + if len(proxyPath) > 1 { + if originalPath[len(originalPath)-1] == '/' && proxyPath[len(proxyPath)-1] != '/' { + return proxyPath + "/" + } + } + + return proxyPath +} diff --git a/pkg/api/dataproxy_test.go b/pkg/api/dataproxy_test.go new file mode 100644 index 00000000000..a1d7cf68a37 --- /dev/null +++ b/pkg/api/dataproxy_test.go @@ -0,0 +1,19 @@ +package api + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestDataProxy(t *testing.T) { + Convey("Data proxy test", t, func() { + Convey("Should append trailing slash to proxy path if original path has a trailing slash", func() { + So(ensureProxyPathTrailingSlash("/api/datasources/proxy/6/api/v1/query_range/", "api/v1/query_range/"), ShouldEqual, "api/v1/query_range/") + }) + + Convey("Should not append trailing slash to proxy path if original path doesn't have a trailing slash", func() { + So(ensureProxyPathTrailingSlash("/api/datasources/proxy/6/api/v1/query_range", "api/v1/query_range"), ShouldEqual, "api/v1/query_range") + }) + }) +} diff --git a/pkg/api/datasources.go b/pkg/api/datasources.go index ed8fc5d2a66..e7614614076 100644 --- a/pkg/api/datasources.go +++ b/pkg/api/datasources.go @@ -14,14 +14,14 @@ func GetDataSources(c *m.ReqContext) Response { query := m.GetDataSourcesQuery{OrgId: c.OrgId} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to query datasources", err) + return Error(500, "Failed to query datasources", err) } result := make(dtos.DataSourceList, 0) for _, ds := range query.Result { dsItem := dtos.DataSourceListItemDTO{ - Id: ds.Id, OrgId: ds.OrgId, + Id: ds.Id, Name: ds.Name, Url: ds.Url, Type: ds.Type, @@ -46,7 +46,7 @@ func GetDataSources(c *m.ReqContext) Response { sort.Sort(result) - return Json(200, &result) + return JSON(200, &result) } func GetDataSourceById(c *m.ReqContext) Response { @@ -57,66 +57,69 @@ func GetDataSourceById(c *m.ReqContext) Response { if err := bus.Dispatch(&query); err != nil { if err == m.ErrDataSourceNotFound { - return ApiError(404, "Data source not found", nil) + return Error(404, "Data source not found", nil) } - return ApiError(500, "Failed to query datasources", err) + return Error(500, "Failed to query datasources", err) } ds := query.Result dtos := convertModelToDtos(ds) - return Json(200, &dtos) + return JSON(200, &dtos) } func DeleteDataSourceById(c *m.ReqContext) Response { id := c.ParamsInt64(":id") if id <= 0 { - return ApiError(400, "Missing valid datasource id", nil) + return Error(400, "Missing valid datasource id", nil) } ds, err := getRawDataSourceById(id, c.OrgId) if err != nil { - return ApiError(400, "Failed to delete datasource", nil) + return Error(400, "Failed to delete datasource", nil) } if ds.ReadOnly { - return ApiError(403, "Cannot delete read-only data source", nil) + return Error(403, "Cannot delete read-only data source", nil) } cmd := &m.DeleteDataSourceByIdCommand{Id: id, OrgId: c.OrgId} err = bus.Dispatch(cmd) if err != nil { - return ApiError(500, "Failed to delete datasource", err) + return Error(500, "Failed to delete datasource", err) } - return ApiSuccess("Data source deleted") + return Success("Data source deleted") } func DeleteDataSourceByName(c *m.ReqContext) Response { name := c.Params(":name") if name == "" { - return ApiError(400, "Missing valid datasource name", nil) + return Error(400, "Missing valid datasource name", nil) } getCmd := &m.GetDataSourceByNameQuery{Name: name, OrgId: c.OrgId} if err := bus.Dispatch(getCmd); err != nil { - return ApiError(500, "Failed to delete datasource", err) + if err == m.ErrDataSourceNotFound { + return Error(404, "Data source not found", nil) + } + return Error(500, "Failed to delete datasource", err) } if getCmd.Result.ReadOnly { - return ApiError(403, "Cannot delete read-only data source", nil) + return Error(403, "Cannot delete read-only data source", nil) } cmd := &m.DeleteDataSourceByNameCommand{Name: name, OrgId: c.OrgId} err := bus.Dispatch(cmd) if err != nil { - return ApiError(500, "Failed to delete datasource", err) + return Error(500, "Failed to delete datasource", err) } - return ApiSuccess("Data source deleted") + return Success("Data source deleted") } func AddDataSource(c *m.ReqContext, cmd m.AddDataSourceCommand) Response { @@ -124,14 +127,14 @@ func AddDataSource(c *m.ReqContext, cmd m.AddDataSourceCommand) Response { if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrDataSourceNameExists { - return ApiError(409, err.Error(), err) + return Error(409, err.Error(), err) } - return ApiError(500, "Failed to add datasource", err) + return Error(500, "Failed to add datasource", err) } ds := convertModelToDtos(cmd.Result) - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "message": "Datasource added", "id": cmd.Result.Id, "name": cmd.Result.Name, @@ -143,29 +146,42 @@ func UpdateDataSource(c *m.ReqContext, cmd m.UpdateDataSourceCommand) Response { cmd.OrgId = c.OrgId cmd.Id = c.ParamsInt64(":id") - err := fillWithSecureJsonData(&cmd) + err := fillWithSecureJSONData(&cmd) if err != nil { - return ApiError(500, "Failed to update datasource", err) + return Error(500, "Failed to update datasource", err) } err = bus.Dispatch(&cmd) if err != nil { if err == m.ErrDataSourceUpdatingOldVersion { - return ApiError(500, "Failed to update datasource. Reload new version and try again", err) - } else { - return ApiError(500, "Failed to update datasource", err) + return Error(500, "Failed to update datasource. Reload new version and try again", err) } + return Error(500, "Failed to update datasource", err) } - ds := convertModelToDtos(cmd.Result) - return Json(200, util.DynMap{ + + query := m.GetDataSourceByIdQuery{ + Id: cmd.Id, + OrgId: c.OrgId, + } + + if err := bus.Dispatch(&query); err != nil { + if err == m.ErrDataSourceNotFound { + return Error(404, "Data source not found", nil) + } + return Error(500, "Failed to query datasources", err) + } + + dtos := convertModelToDtos(query.Result) + + return JSON(200, util.DynMap{ "message": "Datasource updated", "id": cmd.Id, "name": cmd.Name, - "datasource": ds, + "datasource": dtos, }) } -func fillWithSecureJsonData(cmd *m.UpdateDataSourceCommand) error { +func fillWithSecureJSONData(cmd *m.UpdateDataSourceCommand) error { if len(cmd.SecureJsonData) == 0 { return nil } @@ -179,8 +195,8 @@ func fillWithSecureJsonData(cmd *m.UpdateDataSourceCommand) error { return m.ErrDatasourceIsReadOnly } - secureJsonData := ds.SecureJsonData.Decrypt() - for k, v := range secureJsonData { + secureJSONData := ds.SecureJsonData.Decrypt() + for k, v := range secureJSONData { if _, ok := cmd.SecureJsonData[k]; !ok { cmd.SecureJsonData[k] = v @@ -190,10 +206,10 @@ func fillWithSecureJsonData(cmd *m.UpdateDataSourceCommand) error { return nil } -func getRawDataSourceById(id int64, orgId int64) (*m.DataSource, error) { +func getRawDataSourceById(id int64, orgID int64) (*m.DataSource, error) { query := m.GetDataSourceByIdQuery{ Id: id, - OrgId: orgId, + OrgId: orgID, } if err := bus.Dispatch(&query); err != nil { @@ -209,14 +225,14 @@ func GetDataSourceByName(c *m.ReqContext) Response { if err := bus.Dispatch(&query); err != nil { if err == m.ErrDataSourceNotFound { - return ApiError(404, "Data source not found", nil) + return Error(404, "Data source not found", nil) } - return ApiError(500, "Failed to query datasources", err) + return Error(500, "Failed to query datasources", err) } dtos := convertModelToDtos(query.Result) dtos.ReadOnly = true - return Json(200, &dtos) + return JSON(200, &dtos) } // Get /api/datasources/id/:name @@ -225,9 +241,9 @@ func GetDataSourceIdByName(c *m.ReqContext) Response { if err := bus.Dispatch(&query); err != nil { if err == m.ErrDataSourceNotFound { - return ApiError(404, "Data source not found", nil) + return Error(404, "Data source not found", nil) } - return ApiError(500, "Failed to query datasources", err) + return Error(500, "Failed to query datasources", err) } ds := query.Result @@ -235,7 +251,7 @@ func GetDataSourceIdByName(c *m.ReqContext) Response { Id: ds.Id, } - return Json(200, &dtos) + return JSON(200, &dtos) } func convertModelToDtos(ds *m.DataSource) dtos.DataSource { diff --git a/pkg/api/datasources_test.go b/pkg/api/datasources_test.go index 490393727d6..6e52a27758b 100644 --- a/pkg/api/datasources_test.go +++ b/pkg/api/datasources_test.go @@ -46,5 +46,13 @@ func TestDataSourcesProxy(t *testing.T) { So(respJSON[3]["name"], ShouldEqual, "ZZZ") }) }) + + Convey("Should be able to save a data source", func() { + loggedInUserScenario("When calling DELETE on non-existing", "/api/datasources/name/12345", func(sc *scenarioContext) { + sc.handlerFunc = DeleteDataSourceByName + sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() + So(sc.resp.Code, ShouldEqual, 404) + }) + }) }) } diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index d30f2697f3f..c037831f341 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -1,35 +1,78 @@ package dtos import ( + "fmt" "time" "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" - m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/models" ) type AlertRule struct { - Id int64 `json:"id"` - DashboardId int64 `json:"dashboardId"` - PanelId int64 `json:"panelId"` - Name string `json:"name"` - Message string `json:"message"` - State m.AlertStateType `json:"state"` - NewStateDate time.Time `json:"newStateDate"` - EvalDate time.Time `json:"evalDate"` - EvalData *simplejson.Json `json:"evalData"` - ExecutionError string `json:"executionError"` - Url string `json:"url"` - CanEdit bool `json:"canEdit"` + Id int64 `json:"id"` + DashboardId int64 `json:"dashboardId"` + PanelId int64 `json:"panelId"` + Name string `json:"name"` + Message string `json:"message"` + State models.AlertStateType `json:"state"` + NewStateDate time.Time `json:"newStateDate"` + EvalDate time.Time `json:"evalDate"` + EvalData *simplejson.Json `json:"evalData"` + ExecutionError string `json:"executionError"` + Url string `json:"url"` + CanEdit bool `json:"canEdit"` +} + +func formatShort(interval time.Duration) string { + var result string + + hours := interval / time.Hour + if hours > 0 { + result += fmt.Sprintf("%dh", hours) + } + + remaining := interval - (hours * time.Hour) + mins := remaining / time.Minute + if mins > 0 { + result += fmt.Sprintf("%dm", mins) + } + + remaining = remaining - (mins * time.Minute) + seconds := remaining / time.Second + if seconds > 0 { + result += fmt.Sprintf("%ds", seconds) + } + + return result +} + +func NewAlertNotification(notification *models.AlertNotification) *AlertNotification { + return &AlertNotification{ + Id: notification.Id, + Name: notification.Name, + Type: notification.Type, + IsDefault: notification.IsDefault, + Created: notification.Created, + Updated: notification.Updated, + Frequency: formatShort(notification.Frequency), + SendReminder: notification.SendReminder, + DisableResolveMessage: notification.DisableResolveMessage, + Settings: notification.Settings, + } } type AlertNotification struct { - Id int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + IsDefault bool `json:"isDefault"` + SendReminder bool `json:"sendReminder"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Frequency string `json:"frequency"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Settings *simplejson.Json `json:"settings"` } type AlertTestCommand struct { @@ -39,7 +82,7 @@ type AlertTestCommand struct { type AlertTestResult struct { Firing bool `json:"firing"` - State m.AlertStateType `json:"state"` + State models.AlertStateType `json:"state"` ConditionEvals string `json:"conditionEvals"` TimeMs string `json:"timeMs"` Error string `json:"error,omitempty"` @@ -59,9 +102,12 @@ type EvalMatch struct { } type NotificationTestCommand struct { - Name string `json:"name"` - Type string `json:"type"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name"` + Type string `json:"type"` + SendReminder bool `json:"sendReminder"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Frequency string `json:"frequency"` + Settings *simplejson.Json `json:"settings"` } type PauseAlertCommand struct { diff --git a/pkg/api/dtos/alerting_test.go b/pkg/api/dtos/alerting_test.go new file mode 100644 index 00000000000..f4c09f202cb --- /dev/null +++ b/pkg/api/dtos/alerting_test.go @@ -0,0 +1,35 @@ +package dtos + +import ( + "testing" + "time" +) + +func TestFormatShort(t *testing.T) { + tcs := []struct { + interval time.Duration + expected string + }{ + {interval: time.Hour, expected: "1h"}, + {interval: time.Hour + time.Minute, expected: "1h1m"}, + {interval: (time.Hour * 10) + time.Minute, expected: "10h1m"}, + {interval: (time.Hour * 10) + (time.Minute * 10) + time.Second, expected: "10h10m1s"}, + {interval: time.Minute * 10, expected: "10m"}, + } + + for _, tc := range tcs { + got := formatShort(tc.interval) + if got != tc.expected { + t.Errorf("expected %s got %s interval: %v", tc.expected, got, tc.interval) + } + + parsed, err := time.ParseDuration(tc.expected) + if err != nil { + t.Fatalf("could not parse expected duration") + } + + if parsed != tc.interval { + t.Errorf("expects the parsed duration to equal the interval. Got %v expected: %v", parsed, tc.interval) + } + } +} diff --git a/pkg/api/dtos/dashboard.go b/pkg/api/dtos/dashboard.go index e4c66aebbda..39a6dca580d 100644 --- a/pkg/api/dtos/dashboard.go +++ b/pkg/api/dtos/dashboard.go @@ -28,6 +28,7 @@ type DashboardMeta struct { FolderId int64 `json:"folderId"` FolderTitle string `json:"folderTitle"` FolderUrl string `json:"folderUrl"` + Provisioned bool `json:"provisioned"` } type DashboardFullWithMeta struct { diff --git a/pkg/api/dtos/index.go b/pkg/api/dtos/index.go index 8c7f505277d..bd3ac76eec8 100644 --- a/pkg/api/dtos/index.go +++ b/pkg/api/dtos/index.go @@ -13,6 +13,8 @@ type IndexViewData struct { Theme string NewGrafanaVersionExists bool NewGrafanaVersion string + AppName string + AppNameBodyClass string } type PluginCss struct { diff --git a/pkg/api/dtos/models.go b/pkg/api/dtos/models.go index a702b06fad5..6a130e62158 100644 --- a/pkg/api/dtos/models.go +++ b/pkg/api/dtos/models.go @@ -22,21 +22,22 @@ type LoginCommand struct { } type CurrentUser struct { - IsSignedIn bool `json:"isSignedIn"` - Id int64 `json:"id"` - Login string `json:"login"` - Email string `json:"email"` - Name string `json:"name"` - LightTheme bool `json:"lightTheme"` - OrgCount int `json:"orgCount"` - OrgId int64 `json:"orgId"` - OrgName string `json:"orgName"` - OrgRole m.RoleType `json:"orgRole"` - IsGrafanaAdmin bool `json:"isGrafanaAdmin"` - GravatarUrl string `json:"gravatarUrl"` - Timezone string `json:"timezone"` - Locale string `json:"locale"` - HelpFlags1 m.HelpFlags1 `json:"helpFlags1"` + IsSignedIn bool `json:"isSignedIn"` + Id int64 `json:"id"` + Login string `json:"login"` + Email string `json:"email"` + Name string `json:"name"` + LightTheme bool `json:"lightTheme"` + OrgCount int `json:"orgCount"` + OrgId int64 `json:"orgId"` + OrgName string `json:"orgName"` + OrgRole m.RoleType `json:"orgRole"` + IsGrafanaAdmin bool `json:"isGrafanaAdmin"` + GravatarUrl string `json:"gravatarUrl"` + Timezone string `json:"timezone"` + Locale string `json:"locale"` + HelpFlags1 m.HelpFlags1 `json:"helpFlags1"` + HasEditPermissionInFolders bool `json:"hasEditPermissionInFolders"` } type MetricRequest struct { @@ -50,6 +51,10 @@ type UserStars struct { } func GetGravatarUrl(text string) string { + if setting.DisableGravatar { + return setting.AppSubUrl + "/public/img/user_profile.png" + } + if text == "" { return "" } diff --git a/pkg/api/dtos/plugins.go b/pkg/api/dtos/plugins.go index f4281f877b3..edc9d96d1ac 100644 --- a/pkg/api/dtos/plugins.go +++ b/pkg/api/dtos/plugins.go @@ -19,9 +19,9 @@ type PluginSetting struct { JsonData map[string]interface{} `json:"jsonData"` DefaultNavUrl string `json:"defaultNavUrl"` - LatestVersion string `json:"latestVersion"` - HasUpdate bool `json:"hasUpdate"` - State string `json:"state"` + LatestVersion string `json:"latestVersion"` + HasUpdate bool `json:"hasUpdate"` + State plugins.PluginState `json:"state"` } type PluginListItem struct { @@ -34,7 +34,7 @@ type PluginListItem struct { LatestVersion string `json:"latestVersion"` HasUpdate bool `json:"hasUpdate"` DefaultNavUrl string `json:"defaultNavUrl"` - State string `json:"state"` + State plugins.PluginState `json:"state"` } type PluginList []PluginListItem @@ -57,4 +57,5 @@ type ImportDashboardCommand struct { Overwrite bool `json:"overwrite"` Dashboard *simplejson.Json `json:"dashboard"` Inputs []plugins.ImportDashboardInput `json:"inputs"` + FolderId int64 `json:"folderId"` } diff --git a/pkg/api/dtos/prefs.go b/pkg/api/dtos/prefs.go index 97e20bb4011..ef1ea4040e5 100644 --- a/pkg/api/dtos/prefs.go +++ b/pkg/api/dtos/prefs.go @@ -2,12 +2,12 @@ package dtos type Prefs struct { Theme string `json:"theme"` - HomeDashboardId int64 `json:"homeDashboardId"` + HomeDashboardID int64 `json:"homeDashboardId"` Timezone string `json:"timezone"` } type UpdatePrefsCmd struct { Theme string `json:"theme"` - HomeDashboardId int64 `json:"homeDashboardId"` + HomeDashboardID int64 `json:"homeDashboardId"` Timezone string `json:"timezone"` } diff --git a/pkg/api/folder.go b/pkg/api/folder.go index 143892fa6e8..0e08343b556 100644 --- a/pkg/api/folder.go +++ b/pkg/api/folder.go @@ -28,30 +28,30 @@ func GetFolders(c *m.ReqContext) Response { }) } - return Json(200, result) + return JSON(200, result) } -func GetFolderByUid(c *m.ReqContext) Response { +func GetFolderByUID(c *m.ReqContext) Response { s := dashboards.NewFolderService(c.OrgId, c.SignedInUser) - folder, err := s.GetFolderByUid(c.Params(":uid")) + folder, err := s.GetFolderByUID(c.Params(":uid")) if err != nil { return toFolderError(err) } g := guardian.New(folder.Id, c.OrgId, c.SignedInUser) - return Json(200, toFolderDto(g, folder)) + return JSON(200, toFolderDto(g, folder)) } -func GetFolderById(c *m.ReqContext) Response { +func GetFolderByID(c *m.ReqContext) Response { s := dashboards.NewFolderService(c.OrgId, c.SignedInUser) - folder, err := s.GetFolderById(c.ParamsInt64(":id")) + folder, err := s.GetFolderByID(c.ParamsInt64(":id")) if err != nil { return toFolderError(err) } g := guardian.New(folder.Id, c.OrgId, c.SignedInUser) - return Json(200, toFolderDto(g, folder)) + return JSON(200, toFolderDto(g, folder)) } func CreateFolder(c *m.ReqContext, cmd m.CreateFolderCommand) Response { @@ -62,7 +62,7 @@ func CreateFolder(c *m.ReqContext, cmd m.CreateFolderCommand) Response { } g := guardian.New(cmd.Result.Id, c.OrgId, c.SignedInUser) - return Json(200, toFolderDto(g, cmd.Result)) + return JSON(200, toFolderDto(g, cmd.Result)) } func UpdateFolder(c *m.ReqContext, cmd m.UpdateFolderCommand) Response { @@ -73,7 +73,7 @@ func UpdateFolder(c *m.ReqContext, cmd m.UpdateFolderCommand) Response { } g := guardian.New(cmd.Result.Id, c.OrgId, c.SignedInUser) - return Json(200, toFolderDto(g, cmd.Result)) + return JSON(200, toFolderDto(g, cmd.Result)) } func DeleteFolder(c *m.ReqContext) Response { @@ -83,7 +83,7 @@ func DeleteFolder(c *m.ReqContext) Response { return toFolderError(err) } - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "title": f.Title, "message": fmt.Sprintf("Folder %s deleted", f.Title), }) @@ -95,7 +95,7 @@ func toFolderDto(g guardian.DashboardGuardian, folder *m.Folder) dtos.Folder { canAdmin, _ := g.CanAdmin() // Finding creator and last updater of the folder - updater, creator := "Anonymous", "Anonymous" + updater, creator := anonString, anonString if folder.CreatedBy > 0 { creator = getUserLogin(folder.CreatedBy) } @@ -127,20 +127,20 @@ func toFolderError(err error) Response { err == m.ErrDashboardTypeMismatch || err == m.ErrDashboardInvalidUid || err == m.ErrDashboardUidToLong { - return ApiError(400, err.Error(), nil) + return Error(400, err.Error(), nil) } if err == m.ErrFolderAccessDenied { - return ApiError(403, "Access denied", err) + return Error(403, "Access denied", err) } if err == m.ErrFolderNotFound { - return Json(404, util.DynMap{"status": "not-found", "message": m.ErrFolderNotFound.Error()}) + return JSON(404, util.DynMap{"status": "not-found", "message": m.ErrFolderNotFound.Error()}) } if err == m.ErrFolderVersionMismatch { - return Json(412, util.DynMap{"status": "version-mismatch", "message": m.ErrFolderVersionMismatch.Error()}) + return JSON(412, util.DynMap{"status": "version-mismatch", "message": m.ErrFolderVersionMismatch.Error()}) } - return ApiError(500, "Folder API error", err) + return Error(500, "Folder API error", err) } diff --git a/pkg/api/folder_permission.go b/pkg/api/folder_permission.go index 1b04eb20e53..d19ec848ab2 100644 --- a/pkg/api/folder_permission.go +++ b/pkg/api/folder_permission.go @@ -12,7 +12,7 @@ import ( func GetFolderPermissionList(c *m.ReqContext) Response { s := dashboards.NewFolderService(c.OrgId, c.SignedInUser) - folder, err := s.GetFolderByUid(c.Params(":uid")) + folder, err := s.GetFolderByUID(c.Params(":uid")) if err != nil { return toFolderError(err) @@ -26,24 +26,30 @@ func GetFolderPermissionList(c *m.ReqContext) Response { acl, err := g.GetAcl() if err != nil { - return ApiError(500, "Failed to get folder permissions", err) + return Error(500, "Failed to get folder permissions", err) } for _, perm := range acl { perm.FolderId = folder.Id perm.DashboardId = 0 + perm.UserAvatarUrl = dtos.GetGravatarUrl(perm.UserEmail) + + if perm.TeamId > 0 { + perm.TeamAvatarUrl = dtos.GetGravatarUrlWithDefault(perm.TeamEmail, perm.Team) + } + if perm.Slug != "" { perm.Url = m.GetDashboardFolderUrl(perm.IsFolder, perm.Uid, perm.Slug) } } - return Json(200, acl) + return JSON(200, acl) } func UpdateFolderPermissions(c *m.ReqContext, apiCmd dtos.UpdateDashboardAclCommand) Response { s := dashboards.NewFolderService(c.OrgId, c.SignedInUser) - folder, err := s.GetFolderByUid(c.Params(":uid")) + folder, err := s.GetFolderByUID(c.Params(":uid")) if err != nil { return toFolderError(err) @@ -79,13 +85,13 @@ func UpdateFolderPermissions(c *m.ReqContext, apiCmd dtos.UpdateDashboardAclComm if err != nil { if err == guardian.ErrGuardianPermissionExists || err == guardian.ErrGuardianOverride { - return ApiError(400, err.Error(), err) + return Error(400, err.Error(), err) } - return ApiError(500, "Error while checking folder permissions", err) + return Error(500, "Error while checking folder permissions", err) } - return ApiError(403, "Cannot remove own admin permission for a folder", nil) + return Error(403, "Cannot remove own admin permission for a folder", nil) } if err := bus.Dispatch(&cmd); err != nil { @@ -97,11 +103,11 @@ func UpdateFolderPermissions(c *m.ReqContext, apiCmd dtos.UpdateDashboardAclComm } if err == m.ErrFolderAclInfoMissing || err == m.ErrFolderPermissionFolderEmpty { - return ApiError(409, err.Error(), err) + return Error(409, err.Error(), err) } - return ApiError(500, "Failed to create permission", err) + return Error(500, "Failed to create permission", err) } - return ApiSuccess("Folder permissions updated") + return Success("Folder permissions updated") } diff --git a/pkg/api/folder_permission_test.go b/pkg/api/folder_permission_test.go index 00d025fdce2..64a746ca937 100644 --- a/pkg/api/folder_permission_test.go +++ b/pkg/api/folder_permission_test.go @@ -17,7 +17,7 @@ func TestFolderPermissionApiEndpoint(t *testing.T) { Convey("Folder permissions test", t, func() { Convey("Given folder not exists", func() { mock := &fakeFolderService{ - GetFolderByUidError: m.ErrFolderNotFound, + GetFolderByUIDError: m.ErrFolderNotFound, } origNewFolderService := dashboards.NewFolderService @@ -49,7 +49,7 @@ func TestFolderPermissionApiEndpoint(t *testing.T) { guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{CanAdminValue: false}) mock := &fakeFolderService{ - GetFolderByUidResult: &m.Folder{ + GetFolderByUIDResult: &m.Folder{ Id: 1, Uid: "uid", Title: "Folder", @@ -96,7 +96,7 @@ func TestFolderPermissionApiEndpoint(t *testing.T) { }) mock := &fakeFolderService{ - GetFolderByUidResult: &m.Folder{ + GetFolderByUIDResult: &m.Folder{ Id: 1, Uid: "uid", Title: "Folder", @@ -142,7 +142,7 @@ func TestFolderPermissionApiEndpoint(t *testing.T) { }) mock := &fakeFolderService{ - GetFolderByUidResult: &m.Folder{ + GetFolderByUIDResult: &m.Folder{ Id: 1, Uid: "uid", Title: "Folder", @@ -178,7 +178,7 @@ func TestFolderPermissionApiEndpoint(t *testing.T) { ) mock := &fakeFolderService{ - GetFolderByUidResult: &m.Folder{ + GetFolderByUIDResult: &m.Folder{ Id: 1, Uid: "uid", Title: "Folder", @@ -226,7 +226,7 @@ func updateFolderPermissionScenario(desc string, url string, routePattern string sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.OrgId = TestOrgID sc.context.UserId = TestUserID diff --git a/pkg/api/folder_test.go b/pkg/api/folder_test.go index 7cefdcf8544..880de338c8f 100644 --- a/pkg/api/folder_test.go +++ b/pkg/api/folder_test.go @@ -133,16 +133,6 @@ func TestFoldersApiEndpoint(t *testing.T) { }) } -func callGetFolderByUid(sc *scenarioContext) { - sc.handlerFunc = GetFolderByUid - sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() -} - -func callDeleteFolder(sc *scenarioContext) { - sc.handlerFunc = DeleteFolder - sc.fakeReqWithParams("DELETE", sc.url, map[string]string{}).exec() -} - func callCreateFolder(sc *scenarioContext) { sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec() } @@ -152,7 +142,7 @@ func createFolderScenario(desc string, url string, routePattern string, mock *fa defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.SignedInUser = &m.SignedInUser{OrgId: TestOrgID, UserId: TestUserID} @@ -181,7 +171,7 @@ func updateFolderScenario(desc string, url string, routePattern string, mock *fa defer bus.ClearBusHandlers() sc := setupScenarioContext(url) - sc.defaultHandler = wrap(func(c *m.ReqContext) Response { + sc.defaultHandler = Wrap(func(c *m.ReqContext) Response { sc.context = c sc.context.SignedInUser = &m.SignedInUser{OrgId: TestOrgID, UserId: TestUserID} @@ -204,10 +194,10 @@ func updateFolderScenario(desc string, url string, routePattern string, mock *fa type fakeFolderService struct { GetFoldersResult []*m.Folder GetFoldersError error - GetFolderByUidResult *m.Folder - GetFolderByUidError error - GetFolderByIdResult *m.Folder - GetFolderByIdError error + GetFolderByUIDResult *m.Folder + GetFolderByUIDError error + GetFolderByIDResult *m.Folder + GetFolderByIDError error CreateFolderResult *m.Folder CreateFolderError error UpdateFolderResult *m.Folder @@ -221,12 +211,12 @@ func (s *fakeFolderService) GetFolders(limit int) ([]*m.Folder, error) { return s.GetFoldersResult, s.GetFoldersError } -func (s *fakeFolderService) GetFolderById(id int64) (*m.Folder, error) { - return s.GetFolderByIdResult, s.GetFolderByIdError +func (s *fakeFolderService) GetFolderByID(id int64) (*m.Folder, error) { + return s.GetFolderByIDResult, s.GetFolderByIDError } -func (s *fakeFolderService) GetFolderByUid(uid string) (*m.Folder, error) { - return s.GetFolderByUidResult, s.GetFolderByUidError +func (s *fakeFolderService) GetFolderByUID(uid string) (*m.Folder, error) { + return s.GetFolderByUIDResult, s.GetFolderByUIDError } func (s *fakeFolderService) CreateFolder(cmd *m.CreateFolderCommand) error { @@ -234,7 +224,7 @@ func (s *fakeFolderService) CreateFolder(cmd *m.CreateFolderCommand) error { return s.CreateFolderError } -func (s *fakeFolderService) UpdateFolder(existingUid string, cmd *m.UpdateFolderCommand) error { +func (s *fakeFolderService) UpdateFolder(existingUID string, cmd *m.UpdateFolderCommand) error { cmd.Result = s.UpdateFolderResult return s.UpdateFolderError } diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go index 5cd52122c3f..65affe83e98 100644 --- a/pkg/api/frontendsettings.go +++ b/pkg/api/frontendsettings.go @@ -11,7 +11,7 @@ import ( "github.com/grafana/grafana/pkg/util" ) -func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { +func (hs *HTTPServer) getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { orgDataSources := make([]*m.DataSource, 0) if c.OrgId != 0 { @@ -22,7 +22,20 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { return nil, err } - orgDataSources = query.Result + dsFilterQuery := m.DatasourcesPermissionFilterQuery{ + User: c.SignedInUser, + Datasources: query.Result, + } + + if err := bus.Dispatch(&dsFilterQuery); err != nil { + if err != bus.ErrHandlerNotFound { + return nil, err + } + + orgDataSources = query.Result + } else { + orgDataSources = dsFilterQuery.Result + } } datasources := make(map[string]interface{}) @@ -120,6 +133,10 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { panels := map[string]interface{}{} for _, panel := range enabledPlugins.Panels { + if panel.State == plugins.PluginStateAlpha && !hs.Cfg.EnableAlphaPanels { + continue + } + panels[panel.Id] = map[string]interface{}{ "module": panel.Module, "baseUrl": panel.BaseUrl, @@ -132,19 +149,22 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { } jsonObj := map[string]interface{}{ - "defaultDatasource": defaultDatasource, - "datasources": datasources, - "panels": panels, - "appSubUrl": setting.AppSubUrl, - "allowOrgCreate": (setting.AllowUserOrgCreate && c.IsSignedIn) || c.IsGrafanaAdmin, - "authProxyEnabled": setting.AuthProxyEnabled, - "ldapEnabled": setting.LdapEnabled, - "alertingEnabled": setting.AlertingEnabled, - "googleAnalyticsId": setting.GoogleAnalyticsId, - "disableLoginForm": setting.DisableLoginForm, - "externalUserMngInfo": setting.ExternalUserMngInfo, - "externalUserMngLinkUrl": setting.ExternalUserMngLinkUrl, - "externalUserMngLinkName": setting.ExternalUserMngLinkName, + "defaultDatasource": defaultDatasource, + "datasources": datasources, + "panels": panels, + "appSubUrl": setting.AppSubUrl, + "allowOrgCreate": (setting.AllowUserOrgCreate && c.IsSignedIn) || c.IsGrafanaAdmin, + "authProxyEnabled": setting.AuthProxyEnabled, + "ldapEnabled": setting.LdapEnabled, + "alertingEnabled": setting.AlertingEnabled, + "alertingErrorOrTimeout": setting.AlertingErrorOrTimeout, + "alertingNoDataOrNullValues": setting.AlertingNoDataOrNullValues, + "exploreEnabled": setting.ExploreEnabled, + "googleAnalyticsId": setting.GoogleAnalyticsId, + "disableLoginForm": setting.DisableLoginForm, + "externalUserMngInfo": setting.ExternalUserMngInfo, + "externalUserMngLinkUrl": setting.ExternalUserMngLinkUrl, + "externalUserMngLinkName": setting.ExternalUserMngLinkName, "buildInfo": map[string]interface{}{ "version": setting.BuildVersion, "commit": setting.BuildCommit, @@ -152,6 +172,7 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) { "latestVersion": plugins.GrafanaLatestVersion, "hasUpdate": plugins.GrafanaHasUpdate, "env": setting.Env, + "isEnterprise": setting.IsEnterprise, }, } @@ -179,8 +200,8 @@ func getPanelSort(id string) int { return sort } -func GetFrontendSettings(c *m.ReqContext) { - settings, err := getFrontendSettingsMap(c) +func (hs *HTTPServer) GetFrontendSettings(c *m.ReqContext) { + settings, err := hs.getFrontendSettingsMap(c) if err != nil { c.JsonApiErr(400, "Failed to get frontend settings", err) return diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index b911780913d..ce28e4716ee 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -11,11 +11,11 @@ import ( "path" "time" + "github.com/grafana/grafana/pkg/api/routing" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - gocache "github.com/patrickmn/go-cache" macaron "gopkg.in/macaron.v1" "github.com/grafana/grafana/pkg/api/live" @@ -26,40 +26,71 @@ import ( "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/cache" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/hooks" + "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" ) -type HttpServer struct { +func init() { + registry.Register(®istry.Descriptor{ + Name: "HTTPServer", + Instance: &HTTPServer{}, + InitPriority: registry.High, + }) +} + +type HTTPServer struct { log log.Logger macaron *macaron.Macaron context context.Context streamManager *live.StreamManager - cache *gocache.Cache + httpSrv *http.Server - httpSrv *http.Server + RouteRegister routing.RouteRegister `inject:""` + Bus bus.Bus `inject:""` + RenderService rendering.Service `inject:""` + Cfg *setting.Cfg `inject:""` + HooksService *hooks.HooksService `inject:""` + CacheService *cache.CacheService `inject:""` + DatasourceCache datasources.CacheService `inject:""` } -func NewHttpServer() *HttpServer { - return &HttpServer{ - log: log.New("http.server"), - cache: gocache.New(5*time.Minute, 10*time.Minute), - } -} +func (hs *HTTPServer) Init() error { + hs.log = log.New("http.server") -func (hs *HttpServer) Start(ctx context.Context) error { - var err error - - hs.context = ctx hs.streamManager = live.NewStreamManager() hs.macaron = hs.newMacaron() hs.registerRoutes() + return nil +} + +func (hs *HTTPServer) Run(ctx context.Context) error { + var err error + + hs.context = ctx + + hs.applyRoutes() hs.streamManager.Run(ctx) listenAddr := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort) - hs.log.Info("Initializing HTTP Server", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath) + hs.log.Info("HTTP Server Listen", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath) hs.httpSrv = &http.Server{Addr: listenAddr, Handler: hs.macaron} + + // handle http shutdown on server context done + go func() { + <-ctx.Done() + // Hacky fix for race condition between ListenAndServe and Shutdown + time.Sleep(time.Millisecond * 100) + if err := hs.httpSrv.Shutdown(context.Background()); err != nil { + hs.log.Error("Failed to shutdown server", "error", err) + } + }() + switch setting.Protocol { case setting.HTTP: err = hs.httpSrv.ListenAndServe() @@ -74,12 +105,15 @@ func (hs *HttpServer) Start(ctx context.Context) error { return nil } case setting.SOCKET: - ln, err := net.Listen("unix", setting.SocketPath) + ln, err := net.ListenUnix("unix", &net.UnixAddr{Name: setting.SocketPath, Net: "unix"}) if err != nil { hs.log.Debug("server was shutdown gracefully") return nil } + // Make socket writable by group + os.Chmod(setting.SocketPath, 0660) + err = hs.httpSrv.Serve(ln) if err != nil { hs.log.Debug("server was shutdown gracefully") @@ -93,13 +127,7 @@ func (hs *HttpServer) Start(ctx context.Context) error { return err } -func (hs *HttpServer) Shutdown(ctx context.Context) error { - err := hs.httpSrv.Shutdown(ctx) - hs.log.Info("Stopped HTTP server") - return err -} - -func (hs *HttpServer) listenAndServeTLS(certfile, keyfile string) error { +func (hs *HTTPServer) listenAndServeTLS(certfile, keyfile string) error { if certfile == "" { return fmt.Errorf("cert_file cannot be empty when using HTTPS") } @@ -136,15 +164,35 @@ func (hs *HttpServer) listenAndServeTLS(certfile, keyfile string) error { } hs.httpSrv.TLSConfig = tlsCfg - hs.httpSrv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0) + hs.httpSrv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) return hs.httpSrv.ListenAndServeTLS(setting.CertFile, setting.KeyFile) } -func (hs *HttpServer) newMacaron() *macaron.Macaron { +func (hs *HTTPServer) newMacaron() *macaron.Macaron { macaron.Env = setting.Env m := macaron.New() + // automatically set HEAD for every GET + m.SetAutoHead(true) + + return m +} + +func (hs *HTTPServer) applyRoutes() { + // start with middlewares & static routes + hs.addMiddlewaresAndStaticRoutes() + // then add view routes & api routes + hs.RouteRegister.Register(hs.macaron) + // then custom app proxy routes + hs.initAppPluginRoutes(hs.macaron) + // lastly not found route + hs.macaron.NotFound(hs.NotFoundHandler) +} + +func (hs *HTTPServer) addMiddlewaresAndStaticRoutes() { + m := hs.macaron + m.Use(middleware.Logger()) if setting.EnableGzip { @@ -156,14 +204,15 @@ func (hs *HttpServer) newMacaron() *macaron.Macaron { for _, route := range plugins.StaticRoutes { pluginRoute := path.Join("/public/plugins/", route.PluginId) hs.log.Debug("Plugins: Adding route", "route", pluginRoute, "dir", route.Directory) - hs.mapStatic(m, route.Directory, "", pluginRoute) + hs.mapStatic(hs.macaron, route.Directory, "", pluginRoute) } + hs.mapStatic(m, setting.StaticRootPath, "build", "public/build") hs.mapStatic(m, setting.StaticRootPath, "", "public") hs.mapStatic(m, setting.StaticRootPath, "robots.txt", "robots.txt") if setting.ImageUploadProvider == "local" { - hs.mapStatic(m, setting.ImagesDir, "", "/public/img/attachments") + hs.mapStatic(m, hs.Cfg.ImagesDir, "", "/public/img/attachments") } m.Use(macaron.Renderer(macaron.RenderOptions{ @@ -175,7 +224,7 @@ func (hs *HttpServer) newMacaron() *macaron.Macaron { m.Use(hs.healthHandler) m.Use(hs.metricsEndpoint) m.Use(middleware.GetContextHandler()) - m.Use(middleware.Sessioner(&setting.SessionOptions)) + m.Use(middleware.Sessioner(&setting.SessionOptions, setting.SessionConnMaxLifetime)) m.Use(middleware.OrgRedirect()) // needs to be after context handler @@ -183,12 +232,15 @@ func (hs *HttpServer) newMacaron() *macaron.Macaron { m.Use(middleware.ValidateHostHeader(setting.Domain)) } + m.Use(middleware.HandleNoCacheHeader()) m.Use(middleware.AddDefaultResponseHeaders()) - - return m } -func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) { +func (hs *HTTPServer) metricsEndpoint(ctx *macaron.Context) { + if !hs.Cfg.MetricsEndpointEnabled { + return + } + if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/metrics" { return } @@ -197,7 +249,7 @@ func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) { ServeHTTP(ctx.Resp, ctx.Req.Request) } -func (hs *HttpServer) healthHandler(ctx *macaron.Context) { +func (hs *HTTPServer) healthHandler(ctx *macaron.Context) { notHeadOrGet := ctx.Req.Method != http.MethodGet && ctx.Req.Method != http.MethodHead if notHeadOrGet || ctx.Req.URL.Path != "/api/health" { return @@ -221,11 +273,17 @@ func (hs *HttpServer) healthHandler(ctx *macaron.Context) { ctx.Resp.Write(dataBytes) } -func (hs *HttpServer) mapStatic(m *macaron.Macaron, rootDir string, dir string, prefix string) { +func (hs *HTTPServer) mapStatic(m *macaron.Macaron, rootDir string, dir string, prefix string) { headers := func(c *macaron.Context) { c.Resp.Header().Set("Cache-Control", "public, max-age=3600") } + if prefix == "public/build" { + headers = func(c *macaron.Context) { + c.Resp.Header().Set("Cache-Control", "public, max-age=31536000") + } + } + if setting.Env == setting.DEV { headers = func(c *macaron.Context) { c.Resp.Header().Set("Cache-Control", "max-age=0, must-revalidate, no-cache") diff --git a/pkg/api/index.go b/pkg/api/index.go index e50c59e082a..253fa9c17af 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -11,13 +11,19 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { - settings, err := getFrontendSettingsMap(c) +const ( + // Themes + lightName = "light" + darkName = "dark" +) + +func (hs *HTTPServer) setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { + settings, err := hs.getFrontendSettingsMap(c) if err != nil { return nil, err } - prefsQuery := m.GetPreferencesWithDefaultsQuery{OrgId: c.OrgId, UserId: c.UserId} + prefsQuery := m.GetPreferencesWithDefaultsQuery{User: c.SignedInUser} if err := bus.Dispatch(&prefsQuery); err != nil { return nil, err } @@ -32,44 +38,52 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { locale = parts[0] } - appUrl := setting.AppUrl - appSubUrl := setting.AppSubUrl + appURL := setting.AppUrl + appSubURL := setting.AppSubUrl // special case when doing localhost call from phantomjs if c.IsRenderCall { - appUrl = fmt.Sprintf("%s://localhost:%s", setting.Protocol, setting.HttpPort) - appSubUrl = "" + appURL = fmt.Sprintf("%s://localhost:%s", setting.Protocol, setting.HttpPort) + appSubURL = "" settings["appSubUrl"] = "" } + hasEditPermissionInFoldersQuery := m.HasEditPermissionInFoldersQuery{SignedInUser: c.SignedInUser} + if err := bus.Dispatch(&hasEditPermissionInFoldersQuery); err != nil { + return nil, err + } + var data = dtos.IndexViewData{ User: &dtos.CurrentUser{ - Id: c.UserId, - IsSignedIn: c.IsSignedIn, - Login: c.Login, - Email: c.Email, - Name: c.Name, - OrgCount: c.OrgCount, - OrgId: c.OrgId, - OrgName: c.OrgName, - OrgRole: c.OrgRole, - GravatarUrl: dtos.GetGravatarUrl(c.Email), - IsGrafanaAdmin: c.IsGrafanaAdmin, - LightTheme: prefs.Theme == "light", - Timezone: prefs.Timezone, - Locale: locale, - HelpFlags1: c.HelpFlags1, + Id: c.UserId, + IsSignedIn: c.IsSignedIn, + Login: c.Login, + Email: c.Email, + Name: c.Name, + OrgCount: c.OrgCount, + OrgId: c.OrgId, + OrgName: c.OrgName, + OrgRole: c.OrgRole, + GravatarUrl: dtos.GetGravatarUrl(c.Email), + IsGrafanaAdmin: c.IsGrafanaAdmin, + LightTheme: prefs.Theme == lightName, + Timezone: prefs.Timezone, + Locale: locale, + HelpFlags1: c.HelpFlags1, + HasEditPermissionInFolders: hasEditPermissionInFoldersQuery.Result, }, Settings: settings, Theme: prefs.Theme, - AppUrl: appUrl, - AppSubUrl: appSubUrl, + AppUrl: appURL, + AppSubUrl: appSubURL, GoogleAnalyticsId: setting.GoogleAnalyticsId, GoogleTagManagerId: setting.GoogleTagManagerId, BuildVersion: setting.BuildVersion, BuildCommit: setting.BuildCommit, NewGrafanaVersion: plugins.GrafanaLatestVersion, NewGrafanaVersionExists: plugins.GrafanaHasUpdate, + AppName: setting.ApplicationName, + AppNameBodyClass: getAppNameBodyClass(setting.ApplicationName), } if setting.DisableGravatar { @@ -80,23 +94,32 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { data.User.Name = data.User.Login } - themeUrlParam := c.Query("theme") - if themeUrlParam == "light" { + themeURLParam := c.Query("theme") + if themeURLParam == lightName { data.User.LightTheme = true - data.Theme = "light" + data.Theme = lightName + } else if themeURLParam == darkName { + data.User.LightTheme = false + data.Theme = darkName } - if c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR { + if hasEditPermissionInFoldersQuery.Result { + children := []*dtos.NavLink{ + {Text: "Dashboard", Icon: "gicon gicon-dashboard-new", Url: setting.AppSubUrl + "/dashboard/new"}, + } + + if c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR { + children = append(children, &dtos.NavLink{Text: "Folder", SubTitle: "Create a new folder to organize your dashboards", Id: "folder", Icon: "gicon gicon-folder-new", Url: setting.AppSubUrl + "/dashboards/folder/new"}) + } + + children = append(children, &dtos.NavLink{Text: "Import", SubTitle: "Import dashboard from file or Grafana.com", Id: "import", Icon: "gicon gicon-dashboard-import", Url: setting.AppSubUrl + "/dashboard/import"}) + data.NavTree = append(data.NavTree, &dtos.NavLink{ - Text: "Create", - Id: "create", - Icon: "fa fa-fw fa-plus", - Url: setting.AppSubUrl + "/dashboard/new", - Children: []*dtos.NavLink{ - {Text: "Dashboard", Icon: "gicon gicon-dashboard-new", Url: setting.AppSubUrl + "/dashboard/new"}, - {Text: "Folder", SubTitle: "Create a new folder to organize your dashboards", Id: "folder", Icon: "gicon gicon-folder-new", Url: setting.AppSubUrl + "/dashboards/folder/new"}, - {Text: "Import", SubTitle: "Import dashboard from file or Grafana.com", Id: "import", Icon: "gicon gicon-dashboard-import", Url: setting.AppSubUrl + "/dashboard/import"}, - }, + Text: "Create", + Id: "create", + Icon: "fa fa-fw fa-plus", + Url: setting.AppSubUrl + "/dashboard/new", + Children: children, }) } @@ -117,10 +140,28 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { Children: dashboardChildNavs, }) + if setting.ExploreEnabled && (c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR) { + data.NavTree = append(data.NavTree, &dtos.NavLink{ + Text: "Explore", + Id: "explore", + SubTitle: "Explore your data", + Icon: "fa fa-rocket", + Url: setting.AppSubUrl + "/explore", + Children: []*dtos.NavLink{ + {Text: "New tab", Icon: "gicon gicon-dashboard-new", Url: setting.AppSubUrl + "/explore"}, + }, + }) + } + if c.IsSignedIn { + // Only set login if it's different from the name + var login string + if c.SignedInUser.Login != c.SignedInUser.NameOrFallback() { + login = c.SignedInUser.Login + } profileNode := &dtos.NavLink{ Text: c.SignedInUser.NameOrFallback(), - SubTitle: c.SignedInUser.Login, + SubTitle: login, Id: "profile", Img: data.User.GravatarUrl, Url: setting.AppSubUrl + "/profile", @@ -204,7 +245,7 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { } } - if c.OrgRole == m.ROLE_ADMIN { + if c.IsGrafanaAdmin || c.OrgRole == m.ROLE_ADMIN { cfgNode := &dtos.NavLink{ Id: "cfg", Text: "Configuration", @@ -258,10 +299,24 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { }, } - if c.IsGrafanaAdmin { + if c.OrgRole != m.ROLE_ADMIN { + cfgNode = &dtos.NavLink{ + Id: "cfg", + Text: "Configuration", + SubTitle: "Organization: " + c.OrgName, + Icon: "gicon gicon-cog", + Url: setting.AppSubUrl + "/admin/users", + Children: make([]*dtos.NavLink, 0), + } + } + + if c.OrgRole == m.ROLE_ADMIN && c.IsGrafanaAdmin { cfgNode.Children = append(cfgNode.Children, &dtos.NavLink{ Divider: true, HideFromTabs: true, Id: "admin-divider", Text: "Text", }) + } + + if c.IsGrafanaAdmin { cfgNode.Children = append(cfgNode.Children, &dtos.NavLink{ Text: "Server Admin", HideFromTabs: true, @@ -284,6 +339,7 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { data.NavTree = append(data.NavTree, &dtos.NavLink{ Text: "Help", + SubTitle: fmt.Sprintf(`%s v%s (%s)`, setting.ApplicationName, setting.BuildVersion, setting.BuildCommit), Id: "help", Url: "#", Icon: "gicon gicon-question", @@ -295,28 +351,41 @@ func setIndexViewData(c *m.ReqContext) (*dtos.IndexViewData, error) { }, }) + hs.HooksService.RunIndexDataHooks(&data) return &data, nil } -func Index(c *m.ReqContext) { - if data, err := setIndexViewData(c); err != nil { +func (hs *HTTPServer) Index(c *m.ReqContext) { + data, err := hs.setIndexViewData(c) + if err != nil { c.Handle(500, "Failed to get settings", err) return - } else { - c.HTML(200, "index", data) } + c.HTML(200, "index", data) } -func NotFoundHandler(c *m.ReqContext) { +func (hs *HTTPServer) NotFoundHandler(c *m.ReqContext) { if c.IsApiRequest() { c.JsonApiErr(404, "Not found", nil) return } - if data, err := setIndexViewData(c); err != nil { + data, err := hs.setIndexViewData(c) + if err != nil { c.Handle(500, "Failed to get settings", err) return - } else { - c.HTML(404, "index", data) + } + + c.HTML(404, "index", data) +} + +func getAppNameBodyClass(name string) string { + switch name { + case setting.APP_NAME: + return "app-grafana" + case setting.APP_NAME_ENTERPRISE: + return "app-enterprise" + default: + return "" } } diff --git a/pkg/api/live/conn.go b/pkg/api/live/conn.go index f2a041d7631..0fae7f75b73 100644 --- a/pkg/api/live/conn.go +++ b/pkg/api/live/conn.go @@ -70,7 +70,7 @@ func (c *connection) readPump() { func (c *connection) handleMessage(message []byte) { json, err := simplejson.NewJson(message) if err != nil { - log.Error(3, "Unreadable message on websocket channel:", err) + log.Error(3, "Unreadable message on websocket channel. error: %v", err) } msgType := json.Get("action").MustString() diff --git a/pkg/api/live/hub.go b/pkg/api/live/hub.go index 37ab5667e55..9708bc515d1 100644 --- a/pkg/api/live/hub.go +++ b/pkg/api/live/hub.go @@ -37,9 +37,6 @@ func newHub() *hub { } } -func (h *hub) removeConnection() { -} - func (h *hub) run(ctx context.Context) { for { select { diff --git a/pkg/api/login.go b/pkg/api/login.go index 2ca2ce5a3e2..1083f89adfd 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -14,11 +14,11 @@ import ( ) const ( - VIEW_INDEX = "index" + ViewIndex = "index" ) -func LoginView(c *m.ReqContext) { - viewData, err := setIndexViewData(c) +func (hs *HTTPServer) LoginView(c *m.ReqContext) { + viewData, err := hs.setIndexViewData(c) if err != nil { c.Handle(500, "Failed to get settings", err) return @@ -40,7 +40,7 @@ func LoginView(c *m.ReqContext) { } if !tryLoginUsingRememberCookie(c) { - c.HTML(200, VIEW_INDEX, viewData) + c.HTML(200, ViewIndex, viewData) return } @@ -78,7 +78,13 @@ func tryLoginUsingRememberCookie(c *m.ReqContext) bool { user := userQuery.Result // validate remember me cookie - if val, _ := c.GetSuperSecureCookie(user.Rands+user.Password, setting.CookieRememberName); val != user.Login { + signingKey := user.Rands + user.Password + if len(signingKey) < 10 { + c.Logger.Error("Invalid user signingKey") + return false + } + + if val, _ := c.GetSuperSecureCookie(signingKey, setting.CookieRememberName); val != user.Login { return false } @@ -87,7 +93,7 @@ func tryLoginUsingRememberCookie(c *m.ReqContext) bool { return true } -func LoginApiPing(c *m.ReqContext) { +func LoginAPIPing(c *m.ReqContext) { if !tryLoginUsingRememberCookie(c) { c.JsonApiErr(401, "Unauthorized", nil) return @@ -98,21 +104,22 @@ func LoginApiPing(c *m.ReqContext) { func LoginPost(c *m.ReqContext, cmd dtos.LoginCommand) Response { if setting.DisableLoginForm { - return ApiError(401, "Login is disabled", nil) + return Error(401, "Login is disabled", nil) } - authQuery := login.LoginUserQuery{ - Username: cmd.User, - Password: cmd.Password, - IpAddress: c.Req.RemoteAddr, + authQuery := &m.LoginUserQuery{ + ReqContext: c, + Username: cmd.User, + Password: cmd.Password, + IpAddress: c.Req.RemoteAddr, } - if err := bus.Dispatch(&authQuery); err != nil { + if err := bus.Dispatch(authQuery); err != nil { if err == login.ErrInvalidCredentials || err == login.ErrTooManyLoginAttempts { - return ApiError(401, "Invalid username or password", err) + return Error(401, "Invalid username or password", err) } - return ApiError(500, "Error while trying to authenticate user", err) + return Error(500, "Error while trying to authenticate user", err) } user := authQuery.User @@ -130,7 +137,7 @@ func LoginPost(c *m.ReqContext, cmd dtos.LoginCommand) Response { metrics.M_Api_Login_Post.Inc() - return Json(200, result) + return JSON(200, result) } func loginUserWithUser(user *m.User, c *m.ReqContext) { @@ -154,5 +161,9 @@ func Logout(c *m.ReqContext) { c.SetCookie(setting.CookieUserName, "", -1, setting.AppSubUrl+"/") c.SetCookie(setting.CookieRememberName, "", -1, setting.AppSubUrl+"/") c.Session.Destory(c.Context) - c.Redirect(setting.AppSubUrl + "/login") + if setting.SignoutRedirectUrl != "" { + c.Redirect(setting.SignoutRedirectUrl) + } else { + c.Redirect(setting.AppSubUrl + "/login") + } } diff --git a/pkg/api/login_oauth.go b/pkg/api/login_oauth.go index 1dba38e9cbd..fe4fa93b621 100644 --- a/pkg/api/login_oauth.go +++ b/pkg/api/login_oauth.go @@ -6,7 +6,6 @@ import ( "crypto/tls" "crypto/x509" "encoding/base64" - "errors" "fmt" "io/ioutil" "net/http" @@ -16,22 +15,15 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/login" "github.com/grafana/grafana/pkg/metrics" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/social" ) -var ( - ErrProviderDeniedRequest = errors.New("Login provider denied login request") - ErrEmailNotAllowed = errors.New("Required email domain not fulfilled") - ErrSignUpNotAllowed = errors.New("Signup is not allowed for this adapter") - ErrUsersQuotaReached = errors.New("Users quota reached") - ErrNoEmail = errors.New("Login provider didn't return an email address") - oauthLogger = log.New("oauth") -) +var oauthLogger = log.New("oauth") func GenStateString() string { rnd := make([]byte, 32) @@ -56,7 +48,7 @@ func OAuthLogin(ctx *m.ReqContext) { if errorParam != "" { errorDesc := ctx.Query("error_description") oauthLogger.Error("failed to login ", "error", errorParam, "errorDesc", errorDesc) - redirectWithError(ctx, ErrProviderDeniedRequest, "error", errorParam, "errorDesc", errorDesc) + redirectWithError(ctx, login.ErrProviderDeniedRequest, "error", errorParam, "errorDesc", errorDesc) return } @@ -86,6 +78,7 @@ func OAuthLogin(ctx *m.ReqContext) { // handle call back tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{ InsecureSkipVerify: setting.OAuthService.OAuthInfos[name].TlsSkipVerify, }, @@ -149,54 +142,43 @@ func OAuthLogin(ctx *m.ReqContext) { // validate that we got at least an email address if userInfo.Email == "" { - redirectWithError(ctx, ErrNoEmail) + redirectWithError(ctx, login.ErrNoEmail) return } // validate that the email is allowed to login to grafana if !connect.IsEmailAllowed(userInfo.Email) { - redirectWithError(ctx, ErrEmailNotAllowed) + redirectWithError(ctx, login.ErrEmailNotAllowed) return } - userQuery := m.GetUserByEmailQuery{Email: userInfo.Email} - err = bus.Dispatch(&userQuery) + extUser := &m.ExternalUserInfo{ + AuthModule: "oauth_" + name, + AuthId: userInfo.Id, + Name: userInfo.Name, + Login: userInfo.Login, + Email: userInfo.Email, + OrgRoles: map[int64]m.RoleType{}, + } - // create account if missing - if err == m.ErrUserNotFound { - if !connect.IsSignupAllowed() { - redirectWithError(ctx, ErrSignUpNotAllowed) - return - } - limitReached, err := quota.QuotaReached(ctx, "user") - if err != nil { - ctx.Handle(500, "Failed to get user quota", err) - return - } - if limitReached { - redirectWithError(ctx, ErrUsersQuotaReached) - return - } - cmd := m.CreateUserCommand{ - Login: userInfo.Login, - Email: userInfo.Email, - Name: userInfo.Name, - Company: userInfo.Company, - DefaultOrgRole: userInfo.Role, - } + if userInfo.Role != "" { + extUser.OrgRoles[1] = m.RoleType(userInfo.Role) + } - if err = bus.Dispatch(&cmd); err != nil { - ctx.Handle(500, "Failed to create account", err) - return - } - - userQuery.Result = &cmd.Result - } else if err != nil { - ctx.Handle(500, "Unexpected error", err) + // add/update user in grafana + cmd := &m.UpsertUserCommand{ + ReqContext: ctx, + ExternalUser: extUser, + SignupAllowed: connect.IsSignupAllowed(), + } + err = bus.Dispatch(cmd) + if err != nil { + redirectWithError(ctx, err) + return } // login - loginUserWithUser(userQuery.Result, ctx) + loginUserWithUser(cmd.Result, ctx) metrics.M_Api_Login_OAuth.Inc() diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index 5d395d655a9..6e5ae0f8761 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -13,21 +13,24 @@ import ( ) // POST /api/tsdb/query -func QueryMetrics(c *m.ReqContext, reqDto dtos.MetricRequest) Response { +func (hs *HTTPServer) QueryMetrics(c *m.ReqContext, reqDto dtos.MetricRequest) Response { timeRange := tsdb.NewTimeRange(reqDto.From, reqDto.To) if len(reqDto.Queries) == 0 { - return ApiError(400, "No queries found in query", nil) + return Error(400, "No queries found in query", nil) } - dsId, err := reqDto.Queries[0].Get("datasourceId").Int64() + datasourceId, err := reqDto.Queries[0].Get("datasourceId").Int64() if err != nil { - return ApiError(400, "Query missing datasourceId", nil) + return Error(400, "Query missing datasourceId", nil) } - dsQuery := m.GetDataSourceByIdQuery{Id: dsId, OrgId: c.OrgId} - if err := bus.Dispatch(&dsQuery); err != nil { - return ApiError(500, "failed to fetch data source", err) + ds, err := hs.DatasourceCache.GetDatasource(datasourceId, c.SignedInUser, c.SkipCache) + if err != nil { + if err == m.ErrDataSourceAccessDenied { + return Error(403, "Access denied to datasource", err) + } + return Error(500, "Unable to load datasource meta data", err) } request := &tsdb.TsdbQuery{TimeRange: timeRange} @@ -38,13 +41,13 @@ func QueryMetrics(c *m.ReqContext, reqDto dtos.MetricRequest) Response { MaxDataPoints: query.Get("maxDataPoints").MustInt64(100), IntervalMs: query.Get("intervalMs").MustInt64(1000), Model: query, - DataSource: dsQuery.Result, + DataSource: ds, }) } - resp, err := tsdb.HandleRequest(context.Background(), dsQuery.Result, request) + resp, err := tsdb.HandleRequest(c.Req.Context(), ds, request) if err != nil { - return ApiError(500, "Metric request error", err) + return Error(500, "Metric request error", err) } statusCode := 200 @@ -52,11 +55,11 @@ func QueryMetrics(c *m.ReqContext, reqDto dtos.MetricRequest) Response { if res.Error != nil { res.ErrorString = res.Error.Error() resp.Message = res.ErrorString - statusCode = 500 + statusCode = 400 } } - return Json(statusCode, &resp) + return JSON(statusCode, &resp) } // GET /api/tsdb/testdata/scenarios @@ -72,22 +75,22 @@ func GetTestDataScenarios(c *m.ReqContext) Response { }) } - return Json(200, &result) + return JSON(200, &result) } -// Genereates a index out of range error +// Generates a index out of range error func GenerateError(c *m.ReqContext) Response { var array []string - return Json(200, array[20]) + return JSON(200, array[20]) } // GET /api/tsdb/testdata/gensql -func GenerateSqlTestData(c *m.ReqContext) Response { +func GenerateSQLTestData(c *m.ReqContext) Response { if err := bus.Dispatch(&m.InsertSqlTestDataCommand{}); err != nil { - return ApiError(500, "Failed to insert test data", err) + return Error(500, "Failed to insert test data", err) } - return Json(200, &util.DynMap{"message": "OK"}) + return JSON(200, &util.DynMap{"message": "OK"}) } // GET /api/tsdb/testdata/random-walk @@ -99,7 +102,7 @@ func GetTestDataRandomWalk(c *m.ReqContext) Response { timeRange := tsdb.NewTimeRange(from, to) request := &tsdb.TsdbQuery{TimeRange: timeRange} - dsInfo := &m.DataSource{Type: "grafana-testdata-datasource"} + dsInfo := &m.DataSource{Type: "testdata"} request.Queries = append(request.Queries, &tsdb.Query{ RefId: "A", IntervalMs: intervalMs, @@ -111,8 +114,8 @@ func GetTestDataRandomWalk(c *m.ReqContext) Response { resp, err := tsdb.HandleRequest(context.Background(), dsInfo, request) if err != nil { - return ApiError(500, "Metric request error", err) + return Error(500, "Metric request error", err) } - return Json(200, &resp) + return JSON(200, &resp) } diff --git a/pkg/api/org.go b/pkg/api/org.go index 5f20559dbbe..8d0d48bad13 100644 --- a/pkg/api/org.go +++ b/pkg/api/org.go @@ -15,7 +15,7 @@ func GetOrgCurrent(c *m.ReqContext) Response { } // GET /api/orgs/:orgId -func GetOrgById(c *m.ReqContext) Response { +func GetOrgByID(c *m.ReqContext) Response { return getOrgHelper(c.ParamsInt64(":orgId")) } @@ -24,10 +24,10 @@ func GetOrgByName(c *m.ReqContext) Response { query := m.GetOrgByNameQuery{Name: c.Params(":name")} if err := bus.Dispatch(&query); err != nil { if err == m.ErrOrgNotFound { - return ApiError(404, "Organization not found", err) + return Error(404, "Organization not found", err) } - return ApiError(500, "Failed to get organization", err) + return Error(500, "Failed to get organization", err) } org := query.Result result := m.OrgDetailsDTO{ @@ -43,18 +43,18 @@ func GetOrgByName(c *m.ReqContext) Response { }, } - return Json(200, &result) + return JSON(200, &result) } -func getOrgHelper(orgId int64) Response { - query := m.GetOrgByIdQuery{Id: orgId} +func getOrgHelper(orgID int64) Response { + query := m.GetOrgByIdQuery{Id: orgID} if err := bus.Dispatch(&query); err != nil { if err == m.ErrOrgNotFound { - return ApiError(404, "Organization not found", err) + return Error(404, "Organization not found", err) } - return ApiError(500, "Failed to get organization", err) + return Error(500, "Failed to get organization", err) } org := query.Result @@ -71,26 +71,26 @@ func getOrgHelper(orgId int64) Response { }, } - return Json(200, &result) + return JSON(200, &result) } // POST /api/orgs func CreateOrg(c *m.ReqContext, cmd m.CreateOrgCommand) Response { if !c.IsSignedIn || (!setting.AllowUserOrgCreate && !c.IsGrafanaAdmin) { - return ApiError(403, "Access denied", nil) + return Error(403, "Access denied", nil) } cmd.UserId = c.UserId if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrOrgNameTaken { - return ApiError(409, "Organization name taken", err) + return Error(409, "Organization name taken", err) } - return ApiError(500, "Failed to create organization", err) + return Error(500, "Failed to create organization", err) } metrics.M_Api_Org_Create.Inc() - return Json(200, &util.DynMap{ + return JSON(200, &util.DynMap{ "orgId": cmd.Result.Id, "message": "Organization created", }) @@ -106,16 +106,16 @@ func UpdateOrg(c *m.ReqContext, form dtos.UpdateOrgForm) Response { return updateOrgHelper(form, c.ParamsInt64(":orgId")) } -func updateOrgHelper(form dtos.UpdateOrgForm, orgId int64) Response { - cmd := m.UpdateOrgCommand{Name: form.Name, OrgId: orgId} +func updateOrgHelper(form dtos.UpdateOrgForm, orgID int64) Response { + cmd := m.UpdateOrgCommand{Name: form.Name, OrgId: orgID} if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrOrgNameTaken { - return ApiError(400, "Organization name taken", err) + return Error(400, "Organization name taken", err) } - return ApiError(500, "Failed to update organization", err) + return Error(500, "Failed to update organization", err) } - return ApiSuccess("Organization updated") + return Success("Organization updated") } // PUT /api/org/address @@ -128,9 +128,9 @@ func UpdateOrgAddress(c *m.ReqContext, form dtos.UpdateOrgAddressForm) Response return updateOrgAddressHelper(form, c.ParamsInt64(":orgId")) } -func updateOrgAddressHelper(form dtos.UpdateOrgAddressForm, orgId int64) Response { +func updateOrgAddressHelper(form dtos.UpdateOrgAddressForm, orgID int64) Response { cmd := m.UpdateOrgAddressCommand{ - OrgId: orgId, + OrgId: orgID, Address: m.Address{ Address1: form.Address1, Address2: form.Address2, @@ -142,21 +142,21 @@ func updateOrgAddressHelper(form dtos.UpdateOrgAddressForm, orgId int64) Respons } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update org address", err) + return Error(500, "Failed to update org address", err) } - return ApiSuccess("Address updated") + return Success("Address updated") } // GET /api/orgs/:orgId -func DeleteOrgById(c *m.ReqContext) Response { +func DeleteOrgByID(c *m.ReqContext) Response { if err := bus.Dispatch(&m.DeleteOrgCommand{Id: c.ParamsInt64(":orgId")}); err != nil { if err == m.ErrOrgNotFound { - return ApiError(404, "Failed to delete organization. ID not found", nil) + return Error(404, "Failed to delete organization. ID not found", nil) } - return ApiError(500, "Failed to update organization", err) + return Error(500, "Failed to update organization", err) } - return ApiSuccess("Organization deleted") + return Success("Organization deleted") } func SearchOrgs(c *m.ReqContext) Response { @@ -168,8 +168,8 @@ func SearchOrgs(c *m.ReqContext) Response { } if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to search orgs", err) + return Error(500, "Failed to search orgs", err) } - return Json(200, query.Result) + return JSON(200, query.Result) } diff --git a/pkg/api/org_invite.go b/pkg/api/org_invite.go index 6a727dd95cc..dfb2cf045ed 100644 --- a/pkg/api/org_invite.go +++ b/pkg/api/org_invite.go @@ -16,30 +16,30 @@ func GetPendingOrgInvites(c *m.ReqContext) Response { query := m.GetTempUsersQuery{OrgId: c.OrgId, Status: m.TmpUserInvitePending} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to get invites from db", err) + return Error(500, "Failed to get invites from db", err) } for _, invite := range query.Result { invite.Url = setting.ToAbsUrl("invite/" + invite.Code) } - return Json(200, query.Result) + return JSON(200, query.Result) } func AddOrgInvite(c *m.ReqContext, inviteDto dtos.AddInviteForm) Response { if !inviteDto.Role.IsValid() { - return ApiError(400, "Invalid role specified", nil) + return Error(400, "Invalid role specified", nil) } // first try get existing user userQuery := m.GetUserByLoginQuery{LoginOrEmail: inviteDto.LoginOrEmail} if err := bus.Dispatch(&userQuery); err != nil { if err != m.ErrUserNotFound { - return ApiError(500, "Failed to query db for existing user check", err) + return Error(500, "Failed to query db for existing user check", err) } if setting.DisableLoginForm { - return ApiError(401, "User could not be found", nil) + return Error(401, "User could not be found", nil) } } else { return inviteExistingUserToOrg(c, userQuery.Result, &inviteDto) @@ -56,7 +56,7 @@ func AddOrgInvite(c *m.ReqContext, inviteDto dtos.AddInviteForm) Response { cmd.RemoteAddr = c.Req.RemoteAddr if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to save invite to database", err) + return Error(500, "Failed to save invite to database", err) } // send invite email @@ -74,18 +74,21 @@ func AddOrgInvite(c *m.ReqContext, inviteDto dtos.AddInviteForm) Response { } if err := bus.Dispatch(&emailCmd); err != nil { - return ApiError(500, "Failed to send email invite", err) + if err == m.ErrSmtpNotEnabled { + return Error(412, err.Error(), err) + } + return Error(500, "Failed to send email invite", err) } emailSentCmd := m.UpdateTempUserWithEmailSentCommand{Code: cmd.Result.Code} if err := bus.Dispatch(&emailSentCmd); err != nil { - return ApiError(500, "Failed to update invite with email sent info", err) + return Error(500, "Failed to update invite with email sent info", err) } - return ApiSuccess(fmt.Sprintf("Sent invite to %s", inviteDto.LoginOrEmail)) + return Success(fmt.Sprintf("Sent invite to %s", inviteDto.LoginOrEmail)) } - return ApiSuccess(fmt.Sprintf("Created invite for %s", inviteDto.LoginOrEmail)) + return Success(fmt.Sprintf("Created invite for %s", inviteDto.LoginOrEmail)) } func inviteExistingUserToOrg(c *m.ReqContext, user *m.User, inviteDto *dtos.AddInviteForm) Response { @@ -93,29 +96,28 @@ func inviteExistingUserToOrg(c *m.ReqContext, user *m.User, inviteDto *dtos.AddI createOrgUserCmd := m.AddOrgUserCommand{OrgId: c.OrgId, UserId: user.Id, Role: inviteDto.Role} if err := bus.Dispatch(&createOrgUserCmd); err != nil { if err == m.ErrOrgUserAlreadyAdded { - return ApiError(412, fmt.Sprintf("User %s is already added to organization", inviteDto.LoginOrEmail), err) + return Error(412, fmt.Sprintf("User %s is already added to organization", inviteDto.LoginOrEmail), err) } - return ApiError(500, "Error while trying to create org user", err) - } else { - - if inviteDto.SendEmail && util.IsEmail(user.Email) { - emailCmd := m.SendEmailCommand{ - To: []string{user.Email}, - Template: "invited_to_org.html", - Data: map[string]interface{}{ - "Name": user.NameOrFallback(), - "OrgName": c.OrgName, - "InvitedBy": util.StringsFallback3(c.Name, c.Email, c.Login), - }, - } - - if err := bus.Dispatch(&emailCmd); err != nil { - return ApiError(500, "Failed to send email invited_to_org", err) - } - } - - return ApiSuccess(fmt.Sprintf("Existing Grafana user %s added to org %s", user.NameOrFallback(), c.OrgName)) + return Error(500, "Error while trying to create org user", err) } + + if inviteDto.SendEmail && util.IsEmail(user.Email) { + emailCmd := m.SendEmailCommand{ + To: []string{user.Email}, + Template: "invited_to_org.html", + Data: map[string]interface{}{ + "Name": user.NameOrFallback(), + "OrgName": c.OrgName, + "InvitedBy": util.StringsFallback3(c.Name, c.Email, c.Login), + }, + } + + if err := bus.Dispatch(&emailCmd); err != nil { + return Error(500, "Failed to send email invited_to_org", err) + } + } + + return Success(fmt.Sprintf("Existing Grafana user %s added to org %s", user.NameOrFallback(), c.OrgName)) } func RevokeInvite(c *m.ReqContext) Response { @@ -123,7 +125,7 @@ func RevokeInvite(c *m.ReqContext) Response { return rsp } - return ApiSuccess("Invite revoked") + return Success("Invite revoked") } func GetInviteInfoByCode(c *m.ReqContext) Response { @@ -131,14 +133,14 @@ func GetInviteInfoByCode(c *m.ReqContext) Response { if err := bus.Dispatch(&query); err != nil { if err == m.ErrTempUserNotFound { - return ApiError(404, "Invite not found", nil) + return Error(404, "Invite not found", nil) } - return ApiError(500, "Failed to get invite", err) + return Error(500, "Failed to get invite", err) } invite := query.Result - return Json(200, dtos.InviteInfo{ + return JSON(200, dtos.InviteInfo{ Email: invite.Email, Name: invite.Name, Username: invite.Email, @@ -151,14 +153,14 @@ func CompleteInvite(c *m.ReqContext, completeInvite dtos.CompleteInviteForm) Res if err := bus.Dispatch(&query); err != nil { if err == m.ErrTempUserNotFound { - return ApiError(404, "Invite not found", nil) + return Error(404, "Invite not found", nil) } - return ApiError(500, "Failed to get invite", err) + return Error(500, "Failed to get invite", err) } invite := query.Result if invite.Status != m.TmpUserInvitePending { - return ApiError(412, fmt.Sprintf("Invite cannot be used in status %s", invite.Status), nil) + return Error(412, fmt.Sprintf("Invite cannot be used in status %s", invite.Status), nil) } cmd := m.CreateUserCommand{ @@ -170,7 +172,7 @@ func CompleteInvite(c *m.ReqContext, completeInvite dtos.CompleteInviteForm) Res } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "failed to create user", err) + return Error(500, "failed to create user", err) } user := &cmd.Result @@ -189,14 +191,14 @@ func CompleteInvite(c *m.ReqContext, completeInvite dtos.CompleteInviteForm) Res metrics.M_Api_User_SignUpCompleted.Inc() metrics.M_Api_User_SignUpInvite.Inc() - return ApiSuccess("User created and logged in") + return Success("User created and logged in") } func updateTempUserStatus(code string, status m.TempUserStatus) (bool, Response) { // update temp user status updateTmpUserCmd := m.UpdateTempUserStatusCommand{Code: code, Status: status} if err := bus.Dispatch(&updateTmpUserCmd); err != nil { - return false, ApiError(500, "Failed to update invite status", err) + return false, Error(500, "Failed to update invite status", err) } return true, nil @@ -207,7 +209,7 @@ func applyUserInvite(user *m.User, invite *m.TempUserDTO, setActive bool) (bool, addOrgUserCmd := m.AddOrgUserCommand{OrgId: invite.OrgId, UserId: user.Id, Role: invite.Role} if err := bus.Dispatch(&addOrgUserCmd); err != nil { if err != m.ErrOrgUserAlreadyAdded { - return false, ApiError(500, "Error while trying to create org user", err) + return false, Error(500, "Error while trying to create org user", err) } } @@ -219,7 +221,7 @@ func applyUserInvite(user *m.User, invite *m.TempUserDTO, setActive bool) (bool, if setActive { // set org to active if err := bus.Dispatch(&m.SetUsingOrgCommand{OrgId: invite.OrgId, UserId: user.Id}); err != nil { - return false, ApiError(500, "Failed to set org as active", err) + return false, Error(500, "Failed to set org as active", err) } } diff --git a/pkg/api/org_users.go b/pkg/api/org_users.go index 6d7c2bb94bd..e750662c764 100644 --- a/pkg/api/org_users.go +++ b/pkg/api/org_users.go @@ -20,13 +20,13 @@ func AddOrgUser(c *m.ReqContext, cmd m.AddOrgUserCommand) Response { func addOrgUserHelper(cmd m.AddOrgUserCommand) Response { if !cmd.Role.IsValid() { - return ApiError(400, "Invalid role specified", nil) + return Error(400, "Invalid role specified", nil) } userQuery := m.GetUserByLoginQuery{LoginOrEmail: cmd.LoginOrEmail} err := bus.Dispatch(&userQuery) if err != nil { - return ApiError(404, "User not found", nil) + return Error(404, "User not found", nil) } userToAdd := userQuery.Result @@ -35,17 +35,17 @@ func addOrgUserHelper(cmd m.AddOrgUserCommand) Response { if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrOrgUserAlreadyAdded { - return ApiError(409, "User is already member of this organization", nil) + return Error(409, "User is already member of this organization", nil) } - return ApiError(500, "Could not add user to organization", err) + return Error(500, "Could not add user to organization", err) } - return ApiSuccess("User added to organization") + return Success("User added to organization") } // GET /api/org/users func GetOrgUsersForCurrentOrg(c *m.ReqContext) Response { - return getOrgUsersHelper(c.OrgId, c.Params("query"), c.ParamsInt("limit")) + return getOrgUsersHelper(c.OrgId, c.Query("query"), c.QueryInt("limit")) } // GET /api/orgs/:orgId/users @@ -53,22 +53,22 @@ func GetOrgUsers(c *m.ReqContext) Response { return getOrgUsersHelper(c.ParamsInt64(":orgId"), "", 0) } -func getOrgUsersHelper(orgId int64, query string, limit int) Response { +func getOrgUsersHelper(orgID int64, query string, limit int) Response { q := m.GetOrgUsersQuery{ - OrgId: orgId, + OrgId: orgID, Query: query, Limit: limit, } if err := bus.Dispatch(&q); err != nil { - return ApiError(500, "Failed to get account user", err) + return Error(500, "Failed to get account user", err) } for _, user := range q.Result { user.AvatarUrl = dtos.GetGravatarUrl(user.Email) } - return Json(200, q.Result) + return JSON(200, q.Result) } // PATCH /api/org/users/:userId @@ -87,41 +87,47 @@ func UpdateOrgUser(c *m.ReqContext, cmd m.UpdateOrgUserCommand) Response { func updateOrgUserHelper(cmd m.UpdateOrgUserCommand) Response { if !cmd.Role.IsValid() { - return ApiError(400, "Invalid role specified", nil) + return Error(400, "Invalid role specified", nil) } if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrLastOrgAdmin { - return ApiError(400, "Cannot change role so that there is no organization admin left", nil) + return Error(400, "Cannot change role so that there is no organization admin left", nil) } - return ApiError(500, "Failed update org user", err) + return Error(500, "Failed update org user", err) } - return ApiSuccess("Organization user updated") + return Success("Organization user updated") } // DELETE /api/org/users/:userId func RemoveOrgUserForCurrentOrg(c *m.ReqContext) Response { - userId := c.ParamsInt64(":userId") - return removeOrgUserHelper(c.OrgId, userId) + return removeOrgUserHelper(&m.RemoveOrgUserCommand{ + UserId: c.ParamsInt64(":userId"), + OrgId: c.OrgId, + ShouldDeleteOrphanedUser: true, + }) } // DELETE /api/orgs/:orgId/users/:userId func RemoveOrgUser(c *m.ReqContext) Response { - userId := c.ParamsInt64(":userId") - orgId := c.ParamsInt64(":orgId") - return removeOrgUserHelper(orgId, userId) + return removeOrgUserHelper(&m.RemoveOrgUserCommand{ + UserId: c.ParamsInt64(":userId"), + OrgId: c.ParamsInt64(":orgId"), + }) } -func removeOrgUserHelper(orgId int64, userId int64) Response { - cmd := m.RemoveOrgUserCommand{OrgId: orgId, UserId: userId} - - if err := bus.Dispatch(&cmd); err != nil { +func removeOrgUserHelper(cmd *m.RemoveOrgUserCommand) Response { + if err := bus.Dispatch(cmd); err != nil { if err == m.ErrLastOrgAdmin { - return ApiError(400, "Cannot remove last organization admin", nil) + return Error(400, "Cannot remove last organization admin", nil) } - return ApiError(500, "Failed to remove user from organization", err) + return Error(500, "Failed to remove user from organization", err) } - return ApiSuccess("User removed from organization") + if cmd.UserWasDeleted { + return Success("User deleted") + } + + return Success("User removed from organization") } diff --git a/pkg/api/password.go b/pkg/api/password.go index 31ea5d91b34..7dd901c898e 100644 --- a/pkg/api/password.go +++ b/pkg/api/password.go @@ -12,15 +12,15 @@ func SendResetPasswordEmail(c *m.ReqContext, form dtos.SendResetPasswordEmailFor if err := bus.Dispatch(&userQuery); err != nil { c.Logger.Info("Requested password reset for user that was not found", "user", userQuery.LoginOrEmail) - return ApiError(200, "Email sent", err) + return Error(200, "Email sent", err) } emailCmd := m.SendResetPasswordEmailCommand{User: userQuery.Result} if err := bus.Dispatch(&emailCmd); err != nil { - return ApiError(500, "Failed to send email", err) + return Error(500, "Failed to send email", err) } - return ApiSuccess("Email sent") + return Success("Email sent") } func ResetPassword(c *m.ReqContext, form dtos.ResetUserPasswordForm) Response { @@ -28,13 +28,13 @@ func ResetPassword(c *m.ReqContext, form dtos.ResetUserPasswordForm) Response { if err := bus.Dispatch(&query); err != nil { if err == m.ErrInvalidEmailCode { - return ApiError(400, "Invalid or expired reset password code", nil) + return Error(400, "Invalid or expired reset password code", nil) } - return ApiError(500, "Unknown error validating email code", err) + return Error(500, "Unknown error validating email code", err) } if form.NewPassword != form.ConfirmPassword { - return ApiError(400, "Passwords do not match", nil) + return Error(400, "Passwords do not match", nil) } cmd := m.ChangeUserPasswordCommand{} @@ -42,8 +42,8 @@ func ResetPassword(c *m.ReqContext, form dtos.ResetUserPasswordForm) Response { cmd.NewPassword = util.EncodePassword(form.NewPassword, query.Result.Salt) if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to change user password", err) + return Error(500, "Failed to change user password", err) } - return ApiSuccess("User password changed") + return Success("User password changed") } diff --git a/pkg/api/playlist.go b/pkg/api/playlist.go index 45de40ce337..0963df7d4c4 100644 --- a/pkg/api/playlist.go +++ b/pkg/api/playlist.go @@ -33,7 +33,7 @@ func ValidateOrgPlaylist(c *m.ReqContext) { return } - if len(items) == 0 { + if len(items) == 0 && c.Context.Req.Method != "DELETE" { c.JsonApiErr(404, "Playlist is empty", itemsErr) return } @@ -55,10 +55,10 @@ func SearchPlaylists(c *m.ReqContext) Response { err := bus.Dispatch(&searchQuery) if err != nil { - return ApiError(500, "Search failed", err) + return Error(500, "Search failed", err) } - return Json(200, searchQuery.Result) + return JSON(200, searchQuery.Result) } func GetPlaylist(c *m.ReqContext) Response { @@ -66,7 +66,7 @@ func GetPlaylist(c *m.ReqContext) Response { cmd := m.GetPlaylistByIdQuery{Id: id} if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Playlist not found", err) + return Error(500, "Playlist not found", err) } playlistDTOs, _ := LoadPlaylistItemDTOs(id) @@ -79,7 +79,7 @@ func GetPlaylist(c *m.ReqContext) Response { Items: playlistDTOs, } - return Json(200, dto) + return JSON(200, dto) } func LoadPlaylistItemDTOs(id int64) ([]m.PlaylistItemDTO, error) { @@ -120,21 +120,21 @@ func GetPlaylistItems(c *m.ReqContext) Response { playlistDTOs, err := LoadPlaylistItemDTOs(id) if err != nil { - return ApiError(500, "Could not load playlist items", err) + return Error(500, "Could not load playlist items", err) } - return Json(200, playlistDTOs) + return JSON(200, playlistDTOs) } func GetPlaylistDashboards(c *m.ReqContext) Response { - playlistId := c.ParamsInt64(":id") + playlistID := c.ParamsInt64(":id") - playlists, err := LoadPlaylistDashboards(c.OrgId, c.SignedInUser, playlistId) + playlists, err := LoadPlaylistDashboards(c.OrgId, c.SignedInUser, playlistID) if err != nil { - return ApiError(500, "Could not load dashboards", err) + return Error(500, "Could not load dashboards", err) } - return Json(200, playlists) + return JSON(200, playlists) } func DeletePlaylist(c *m.ReqContext) Response { @@ -142,34 +142,35 @@ func DeletePlaylist(c *m.ReqContext) Response { cmd := m.DeletePlaylistCommand{Id: id, OrgId: c.OrgId} if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to delete playlist", err) + return Error(500, "Failed to delete playlist", err) } - return Json(200, "") + return JSON(200, "") } func CreatePlaylist(c *m.ReqContext, cmd m.CreatePlaylistCommand) Response { cmd.OrgId = c.OrgId if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to create playlist", err) + return Error(500, "Failed to create playlist", err) } - return Json(200, cmd.Result) + return JSON(200, cmd.Result) } func UpdatePlaylist(c *m.ReqContext, cmd m.UpdatePlaylistCommand) Response { cmd.OrgId = c.OrgId + cmd.Id = c.ParamsInt64(":id") if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to save playlist", err) + return Error(500, "Failed to save playlist", err) } playlistDTOs, err := LoadPlaylistItemDTOs(cmd.Id) if err != nil { - return ApiError(500, "Failed to save playlist", err) + return Error(500, "Failed to save playlist", err) } cmd.Result.Items = playlistDTOs - return Json(200, cmd.Result) + return JSON(200, cmd.Result) } diff --git a/pkg/api/playlist_play.go b/pkg/api/playlist_play.go index 1d059e06be5..e82c7b438b4 100644 --- a/pkg/api/playlist_play.go +++ b/pkg/api/playlist_play.go @@ -11,11 +11,11 @@ import ( "github.com/grafana/grafana/pkg/services/search" ) -func populateDashboardsById(dashboardByIds []int64, dashboardIdOrder map[int64]int) (dtos.PlaylistDashboardsSlice, error) { +func populateDashboardsByID(dashboardByIDs []int64, dashboardIDOrder map[int64]int) (dtos.PlaylistDashboardsSlice, error) { result := make(dtos.PlaylistDashboardsSlice, 0) - if len(dashboardByIds) > 0 { - dashboardQuery := m.GetDashboardsQuery{DashboardIds: dashboardByIds} + if len(dashboardByIDs) > 0 { + dashboardQuery := m.GetDashboardsQuery{DashboardIds: dashboardByIDs} if err := bus.Dispatch(&dashboardQuery); err != nil { return result, err } @@ -26,7 +26,7 @@ func populateDashboardsById(dashboardByIds []int64, dashboardIdOrder map[int64]i Slug: item.Slug, Title: item.Title, Uri: "db/" + item.Slug, - Order: dashboardIdOrder[item.Id], + Order: dashboardIDOrder[item.Id], }) } } @@ -34,29 +34,27 @@ func populateDashboardsById(dashboardByIds []int64, dashboardIdOrder map[int64]i return result, nil } -func populateDashboardsByTag(orgId int64, signedInUser *m.SignedInUser, dashboardByTag []string, dashboardTagOrder map[string]int) dtos.PlaylistDashboardsSlice { +func populateDashboardsByTag(orgID int64, signedInUser *m.SignedInUser, dashboardByTag []string, dashboardTagOrder map[string]int) dtos.PlaylistDashboardsSlice { result := make(dtos.PlaylistDashboardsSlice, 0) - if len(dashboardByTag) > 0 { - for _, tag := range dashboardByTag { - searchQuery := search.Query{ - Title: "", - Tags: []string{tag}, - SignedInUser: signedInUser, - Limit: 100, - IsStarred: false, - OrgId: orgId, - } + for _, tag := range dashboardByTag { + searchQuery := search.Query{ + Title: "", + Tags: []string{tag}, + SignedInUser: signedInUser, + Limit: 100, + IsStarred: false, + OrgId: orgID, + } - if err := bus.Dispatch(&searchQuery); err == nil { - for _, item := range searchQuery.Result { - result = append(result, dtos.PlaylistDashboard{ - Id: item.Id, - Title: item.Title, - Uri: item.Uri, - Order: dashboardTagOrder[tag], - }) - } + if err := bus.Dispatch(&searchQuery); err == nil { + for _, item := range searchQuery.Result { + result = append(result, dtos.PlaylistDashboard{ + Id: item.Id, + Title: item.Title, + Uri: item.Uri, + Order: dashboardTagOrder[tag], + }) } } } @@ -64,19 +62,19 @@ func populateDashboardsByTag(orgId int64, signedInUser *m.SignedInUser, dashboar return result } -func LoadPlaylistDashboards(orgId int64, signedInUser *m.SignedInUser, playlistId int64) (dtos.PlaylistDashboardsSlice, error) { - playlistItems, _ := LoadPlaylistItems(playlistId) +func LoadPlaylistDashboards(orgID int64, signedInUser *m.SignedInUser, playlistID int64) (dtos.PlaylistDashboardsSlice, error) { + playlistItems, _ := LoadPlaylistItems(playlistID) - dashboardByIds := make([]int64, 0) + dashboardByIDs := make([]int64, 0) dashboardByTag := make([]string, 0) - dashboardIdOrder := make(map[int64]int) + dashboardIDOrder := make(map[int64]int) dashboardTagOrder := make(map[string]int) for _, i := range playlistItems { if i.Type == "dashboard_by_id" { - dashboardId, _ := strconv.ParseInt(i.Value, 10, 64) - dashboardByIds = append(dashboardByIds, dashboardId) - dashboardIdOrder[dashboardId] = i.Order + dashboardID, _ := strconv.ParseInt(i.Value, 10, 64) + dashboardByIDs = append(dashboardByIDs, dashboardID) + dashboardIDOrder[dashboardID] = i.Order } if i.Type == "dashboard_by_tag" { @@ -87,9 +85,9 @@ func LoadPlaylistDashboards(orgId int64, signedInUser *m.SignedInUser, playlistI result := make(dtos.PlaylistDashboardsSlice, 0) - var k, _ = populateDashboardsById(dashboardByIds, dashboardIdOrder) + var k, _ = populateDashboardsByID(dashboardByIDs, dashboardIDOrder) result = append(result, k...) - result = append(result, populateDashboardsByTag(orgId, signedInUser, dashboardByTag, dashboardTagOrder)...) + result = append(result, populateDashboardsByTag(orgID, signedInUser, dashboardByTag, dashboardTagOrder)...) sort.Sort(result) return result, nil diff --git a/pkg/api/pluginproxy/access_token_provider.go b/pkg/api/pluginproxy/access_token_provider.go new file mode 100644 index 00000000000..22407823ff9 --- /dev/null +++ b/pkg/api/pluginproxy/access_token_provider.go @@ -0,0 +1,171 @@ +package pluginproxy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "sync" + "time" + + "golang.org/x/oauth2" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "golang.org/x/oauth2/jwt" +) + +var ( + tokenCache = tokenCacheType{ + cache: map[string]*jwtToken{}, + } + oauthJwtTokenCache = oauthJwtTokenCacheType{ + cache: map[string]*oauth2.Token{}, + } +) + +type tokenCacheType struct { + cache map[string]*jwtToken + sync.Mutex +} + +type oauthJwtTokenCacheType struct { + cache map[string]*oauth2.Token + sync.Mutex +} + +type accessTokenProvider struct { + route *plugins.AppPluginRoute + datasourceId int64 + datasourceVersion int +} + +type jwtToken struct { + ExpiresOn time.Time `json:"-"` + ExpiresOnString string `json:"expires_on"` + AccessToken string `json:"access_token"` +} + +func newAccessTokenProvider(ds *models.DataSource, pluginRoute *plugins.AppPluginRoute) *accessTokenProvider { + return &accessTokenProvider{ + datasourceId: ds.Id, + datasourceVersion: ds.Version, + route: pluginRoute, + } +} + +func (provider *accessTokenProvider) getAccessToken(data templateData) (string, error) { + tokenCache.Lock() + defer tokenCache.Unlock() + if cachedToken, found := tokenCache.cache[provider.getAccessTokenCacheKey()]; found { + if cachedToken.ExpiresOn.After(time.Now().Add(time.Second * 10)) { + logger.Info("Using token from cache") + return cachedToken.AccessToken, nil + } + } + + urlInterpolated, err := interpolateString(provider.route.TokenAuth.Url, data) + if err != nil { + return "", err + } + + params := make(url.Values) + for key, value := range provider.route.TokenAuth.Params { + interpolatedParam, err := interpolateString(value, data) + if err != nil { + return "", err + } + params.Add(key, interpolatedParam) + } + + getTokenReq, _ := http.NewRequest("POST", urlInterpolated, bytes.NewBufferString(params.Encode())) + getTokenReq.Header.Add("Content-Type", "application/x-www-form-urlencoded") + getTokenReq.Header.Add("Content-Length", strconv.Itoa(len(params.Encode()))) + + resp, err := client.Do(getTokenReq) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + var token jwtToken + if err := json.NewDecoder(resp.Body).Decode(&token); err != nil { + return "", err + } + + expiresOnEpoch, _ := strconv.ParseInt(token.ExpiresOnString, 10, 64) + token.ExpiresOn = time.Unix(expiresOnEpoch, 0) + tokenCache.cache[provider.getAccessTokenCacheKey()] = &token + + logger.Info("Got new access token", "ExpiresOn", token.ExpiresOn) + + return token.AccessToken, nil +} + +func (provider *accessTokenProvider) getJwtAccessToken(ctx context.Context, data templateData) (string, error) { + oauthJwtTokenCache.Lock() + defer oauthJwtTokenCache.Unlock() + if cachedToken, found := oauthJwtTokenCache.cache[provider.getAccessTokenCacheKey()]; found { + if cachedToken.Expiry.After(time.Now().Add(time.Second * 10)) { + logger.Debug("Using token from cache") + return cachedToken.AccessToken, nil + } + } + + conf := &jwt.Config{} + + if val, ok := provider.route.JwtTokenAuth.Params["client_email"]; ok { + interpolatedVal, err := interpolateString(val, data) + if err != nil { + return "", err + } + conf.Email = interpolatedVal + } + + if val, ok := provider.route.JwtTokenAuth.Params["private_key"]; ok { + interpolatedVal, err := interpolateString(val, data) + if err != nil { + return "", err + } + conf.PrivateKey = []byte(interpolatedVal) + } + + if val, ok := provider.route.JwtTokenAuth.Params["token_uri"]; ok { + interpolatedVal, err := interpolateString(val, data) + if err != nil { + return "", err + } + conf.TokenURL = interpolatedVal + } + + conf.Scopes = provider.route.JwtTokenAuth.Scopes + + token, err := getTokenSource(conf, ctx) + if err != nil { + return "", err + } + + oauthJwtTokenCache.cache[provider.getAccessTokenCacheKey()] = token + + logger.Info("Got new access token", "ExpiresOn", token.Expiry) + + return token.AccessToken, nil +} + +var getTokenSource = func(conf *jwt.Config, ctx context.Context) (*oauth2.Token, error) { + tokenSrc := conf.TokenSource(ctx) + token, err := tokenSrc.Token() + if err != nil { + return nil, err + } + + return token, nil +} + +func (provider *accessTokenProvider) getAccessTokenCacheKey() string { + return fmt.Sprintf("%v_%v_%v_%v", provider.datasourceId, provider.datasourceVersion, provider.route.Path, provider.route.Method) +} diff --git a/pkg/api/pluginproxy/access_token_provider_test.go b/pkg/api/pluginproxy/access_token_provider_test.go new file mode 100644 index 00000000000..e75748e4660 --- /dev/null +++ b/pkg/api/pluginproxy/access_token_provider_test.go @@ -0,0 +1,94 @@ +package pluginproxy + +import ( + "context" + "testing" + "time" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + . "github.com/smartystreets/goconvey/convey" + "golang.org/x/oauth2" + "golang.org/x/oauth2/jwt" +) + +func TestAccessToken(t *testing.T) { + Convey("Plugin with JWT token auth route", t, func() { + pluginRoute := &plugins.AppPluginRoute{ + Path: "pathwithjwttoken1", + Url: "https://api.jwt.io/some/path", + Method: "GET", + JwtTokenAuth: &plugins.JwtTokenAuth{ + Url: "https://login.server.com/{{.JsonData.tenantId}}/oauth2/token", + Scopes: []string{ + "https://www.testapi.com/auth/monitoring.read", + "https://www.testapi.com/auth/cloudplatformprojects.readonly", + }, + Params: map[string]string{ + "token_uri": "{{.JsonData.tokenUri}}", + "client_email": "{{.JsonData.clientEmail}}", + "private_key": "{{.SecureJsonData.privateKey}}", + }, + }, + } + + templateData := templateData{ + JsonData: map[string]interface{}{ + "clientEmail": "test@test.com", + "tokenUri": "login.url.com/token", + }, + SecureJsonData: map[string]string{ + "privateKey": "testkey", + }, + } + + ds := &models.DataSource{Id: 1, Version: 2} + + Convey("should fetch token using jwt private key", func() { + getTokenSource = func(conf *jwt.Config, ctx context.Context) (*oauth2.Token, error) { + return &oauth2.Token{AccessToken: "abc"}, nil + } + provider := newAccessTokenProvider(ds, pluginRoute) + token, err := provider.getJwtAccessToken(context.Background(), templateData) + So(err, ShouldBeNil) + + So(token, ShouldEqual, "abc") + }) + + Convey("should set jwt config values", func() { + getTokenSource = func(conf *jwt.Config, ctx context.Context) (*oauth2.Token, error) { + So(conf.Email, ShouldEqual, "test@test.com") + So(conf.PrivateKey, ShouldResemble, []byte("testkey")) + So(len(conf.Scopes), ShouldEqual, 2) + So(conf.Scopes[0], ShouldEqual, "https://www.testapi.com/auth/monitoring.read") + So(conf.Scopes[1], ShouldEqual, "https://www.testapi.com/auth/cloudplatformprojects.readonly") + So(conf.TokenURL, ShouldEqual, "login.url.com/token") + + return &oauth2.Token{AccessToken: "abc"}, nil + } + + provider := newAccessTokenProvider(ds, pluginRoute) + _, err := provider.getJwtAccessToken(context.Background(), templateData) + So(err, ShouldBeNil) + }) + + Convey("should use cached token on second call", func() { + getTokenSource = func(conf *jwt.Config, ctx context.Context) (*oauth2.Token, error) { + return &oauth2.Token{ + AccessToken: "abc", + Expiry: time.Now().Add(1 * time.Minute)}, nil + } + provider := newAccessTokenProvider(ds, pluginRoute) + token1, err := provider.getJwtAccessToken(context.Background(), templateData) + So(err, ShouldBeNil) + So(token1, ShouldEqual, "abc") + + getTokenSource = func(conf *jwt.Config, ctx context.Context) (*oauth2.Token, error) { + return &oauth2.Token{AccessToken: "error: cache not used"}, nil + } + token2, err := provider.getJwtAccessToken(context.Background(), templateData) + So(err, ShouldBeNil) + So(token2, ShouldEqual, "abc") + }) + }) +} diff --git a/pkg/api/pluginproxy/ds_auth_provider.go b/pkg/api/pluginproxy/ds_auth_provider.go new file mode 100644 index 00000000000..edf43085c7c --- /dev/null +++ b/pkg/api/pluginproxy/ds_auth_provider.go @@ -0,0 +1,109 @@ +package pluginproxy + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + "strings" + "text/template" + + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/util" + "golang.org/x/oauth2/google" +) + +//ApplyRoute should use the plugin route data to set auth headers and custom headers +func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route *plugins.AppPluginRoute, ds *m.DataSource) { + proxyPath = strings.TrimPrefix(proxyPath, route.Path) + + data := templateData{ + JsonData: ds.JsonData.Interface().(map[string]interface{}), + SecureJsonData: ds.SecureJsonData.Decrypt(), + } + + interpolatedURL, err := interpolateString(route.Url, data) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } + + routeURL, err := url.Parse(interpolatedURL) + if err != nil { + logger.Error("Error parsing plugin route url", "error", err) + return + } + + req.URL.Scheme = routeURL.Scheme + req.URL.Host = routeURL.Host + req.Host = routeURL.Host + req.URL.Path = util.JoinUrlFragments(routeURL.Path, proxyPath) + + if err := addHeaders(&req.Header, route, data); err != nil { + logger.Error("Failed to render plugin headers", "error", err) + } + + tokenProvider := newAccessTokenProvider(ds, route) + + if route.TokenAuth != nil { + if token, err := tokenProvider.getAccessToken(data); err != nil { + logger.Error("Failed to get access token", "error", err) + } else { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + } + } + + authenticationType := ds.JsonData.Get("authenticationType").MustString("jwt") + if route.JwtTokenAuth != nil && authenticationType == "jwt" { + if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil { + logger.Error("Failed to get access token", "error", err) + } else { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + } + } + + if authenticationType == "gce" { + tokenSrc, err := google.DefaultTokenSource(ctx, route.JwtTokenAuth.Scopes...) + if err != nil { + logger.Error("Failed to get default token from meta data server", "error", err) + } else { + token, err := tokenSrc.Token() + if err != nil { + logger.Error("Failed to get default access token from meta data server", "error", err) + } else { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) + } + } + } + + logger.Info("Requesting", "url", req.URL.String()) +} + +func interpolateString(text string, data templateData) (string, error) { + t, err := template.New("content").Parse(text) + if err != nil { + return "", fmt.Errorf("could not parse template %s", text) + } + + var contentBuf bytes.Buffer + err = t.Execute(&contentBuf, data) + if err != nil { + return "", fmt.Errorf("failed to execute template %s", text) + } + + return contentBuf.String(), nil +} + +func addHeaders(reqHeaders *http.Header, route *plugins.AppPluginRoute, data templateData) error { + for _, header := range route.Headers { + interpolated, err := interpolateString(header.Content, data) + if err != nil { + return err + } + reqHeaders.Add(header.Name, interpolated) + } + + return nil +} diff --git a/pkg/api/pluginproxy/ds_auth_provider_test.go b/pkg/api/pluginproxy/ds_auth_provider_test.go new file mode 100644 index 00000000000..9bd98a339e5 --- /dev/null +++ b/pkg/api/pluginproxy/ds_auth_provider_test.go @@ -0,0 +1,21 @@ +package pluginproxy + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestDsAuthProvider(t *testing.T) { + Convey("When interpolating string", t, func() { + data := templateData{ + SecureJsonData: map[string]string{ + "Test": "0asd+asd", + }, + } + + interpolated, err := interpolateString("{{.SecureJsonData.Test}}", data) + So(err, ShouldBeNil) + So(interpolated, ShouldEqual, "0asd+asd") + }) +} diff --git a/pkg/api/pluginproxy/ds_proxy.go b/pkg/api/pluginproxy/ds_proxy.go index b861a344c75..38a2fd187e3 100644 --- a/pkg/api/pluginproxy/ds_proxy.go +++ b/pkg/api/pluginproxy/ds_proxy.go @@ -2,7 +2,6 @@ package pluginproxy import ( "bytes" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -12,7 +11,6 @@ import ( "net/url" "strconv" "strings" - "text/template" "time" "github.com/opentracing/opentracing-go" @@ -25,20 +23,10 @@ import ( ) var ( - logger log.Logger = log.New("data-proxy-log") - client *http.Client = &http.Client{ - Timeout: time.Second * 30, - Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}, - } - tokenCache = map[int64]*jwtToken{} + logger = log.New("data-proxy-log") + client = newHTTPClient() ) -type jwtToken struct { - ExpiresOn time.Time `json:"-"` - ExpiresOnString string `json:"expires_on"` - AccessToken string `json:"access_token"` -} - type DataSourceProxy struct { ds *m.DataSource ctx *m.ReqContext @@ -48,15 +36,26 @@ type DataSourceProxy struct { plugin *plugins.DataSourcePlugin } +type httpClient interface { + Do(req *http.Request) (*http.Response, error) +} + func NewDataSourceProxy(ds *m.DataSource, plugin *plugins.DataSourcePlugin, ctx *m.ReqContext, proxyPath string) *DataSourceProxy { - targetUrl, _ := url.Parse(ds.Url) + targetURL, _ := url.Parse(ds.Url) return &DataSourceProxy{ ds: ds, plugin: plugin, ctx: ctx, proxyPath: proxyPath, - targetUrl: targetUrl, + targetUrl: targetURL, + } +} + +func newHTTPClient() httpClient { + return &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}, } } @@ -89,6 +88,9 @@ func (proxy *DataSourceProxy) HandleRequest() { span.SetTag("user_id", proxy.ctx.SignedInUser.UserId) span.SetTag("org_id", proxy.ctx.SignedInUser.OrgId) + proxy.addTraceFromHeaderValue(span, "X-Panel-Id", "panel_id") + proxy.addTraceFromHeaderValue(span, "X-Dashboard-Id", "dashboard_id") + opentracing.GlobalTracer().Inject( span.Context(), opentracing.HTTPHeaders, @@ -98,6 +100,36 @@ func (proxy *DataSourceProxy) HandleRequest() { proxy.ctx.Resp.Header().Del("Set-Cookie") } +func (proxy *DataSourceProxy) addTraceFromHeaderValue(span opentracing.Span, headerName string, tagName string) { + panelId := proxy.ctx.Req.Header.Get(headerName) + dashId, err := strconv.Atoi(panelId) + if err == nil { + span.SetTag(tagName, dashId) + } +} + +func (proxy *DataSourceProxy) useCustomHeaders(req *http.Request) { + decryptSdj := proxy.ds.SecureJsonData.Decrypt() + index := 1 + for { + headerNameSuffix := fmt.Sprintf("httpHeaderName%d", index) + headerValueSuffix := fmt.Sprintf("httpHeaderValue%d", index) + if key := proxy.ds.JsonData.Get(headerNameSuffix).MustString(); key != "" { + if val, ok := decryptSdj[headerValueSuffix]; ok { + // remove if exists + if req.Header.Get(key) != "" { + req.Header.Del(key) + } + req.Header.Add(key, val) + logger.Debug("Using custom header ", "CustomHeaders", key) + } + } else { + break + } + index += 1 + } +} + func (proxy *DataSourceProxy) getDirector() func(req *http.Request) { return func(req *http.Request) { req.URL.Scheme = proxy.targetUrl.Scheme @@ -121,12 +153,16 @@ func (proxy *DataSourceProxy) getDirector() func(req *http.Request) { } else { req.URL.Path = util.JoinUrlFragments(proxy.targetUrl.Path, proxy.proxyPath) } - if proxy.ds.BasicAuth { req.Header.Del("Authorization") req.Header.Add("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser, proxy.ds.BasicAuthPassword)) } + // Lookup and use custom headers + if proxy.ds.SecureJsonData != nil { + proxy.useCustomHeaders(req) + } + dsAuth := req.Header.Get("X-DS-Authorization") if len(dsAuth) > 0 { req.Header.Del("X-DS-Authorization") @@ -157,6 +193,11 @@ func (proxy *DataSourceProxy) getDirector() func(req *http.Request) { req.Header.Del("X-Forwarded-Host") req.Header.Del("X-Forwarded-Port") req.Header.Del("X-Forwarded-Proto") + req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) + + // Clear Origin and Referer to avoir CORS issues + req.Header.Del("Origin") + req.Header.Del("Referer") // set X-Forwarded-For header if req.RemoteAddr != "" { @@ -172,18 +213,12 @@ func (proxy *DataSourceProxy) getDirector() func(req *http.Request) { } if proxy.route != nil { - proxy.applyRoute(req) + ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds) } } } func (proxy *DataSourceProxy) validateRequest() error { - if proxy.ds.Type == m.DS_INFLUXDB { - if proxy.ctx.Query("db") != proxy.ds.Database { - return errors.New("Datasource is not configured to allow this database") - } - } - if !checkWhiteList(proxy.ctx, proxy.targetUrl.Host) { return errors.New("Target url is not a valid target") } @@ -270,110 +305,3 @@ func checkWhiteList(c *m.ReqContext, host string) bool { return true } - -func (proxy *DataSourceProxy) applyRoute(req *http.Request) { - proxy.proxyPath = strings.TrimPrefix(proxy.proxyPath, proxy.route.Path) - - data := templateData{ - JsonData: proxy.ds.JsonData.Interface().(map[string]interface{}), - SecureJsonData: proxy.ds.SecureJsonData.Decrypt(), - } - - routeUrl, err := url.Parse(proxy.route.Url) - if err != nil { - logger.Error("Error parsing plugin route url") - return - } - - req.URL.Scheme = routeUrl.Scheme - req.URL.Host = routeUrl.Host - req.Host = routeUrl.Host - req.URL.Path = util.JoinUrlFragments(routeUrl.Path, proxy.proxyPath) - - if err := addHeaders(&req.Header, proxy.route, data); err != nil { - logger.Error("Failed to render plugin headers", "error", err) - } - - if proxy.route.TokenAuth != nil { - if token, err := proxy.getAccessToken(data); err != nil { - logger.Error("Failed to get access token", "error", err) - } else { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) - } - } - - logger.Info("Requesting", "url", req.URL.String()) -} - -func (proxy *DataSourceProxy) getAccessToken(data templateData) (string, error) { - if cachedToken, found := tokenCache[proxy.ds.Id]; found { - if cachedToken.ExpiresOn.After(time.Now().Add(time.Second * 10)) { - logger.Info("Using token from cache") - return cachedToken.AccessToken, nil - } - } - - urlInterpolated, err := interpolateString(proxy.route.TokenAuth.Url, data) - if err != nil { - return "", err - } - - params := make(url.Values) - for key, value := range proxy.route.TokenAuth.Params { - if interpolatedParam, err := interpolateString(value, data); err != nil { - return "", err - } else { - params.Add(key, interpolatedParam) - } - } - - getTokenReq, _ := http.NewRequest("POST", urlInterpolated, bytes.NewBufferString(params.Encode())) - getTokenReq.Header.Add("Content-Type", "application/x-www-form-urlencoded") - getTokenReq.Header.Add("Content-Length", strconv.Itoa(len(params.Encode()))) - - resp, err := client.Do(getTokenReq) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - var token jwtToken - if err := json.NewDecoder(resp.Body).Decode(&token); err != nil { - return "", err - } - - expiresOnEpoch, _ := strconv.ParseInt(token.ExpiresOnString, 10, 64) - token.ExpiresOn = time.Unix(expiresOnEpoch, 0) - tokenCache[proxy.ds.Id] = &token - - logger.Info("Got new access token", "ExpiresOn", token.ExpiresOn) - return token.AccessToken, nil -} - -func interpolateString(text string, data templateData) (string, error) { - t, err := template.New("content").Parse(text) - if err != nil { - return "", errors.New(fmt.Sprintf("Could not parse template %s.", text)) - } - - var contentBuf bytes.Buffer - err = t.Execute(&contentBuf, data) - if err != nil { - return "", errors.New(fmt.Sprintf("Failed to execute template %s.", text)) - } - - return contentBuf.String(), nil -} - -func addHeaders(reqHeaders *http.Header, route *plugins.AppPluginRoute, data templateData) error { - for _, header := range route.Headers { - interpolated, err := interpolateString(header.Content, data) - if err != nil { - return err - } - reqHeaders.Add(header.Name, interpolated) - } - - return nil -} diff --git a/pkg/api/pluginproxy/ds_proxy_test.go b/pkg/api/pluginproxy/ds_proxy_test.go index 3cf67d9178a..c9be169565f 100644 --- a/pkg/api/pluginproxy/ds_proxy_test.go +++ b/pkg/api/pluginproxy/ds_proxy_test.go @@ -1,13 +1,18 @@ package pluginproxy import ( + "bytes" + "fmt" + "io/ioutil" "net/http" "net/url" "testing" + "time" macaron "gopkg.in/macaron.v1" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/setting" @@ -44,6 +49,13 @@ func TestDSRouteRule(t *testing.T) { {Name: "x-header", Content: "my secret {{.SecureJsonData.key}}"}, }, }, + { + Path: "api/common", + Url: "{{.JsonData.dynamicUrl}}", + Headers: []plugins.AppPluginRouteHeader{ + {Name: "x-header", Content: "my secret {{.SecureJsonData.key}}"}, + }, + }, }, } @@ -52,7 +64,8 @@ func TestDSRouteRule(t *testing.T) { ds := &m.DataSource{ JsonData: simplejson.NewFromAny(map[string]interface{}{ - "clientId": "asd", + "clientId": "asd", + "dynamicUrl": "https://dynamic.grafana.com", }), SecureJsonData: map[string][]byte{ "key": key, @@ -70,7 +83,7 @@ func TestDSRouteRule(t *testing.T) { Convey("When matching route path", func() { proxy := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method") proxy.route = plugin.Routes[0] - proxy.applyRoute(req) + ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds) Convey("should add headers and update url", func() { So(req.URL.String(), ShouldEqual, "https://www.google.com/some/method") @@ -78,6 +91,17 @@ func TestDSRouteRule(t *testing.T) { }) }) + Convey("When matching route path and has dynamic url", func() { + proxy := NewDataSourceProxy(ds, plugin, ctx, "api/common/some/method") + proxy.route = plugin.Routes[3] + ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds) + + Convey("should add headers and interpolate the url", func() { + So(req.URL.String(), ShouldEqual, "https://dynamic.grafana.com/some/method") + So(req.Header.Get("x-header"), ShouldEqual, "my secret 123") + }) + }) + Convey("Validating request", func() { Convey("plugin route with valid role", func() { proxy := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method") @@ -100,21 +124,128 @@ func TestDSRouteRule(t *testing.T) { }) }) + Convey("Plugin with multiple routes for token auth", func() { + plugin := &plugins.DataSourcePlugin{ + Routes: []*plugins.AppPluginRoute{ + { + Path: "pathwithtoken1", + Url: "https://api.nr1.io/some/path", + TokenAuth: &plugins.JwtTokenAuth{ + Url: "https://login.server.com/{{.JsonData.tenantId}}/oauth2/token", + Params: map[string]string{ + "grant_type": "client_credentials", + "client_id": "{{.JsonData.clientId}}", + "client_secret": "{{.SecureJsonData.clientSecret}}", + "resource": "https://api.nr1.io", + }, + }, + }, + { + Path: "pathwithtoken2", + Url: "https://api.nr2.io/some/path", + TokenAuth: &plugins.JwtTokenAuth{ + Url: "https://login.server.com/{{.JsonData.tenantId}}/oauth2/token", + Params: map[string]string{ + "grant_type": "client_credentials", + "client_id": "{{.JsonData.clientId}}", + "client_secret": "{{.SecureJsonData.clientSecret}}", + "resource": "https://api.nr2.io", + }, + }, + }, + }, + } + + setting.SecretKey = "password" + key, _ := util.Encrypt([]byte("123"), "password") + + ds := &m.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "clientId": "asd", + "tenantId": "mytenantId", + }), + SecureJsonData: map[string][]byte{ + "clientSecret": key, + }, + } + + req, _ := http.NewRequest("GET", "http://localhost/asd", nil) + ctx := &m.ReqContext{ + Context: &macaron.Context{ + Req: macaron.Request{Request: req}, + }, + SignedInUser: &m.SignedInUser{OrgRole: m.ROLE_EDITOR}, + } + + Convey("When creating and caching access tokens", func() { + var authorizationHeaderCall1 string + var authorizationHeaderCall2 string + + Convey("first call should add authorization header with access token", func() { + json, err := ioutil.ReadFile("./test-data/access-token-1.json") + So(err, ShouldBeNil) + + client = newFakeHTTPClient(json) + proxy1 := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1") + proxy1.route = plugin.Routes[0] + ApplyRoute(proxy1.ctx.Req.Context(), req, proxy1.proxyPath, proxy1.route, proxy1.ds) + + authorizationHeaderCall1 = req.Header.Get("Authorization") + So(req.URL.String(), ShouldEqual, "https://api.nr1.io/some/path") + So(authorizationHeaderCall1, ShouldStartWith, "Bearer eyJ0e") + + Convey("second call to another route should add a different access token", func() { + json2, err := ioutil.ReadFile("./test-data/access-token-2.json") + So(err, ShouldBeNil) + + req, _ := http.NewRequest("GET", "http://localhost/asd", nil) + client = newFakeHTTPClient(json2) + proxy2 := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken2") + proxy2.route = plugin.Routes[1] + ApplyRoute(proxy2.ctx.Req.Context(), req, proxy2.proxyPath, proxy2.route, proxy2.ds) + + authorizationHeaderCall2 = req.Header.Get("Authorization") + + So(req.URL.String(), ShouldEqual, "https://api.nr2.io/some/path") + So(authorizationHeaderCall1, ShouldStartWith, "Bearer eyJ0e") + So(authorizationHeaderCall2, ShouldStartWith, "Bearer eyJ0e") + So(authorizationHeaderCall2, ShouldNotEqual, authorizationHeaderCall1) + + Convey("third call to first route should add cached access token", func() { + req, _ := http.NewRequest("GET", "http://localhost/asd", nil) + + client = newFakeHTTPClient([]byte{}) + proxy3 := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1") + proxy3.route = plugin.Routes[0] + ApplyRoute(proxy3.ctx.Req.Context(), req, proxy3.proxyPath, proxy3.route, proxy3.ds) + + authorizationHeaderCall3 := req.Header.Get("Authorization") + So(req.URL.String(), ShouldEqual, "https://api.nr1.io/some/path") + So(authorizationHeaderCall1, ShouldStartWith, "Bearer eyJ0e") + So(authorizationHeaderCall3, ShouldStartWith, "Bearer eyJ0e") + So(authorizationHeaderCall3, ShouldEqual, authorizationHeaderCall1) + }) + }) + }) + }) + }) + Convey("When proxying graphite", func() { + setting.BuildVersion = "5.3.0" plugin := &plugins.DataSourcePlugin{} ds := &m.DataSource{Url: "htttp://graphite:8080", Type: m.DS_GRAPHITE} ctx := &m.ReqContext{} proxy := NewDataSourceProxy(ds, plugin, ctx, "/render") + req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) + So(err, ShouldBeNil) - requestUrl, _ := url.Parse("http://grafana.com/sub") - req := http.Request{URL: requestUrl} - - proxy.getDirector()(&req) + proxy.getDirector()(req) Convey("Can translate request url and path", func() { So(req.URL.Host, ShouldEqual, "graphite:8080") So(req.URL.Path, ShouldEqual, "/render") + So(req.Header.Get("User-Agent"), ShouldEqual, "Grafana/5.3.0") }) }) @@ -132,10 +263,10 @@ func TestDSRouteRule(t *testing.T) { ctx := &m.ReqContext{} proxy := NewDataSourceProxy(ds, plugin, ctx, "") - requestUrl, _ := url.Parse("http://grafana.com/sub") - req := http.Request{URL: requestUrl} + req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) + So(err, ShouldBeNil) - proxy.getDirector()(&req) + proxy.getDirector()(req) Convey("Should add db to url", func() { So(req.URL.Path, ShouldEqual, "/db/site/") @@ -162,8 +293,8 @@ func TestDSRouteRule(t *testing.T) { ctx := &m.ReqContext{} proxy := NewDataSourceProxy(ds, plugin, ctx, "") - requestUrl, _ := url.Parse("http://grafana.com/sub") - req := http.Request{URL: requestUrl, Header: make(http.Header)} + requestURL, _ := url.Parse("http://grafana.com/sub") + req := http.Request{URL: requestURL, Header: make(http.Header)} cookies := "grafana_user=admin; grafana_remember=99; grafana_sess=11; JSESSION_ID=test" req.Header.Set("Cookie", cookies) @@ -188,8 +319,8 @@ func TestDSRouteRule(t *testing.T) { ctx := &m.ReqContext{} proxy := NewDataSourceProxy(ds, plugin, ctx, "") - requestUrl, _ := url.Parse("http://grafana.com/sub") - req := http.Request{URL: requestUrl, Header: make(http.Header)} + requestURL, _ := url.Parse("http://grafana.com/sub") + req := http.Request{URL: requestURL, Header: make(http.Header)} cookies := "grafana_user=admin; grafana_remember=99; grafana_sess=11; JSESSION_ID=test" req.Header.Set("Cookie", cookies) @@ -200,17 +331,86 @@ func TestDSRouteRule(t *testing.T) { }) }) - Convey("When interpolating string", func() { - data := templateData{ - SecureJsonData: map[string]string{ - "Test": "0asd+asd", + Convey("When proxying a data source with custom headers specified", func() { + plugin := &plugins.DataSourcePlugin{} + + encryptedData, err := util.Encrypt([]byte(`Bearer xf5yhfkpsnmgo`), setting.SecretKey) + ds := &m.DataSource{ + Type: m.DS_PROMETHEUS, + Url: "http://prometheus:9090", + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "httpHeaderName1": "Authorization", + }), + SecureJsonData: map[string][]byte{ + "httpHeaderValue1": encryptedData, }, } - interpolated, err := interpolateString("{{.SecureJsonData.Test}}", data) - So(err, ShouldBeNil) - So(interpolated, ShouldEqual, "0asd+asd") + ctx := &m.ReqContext{} + proxy := NewDataSourceProxy(ds, plugin, ctx, "") + + requestURL, _ := url.Parse("http://grafana.com/sub") + req := http.Request{URL: requestURL, Header: make(http.Header)} + proxy.getDirector()(&req) + + if err != nil { + log.Fatal(4, err.Error()) + } + + Convey("Match header value after decryption", func() { + So(req.Header.Get("Authorization"), ShouldEqual, "Bearer xf5yhfkpsnmgo") + }) }) + Convey("When proxying a custom datasource", func() { + plugin := &plugins.DataSourcePlugin{} + ds := &m.DataSource{ + Type: "custom-datasource", + Url: "http://host/root/", + } + ctx := &m.ReqContext{} + proxy := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/") + req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) + req.Header.Add("Origin", "grafana.com") + req.Header.Add("Referer", "grafana.com") + req.Header.Add("X-Canary", "stillthere") + So(err, ShouldBeNil) + + proxy.getDirector()(req) + + Convey("Should keep user request (including trailing slash)", func() { + So(req.URL.String(), ShouldEqual, "http://host/root/path/to/folder/") + }) + + Convey("Origin and Referer headers should be dropped", func() { + So(req.Header.Get("Origin"), ShouldEqual, "") + So(req.Header.Get("Referer"), ShouldEqual, "") + So(req.Header.Get("X-Canary"), ShouldEqual, "stillthere") + }) + }) }) } + +type httpClientStub struct { + fakeBody []byte +} + +func (c *httpClientStub) Do(req *http.Request) (*http.Response, error) { + bodyJSON, _ := simplejson.NewJson(c.fakeBody) + _, passedTokenCacheTest := bodyJSON.CheckGet("expires_on") + So(passedTokenCacheTest, ShouldBeTrue) + + bodyJSON.Set("expires_on", fmt.Sprint(time.Now().Add(time.Second*60).Unix())) + body, _ := bodyJSON.MarshalJSON() + resp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader(body)), + } + + return resp, nil +} + +func newFakeHTTPClient(fakeBody []byte) httpClient { + return &httpClientStub{ + fakeBody: fakeBody, + } +} diff --git a/pkg/api/pluginproxy/pluginproxy.go b/pkg/api/pluginproxy/pluginproxy.go index eb78250838a..ffbe470cb20 100644 --- a/pkg/api/pluginproxy/pluginproxy.go +++ b/pkg/api/pluginproxy/pluginproxy.go @@ -19,10 +19,10 @@ type templateData struct { SecureJsonData map[string]string } -func getHeaders(route *plugins.AppPluginRoute, orgId int64, appId string) (http.Header, error) { +func getHeaders(route *plugins.AppPluginRoute, orgId int64, appID string) (http.Header, error) { result := http.Header{} - query := m.GetPluginSettingByIdQuery{OrgId: orgId, PluginId: appId} + query := m.GetPluginSettingByIdQuery{OrgId: orgId, PluginId: appID} if err := bus.Dispatch(&query); err != nil { return nil, err @@ -37,16 +37,16 @@ func getHeaders(route *plugins.AppPluginRoute, orgId int64, appId string) (http. return result, err } -func NewApiPluginProxy(ctx *m.ReqContext, proxyPath string, route *plugins.AppPluginRoute, appId string) *httputil.ReverseProxy { - targetUrl, _ := url.Parse(route.Url) +func NewApiPluginProxy(ctx *m.ReqContext, proxyPath string, route *plugins.AppPluginRoute, appID string) *httputil.ReverseProxy { + targetURL, _ := url.Parse(route.Url) director := func(req *http.Request) { - req.URL.Scheme = targetUrl.Scheme - req.URL.Host = targetUrl.Host - req.Host = targetUrl.Host + req.URL.Scheme = targetURL.Scheme + req.URL.Host = targetURL.Host + req.Host = targetURL.Host - req.URL.Path = util.JoinUrlFragments(targetUrl.Path, proxyPath) + req.URL.Path = util.JoinUrlFragments(targetURL.Path, proxyPath) // clear cookie headers req.Header.Del("Cookie") @@ -80,7 +80,7 @@ func NewApiPluginProxy(ctx *m.ReqContext, proxyPath string, route *plugins.AppPl req.Header.Add("X-Grafana-Context", string(ctxJson)) if len(route.Headers) > 0 { - headers, err := getHeaders(route, ctx.OrgId, appId) + headers, err := getHeaders(route, ctx.OrgId, appID) if err != nil { ctx.JsonApiErr(500, "Could not generate plugin route header", err) return diff --git a/pkg/api/pluginproxy/test-data/access-token-1.json b/pkg/api/pluginproxy/test-data/access-token-1.json new file mode 100644 index 00000000000..b91d63fc659 --- /dev/null +++ b/pkg/api/pluginproxy/test-data/access-token-1.json @@ -0,0 +1,9 @@ +{ + "token_type": "Bearer", + "expires_in": "3599", + "ext_expires_in": "0", + "expires_on": "1528740417", + "not_before": "1528736517", + "resource": "https://api.nr1.io", + "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6ImlCakwxUmNxemhpeTRmcHhJeGRacW9oTTJZayIsImtpZCI6ImlCakwxUmNxemhpeTRmcHhJeGRacW9oTTJZayJ9.eyJhdWQiOiJodHRwczovL2FwaS5sb2dhbmFseXRpY3MuaW8iLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9lN2YzZjY2MS1hOTMzLTRiM2YtODE3Ni01MWM0Zjk4MmVjNDgvIiwiaWF0IjoxNTI4NzM2NTE3LCJuYmYiOjE1Mjg3MzY1MTcsImV4cCI6MTUyODc0MDQxNywiYWlvIjoiWTJkZ1lBaStzaWRsT3NmQ2JicGhLMSsremttN0NBQT0iLCJhcHBpZCI6IjdmMzJkYjdjLTZmNmYtNGU4OC05M2Q5LTlhZTEyNmMwYTU1ZiIsImFwcGlkYWNyIjoiMSIsImlkcCI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0L2U3ZjNmNjYxLWE5MzMtNGIzZi04MTc2LTUxYzRmOTgyZWM0OC8iLCJvaWQiOiI1NDQ5ZmJjOS1mYWJhLTRkNjItODE2Yy05ZmMwMzZkMWViN2UiLCJzdWIiOiI1NDQ5ZmJjOS1mYWJhLTRkNjItODE2Yy05ZmMwMzZkMWViN2UiLCJ0aWQiOiJlN2YzZjY2MS1hOTMzLTRiM2YtODE3Ni01MWM0Zjk4MmVjNDgiLCJ1dGkiOiJZQTlQa2lxUy1VV1hMQjhIRnU0U0FBIiwidmVyIjoiMS4wIn0.ga5qudt4LDMKTStAxUmzjyZH8UFBAaFirJqpTdmYny4NtkH6JT2EILvjTjYxlKeTQisvwx9gof0PyicZIab9d6wlMa2xiLzr2nmaOonYClY8fqBaRTgc1xVjrKFw5SCgpx3FnEyJhIWvVPIfaWaogSHcQbIpe4kdk4tz-ccmrx0D1jsziSI4BZcJcX04aJuHZGz9k4mQZ_AA5sQSeQaNuojIng6rYoIifAXFYBZPTbeeeqmiGq8v0IOLeNKbC0POeQCJC_KKBG6Z_MV2KgPxFEzQuX2ZFmRD_wGPteV5TUBxh1kARdqexA3e0zAKSawR9kmrAiZ21lPr4tX2Br_HDg" +} diff --git a/pkg/api/pluginproxy/test-data/access-token-2.json b/pkg/api/pluginproxy/test-data/access-token-2.json new file mode 100644 index 00000000000..2a2a617ad80 --- /dev/null +++ b/pkg/api/pluginproxy/test-data/access-token-2.json @@ -0,0 +1,9 @@ +{ + "token_type": "Bearer", + "expires_in": "3599", + "ext_expires_in": "0", + "expires_on": "1528662059", + "not_before": "1528658159", + "resource": "https://api.nr2.io", + "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6ImlCakwxUmNxemhpeTRmcHhJeGRacW9oTTJZayIsImtpZCI6ImlCakwxUmNxemhpeTRmcHhJeGRacW9oTTJZayJ9.eyJhdWQiOiJodHRwczovL21hbmFnZW1lbnQuYXp1cmUuY29tLyIsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0L2U3ZjNmNjYxLWE5MzMtNGIzZi04MTc2LTUxYzRmOTgyZWM0OC8iLCJpYXQiOjE1Mjg2NTgxNTksIm5iZiI6MTUyODY1ODE1OSwiZXhwIjoxNTI4NjYyMDU5LCJhaW8iOiJZMmRnWUFpK3NpZGxPc2ZDYmJwaEsxKyt6a203Q0FBPSIsImFwcGlkIjoiODg5YjdlZDgtMWFlZC00ODZlLTk3ODktODE5NzcwYmJiNjFhIiwiYXBwaWRhY3IiOiIxIiwiaWRwIjoiaHR0cHM6Ly9zdHMud2luZG93cy5uZXQvZTdmM2Y2NjEtYTkzMy00YjNmLTgxNzYtNTFjNGY5ODJlYzQ4LyIsIm9pZCI6IjY0YzQxNjMyLTliOWUtNDczNy05MTYwLTBlNjAzZTg3NjljYyIsInN1YiI6IjY0YzQxNjMyLTliOWUtNDczNy05MTYwLTBlNjAzZTg3NjljYyIsInRpZCI6ImU3ZjNmNjYxLWE5MzMtNGIzZi04MTc2LTUxYzRmOTgyZWM0OCIsInV0aSI6IkQ1ODZHSGUySDBPd0ptOU0xeVlKQUEiLCJ2ZXIiOiIxLjAifQ.Pw8c8gpoZptw3lGreQoHQaMVOozSaTE5D38Vm2aCHRB3DvD3N-Qcm1x0ZCakUEV2sJd7jvx4XtPFuW7063T0V1deExL4rzzvIo0ZfMmURf9tCTiKFKYibqf8_PtfPSz0t9eNDEUGmWDh1Wgssb4W_H-wPqgl9VPMT7T6ynkfIm0-ODPZTBzgSHiY8C_L1-DkhsK7XiqbUlSDgx9FpfChZS3ah8QhA8geqnb_HVuSktg7WhpxmogSpK5QdrwSE3jsbItpzOfLJ4iBd2ExzS2C0y8H_Coluk3Y1YA07tAxJ6Y7oBv-XwGqNfZhveOCQOzX-U3dFod3fXXysjB0UB89WQ" +} diff --git a/pkg/api/plugins.go b/pkg/api/plugins.go index bc38f4a7775..455420e4688 100644 --- a/pkg/api/plugins.go +++ b/pkg/api/plugins.go @@ -10,7 +10,7 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -func GetPluginList(c *m.ReqContext) Response { +func (hs *HTTPServer) GetPluginList(c *m.ReqContext) Response { typeFilter := c.Query("type") enabledFilter := c.Query("enabled") embeddedFilter := c.Query("embedded") @@ -19,7 +19,7 @@ func GetPluginList(c *m.ReqContext) Response { pluginSettingsMap, err := plugins.GetPluginSettings(c.OrgId) if err != nil { - return ApiError(500, "Failed to get list of plugins", err) + return Error(500, "Failed to get list of plugins", err) } result := make(dtos.PluginList, 0) @@ -39,6 +39,10 @@ func GetPluginList(c *m.ReqContext) Response { continue } + if pluginDef.State == plugins.PluginStateAlpha && !hs.Cfg.EnableAlphaPanels { + continue + } + listItem := dtos.PluginListItem{ Id: pluginDef.Id, Name: pluginDef.Name, @@ -75,92 +79,94 @@ func GetPluginList(c *m.ReqContext) Response { } sort.Sort(result) - return Json(200, result) + return JSON(200, result) } -func GetPluginSettingById(c *m.ReqContext) Response { - pluginId := c.Params(":pluginId") +func GetPluginSettingByID(c *m.ReqContext) Response { + pluginID := c.Params(":pluginId") - if def, exists := plugins.Plugins[pluginId]; !exists { - return ApiError(404, "Plugin not found, no installed plugin with that id", nil) - } else { - - dto := &dtos.PluginSetting{ - Type: def.Type, - Id: def.Id, - Name: def.Name, - Info: &def.Info, - Dependencies: &def.Dependencies, - Includes: def.Includes, - BaseUrl: def.BaseUrl, - Module: def.Module, - DefaultNavUrl: def.DefaultNavUrl, - LatestVersion: def.GrafanaNetVersion, - HasUpdate: def.GrafanaNetHasUpdate, - State: def.State, - } - - query := m.GetPluginSettingByIdQuery{PluginId: pluginId, OrgId: c.OrgId} - if err := bus.Dispatch(&query); err != nil { - if err != m.ErrPluginSettingNotFound { - return ApiError(500, "Failed to get login settings", nil) - } - } else { - dto.Enabled = query.Result.Enabled - dto.Pinned = query.Result.Pinned - dto.JsonData = query.Result.JsonData - } - - return Json(200, dto) + def, exists := plugins.Plugins[pluginID] + if !exists { + return Error(404, "Plugin not found, no installed plugin with that id", nil) } + + dto := &dtos.PluginSetting{ + Type: def.Type, + Id: def.Id, + Name: def.Name, + Info: &def.Info, + Dependencies: &def.Dependencies, + Includes: def.Includes, + BaseUrl: def.BaseUrl, + Module: def.Module, + DefaultNavUrl: def.DefaultNavUrl, + LatestVersion: def.GrafanaNetVersion, + HasUpdate: def.GrafanaNetHasUpdate, + State: def.State, + } + + query := m.GetPluginSettingByIdQuery{PluginId: pluginID, OrgId: c.OrgId} + if err := bus.Dispatch(&query); err != nil { + if err != m.ErrPluginSettingNotFound { + return Error(500, "Failed to get login settings", nil) + } + } else { + dto.Enabled = query.Result.Enabled + dto.Pinned = query.Result.Pinned + dto.JsonData = query.Result.JsonData + } + + return JSON(200, dto) } func UpdatePluginSetting(c *m.ReqContext, cmd m.UpdatePluginSettingCmd) Response { - pluginId := c.Params(":pluginId") + pluginID := c.Params(":pluginId") cmd.OrgId = c.OrgId - cmd.PluginId = pluginId + cmd.PluginId = pluginID if _, ok := plugins.Apps[cmd.PluginId]; !ok { - return ApiError(404, "Plugin not installed.", nil) + return Error(404, "Plugin not installed.", nil) } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update plugin setting", err) + return Error(500, "Failed to update plugin setting", err) } - return ApiSuccess("Plugin settings updated") + return Success("Plugin settings updated") } func GetPluginDashboards(c *m.ReqContext) Response { - pluginId := c.Params(":pluginId") + pluginID := c.Params(":pluginId") - if list, err := plugins.GetPluginDashboards(c.OrgId, pluginId); err != nil { + list, err := plugins.GetPluginDashboards(c.OrgId, pluginID) + if err != nil { if notfound, ok := err.(plugins.PluginNotFoundError); ok { - return ApiError(404, notfound.Error(), nil) + return Error(404, notfound.Error(), nil) } - return ApiError(500, "Failed to get plugin dashboards", err) - } else { - return Json(200, list) + return Error(500, "Failed to get plugin dashboards", err) } + + return JSON(200, list) } func GetPluginMarkdown(c *m.ReqContext) Response { - pluginId := c.Params(":pluginId") + pluginID := c.Params(":pluginId") name := c.Params(":name") - if content, err := plugins.GetPluginMarkdown(pluginId, name); err != nil { + content, err := plugins.GetPluginMarkdown(pluginID, name) + if err != nil { if notfound, ok := err.(plugins.PluginNotFoundError); ok { - return ApiError(404, notfound.Error(), nil) + return Error(404, notfound.Error(), nil) } - return ApiError(500, "Could not get markdown file", err) - } else { - resp := Respond(200, content) - resp.Header("Content-Type", "text/plain; charset=utf-8") - return resp + return Error(500, "Could not get markdown file", err) } + + resp := Respond(200, content) + resp.Header("Content-Type", "text/plain; charset=utf-8") + return resp } func ImportDashboard(c *m.ReqContext, apiCmd dtos.ImportDashboardCommand) Response { @@ -172,12 +178,13 @@ func ImportDashboard(c *m.ReqContext, apiCmd dtos.ImportDashboardCommand) Respon Path: apiCmd.Path, Inputs: apiCmd.Inputs, Overwrite: apiCmd.Overwrite, + FolderId: apiCmd.FolderId, Dashboard: apiCmd.Dashboard, } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to import dashboard", err) + return Error(500, "Failed to import dashboard", err) } - return Json(200, cmd.Result) + return JSON(200, cmd.Result) } diff --git a/pkg/api/preferences.go b/pkg/api/preferences.go index eb0ffa14b39..9b451aa2a6e 100644 --- a/pkg/api/preferences.go +++ b/pkg/api/preferences.go @@ -13,60 +13,61 @@ func SetHomeDashboard(c *m.ReqContext, cmd m.SavePreferencesCommand) Response { cmd.OrgId = c.OrgId if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to set home dashboard", err) + return Error(500, "Failed to set home dashboard", err) } - return ApiSuccess("Home dashboard set") + return Success("Home dashboard set") } // GET /api/user/preferences func GetUserPreferences(c *m.ReqContext) Response { - return getPreferencesFor(c.OrgId, c.UserId) + return getPreferencesFor(c.OrgId, c.UserId, 0) } -func getPreferencesFor(orgId int64, userId int64) Response { - prefsQuery := m.GetPreferencesQuery{UserId: userId, OrgId: orgId} +func getPreferencesFor(orgID, userID, teamID int64) Response { + prefsQuery := m.GetPreferencesQuery{UserId: userID, OrgId: orgID, TeamId: teamID} if err := bus.Dispatch(&prefsQuery); err != nil { - return ApiError(500, "Failed to get preferences", err) + return Error(500, "Failed to get preferences", err) } dto := dtos.Prefs{ Theme: prefsQuery.Result.Theme, - HomeDashboardId: prefsQuery.Result.HomeDashboardId, + HomeDashboardID: prefsQuery.Result.HomeDashboardId, Timezone: prefsQuery.Result.Timezone, } - return Json(200, &dto) + return JSON(200, &dto) } // PUT /api/user/preferences func UpdateUserPreferences(c *m.ReqContext, dtoCmd dtos.UpdatePrefsCmd) Response { - return updatePreferencesFor(c.OrgId, c.UserId, &dtoCmd) + return updatePreferencesFor(c.OrgId, c.UserId, 0, &dtoCmd) } -func updatePreferencesFor(orgId int64, userId int64, dtoCmd *dtos.UpdatePrefsCmd) Response { +func updatePreferencesFor(orgID, userID, teamId int64, dtoCmd *dtos.UpdatePrefsCmd) Response { saveCmd := m.SavePreferencesCommand{ - UserId: userId, - OrgId: orgId, + UserId: userID, + OrgId: orgID, + TeamId: teamId, Theme: dtoCmd.Theme, Timezone: dtoCmd.Timezone, - HomeDashboardId: dtoCmd.HomeDashboardId, + HomeDashboardId: dtoCmd.HomeDashboardID, } if err := bus.Dispatch(&saveCmd); err != nil { - return ApiError(500, "Failed to save preferences", err) + return Error(500, "Failed to save preferences", err) } - return ApiSuccess("Preferences updated") + return Success("Preferences updated") } // GET /api/org/preferences func GetOrgPreferences(c *m.ReqContext) Response { - return getPreferencesFor(c.OrgId, 0) + return getPreferencesFor(c.OrgId, 0, 0) } // PUT /api/org/preferences func UpdateOrgPreferences(c *m.ReqContext, dtoCmd dtos.UpdatePrefsCmd) Response { - return updatePreferencesFor(c.OrgId, 0, &dtoCmd) + return updatePreferencesFor(c.OrgId, 0, 0, &dtoCmd) } diff --git a/pkg/api/quota.go b/pkg/api/quota.go index f92acaf470f..d469f843930 100644 --- a/pkg/api/quota.go +++ b/pkg/api/quota.go @@ -8,60 +8,60 @@ import ( func GetOrgQuotas(c *m.ReqContext) Response { if !setting.Quota.Enabled { - return ApiError(404, "Quotas not enabled", nil) + return Error(404, "Quotas not enabled", nil) } query := m.GetOrgQuotasQuery{OrgId: c.ParamsInt64(":orgId")} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to get org quotas", err) + return Error(500, "Failed to get org quotas", err) } - return Json(200, query.Result) + return JSON(200, query.Result) } func UpdateOrgQuota(c *m.ReqContext, cmd m.UpdateOrgQuotaCmd) Response { if !setting.Quota.Enabled { - return ApiError(404, "Quotas not enabled", nil) + return Error(404, "Quotas not enabled", nil) } cmd.OrgId = c.ParamsInt64(":orgId") cmd.Target = c.Params(":target") if _, ok := setting.Quota.Org.ToMap()[cmd.Target]; !ok { - return ApiError(404, "Invalid quota target", nil) + return Error(404, "Invalid quota target", nil) } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update org quotas", err) + return Error(500, "Failed to update org quotas", err) } - return ApiSuccess("Organization quota updated") + return Success("Organization quota updated") } func GetUserQuotas(c *m.ReqContext) Response { if !setting.Quota.Enabled { - return ApiError(404, "Quotas not enabled", nil) + return Error(404, "Quotas not enabled", nil) } query := m.GetUserQuotasQuery{UserId: c.ParamsInt64(":id")} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to get org quotas", err) + return Error(500, "Failed to get org quotas", err) } - return Json(200, query.Result) + return JSON(200, query.Result) } func UpdateUserQuota(c *m.ReqContext, cmd m.UpdateUserQuotaCmd) Response { if !setting.Quota.Enabled { - return ApiError(404, "Quotas not enabled", nil) + return Error(404, "Quotas not enabled", nil) } cmd.UserId = c.ParamsInt64(":id") cmd.Target = c.Params(":target") if _, ok := setting.Quota.User.ToMap()[cmd.Target]; !ok { - return ApiError(404, "Invalid quota target", nil) + return Error(404, "Invalid quota target", nil) } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update org quotas", err) + return Error(500, "Failed to update org quotas", err) } - return ApiSuccess("Organization quota updated") + return Success("Organization quota updated") } diff --git a/pkg/api/render.go b/pkg/api/render.go index 6e948ed294c..cf672af9bea 100644 --- a/pkg/api/render.go +++ b/pkg/api/render.go @@ -3,44 +3,75 @@ package api import ( "fmt" "net/http" + "runtime" + "strconv" + "strings" + "time" - "github.com/grafana/grafana/pkg/components/renderer" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/util" ) -func RenderToPng(c *m.ReqContext) { +func (hs *HTTPServer) RenderToPng(c *m.ReqContext) { queryReader, err := util.NewUrlQueryReader(c.Req.URL) if err != nil { c.Handle(400, "Render parameters error", err) return } + queryParams := fmt.Sprintf("?%s", c.Req.URL.RawQuery) - renderOpts := &renderer.RenderOpts{ - Path: c.Params("*") + queryParams, - Width: queryReader.Get("width", "800"), - Height: queryReader.Get("height", "400"), - Timeout: queryReader.Get("timeout", "60"), - OrgId: c.OrgId, - UserId: c.UserId, - OrgRole: c.OrgRole, - Timezone: queryReader.Get("tz", ""), - Encoding: queryReader.Get("encoding", ""), + width, err := strconv.Atoi(queryReader.Get("width", "800")) + if err != nil { + c.Handle(400, "Render parameters error", fmt.Errorf("Cannot parse width as int: %s", err)) + return } - pngPath, err := renderer.RenderToPng(renderOpts) + height, err := strconv.Atoi(queryReader.Get("height", "400")) + if err != nil { + c.Handle(400, "Render parameters error", fmt.Errorf("Cannot parse height as int: %s", err)) + return + } - if err != nil && err == renderer.ErrTimeout { + timeout, err := strconv.Atoi(queryReader.Get("timeout", "60")) + if err != nil { + c.Handle(400, "Render parameters error", fmt.Errorf("Cannot parse timeout as int: %s", err)) + return + } + + result, err := hs.RenderService.Render(c.Req.Context(), rendering.Opts{ + Width: width, + Height: height, + Timeout: time.Duration(timeout) * time.Second, + OrgId: c.OrgId, + UserId: c.UserId, + OrgRole: c.OrgRole, + Path: c.Params("*") + queryParams, + Timezone: queryReader.Get("tz", ""), + Encoding: queryReader.Get("encoding", ""), + ConcurrentLimit: 30, + }) + + if err != nil && err == rendering.ErrTimeout { c.Handle(500, err.Error(), err) return } + if err != nil && err == rendering.ErrPhantomJSNotInstalled { + if strings.HasPrefix(runtime.GOARCH, "arm") { + c.Handle(500, "Rendering failed - PhantomJS isn't included in arm build per default", err) + } else { + c.Handle(500, "Rendering failed - PhantomJS isn't installed correctly", err) + } + return + } + if err != nil { c.Handle(500, "Rendering failed.", err) return } c.Resp.Header().Set("Content-Type", "image/png") - http.ServeFile(c.Resp, c.Req.Request, pngPath) + http.ServeFile(c.Resp, c.Req.Request, result.FilePath) } diff --git a/pkg/api/route_register.go b/pkg/api/routing/route_register.go similarity index 63% rename from pkg/api/route_register.go rename to pkg/api/routing/route_register.go index 76ebb633ca1..7a054ad0a24 100644 --- a/pkg/api/route_register.go +++ b/pkg/api/routing/route_register.go @@ -1,9 +1,10 @@ -package api +package routing import ( "net/http" + "strings" - macaron "gopkg.in/macaron.v1" + "gopkg.in/macaron.v1" ) type Router interface { @@ -11,22 +12,43 @@ type Router interface { Get(pattern string, handlers ...macaron.Handler) *macaron.Route } +// RouteRegister allows you to add routes and macaron.Handlers +// that the web server should serve. type RouteRegister interface { + // Get adds a list of handlers to a given route with a GET HTTP verb Get(string, ...macaron.Handler) + + // Post adds a list of handlers to a given route with a POST HTTP verb Post(string, ...macaron.Handler) + + // Delete adds a list of handlers to a given route with a DELETE HTTP verb Delete(string, ...macaron.Handler) + + // Put adds a list of handlers to a given route with a PUT HTTP verb Put(string, ...macaron.Handler) + + // Patch adds a list of handlers to a given route with a PATCH HTTP verb Patch(string, ...macaron.Handler) + + // Any adds a list of handlers to a given route with any HTTP verb Any(string, ...macaron.Handler) + // Group allows you to pass a function that can add multiple routes + // with a shared prefix route. Group(string, func(RouteRegister), ...macaron.Handler) - Register(Router) *macaron.Router + // Insert adds more routes to an existing Group. + Insert(string, func(RouteRegister), ...macaron.Handler) + + // Register iterates over all routes added to the RouteRegister + // and add them to the `Router` pass as an parameter. + Register(Router) } type RegisterNamedMiddleware func(name string) macaron.Handler -func newRouteRegister(namedMiddleware ...RegisterNamedMiddleware) RouteRegister { +// NewRouteRegister creates a new RouteRegister with all middlewares sent as params +func NewRouteRegister(namedMiddleware ...RegisterNamedMiddleware) RouteRegister { return &routeRegister{ prefix: "", routes: []route{}, @@ -49,6 +71,24 @@ type routeRegister struct { groups []*routeRegister } +func (rr *routeRegister) Insert(pattern string, fn func(RouteRegister), handlers ...macaron.Handler) { + + //loop over all groups at current level + for _, g := range rr.groups { + + // apply routes if the prefix matches the pattern + if g.prefix == pattern { + g.Group("", fn) + break + } + + // go down one level if the prefix can be find in the pattern + if strings.HasPrefix(pattern, g.prefix) { + g.Insert(pattern, fn) + } + } +} + func (rr *routeRegister) Group(pattern string, fn func(rr RouteRegister), handlers ...macaron.Handler) { group := &routeRegister{ prefix: rr.prefix + pattern, @@ -61,7 +101,7 @@ func (rr *routeRegister) Group(pattern string, fn func(rr RouteRegister), handle rr.groups = append(rr.groups, group) } -func (rr *routeRegister) Register(router Router) *macaron.Router { +func (rr *routeRegister) Register(router Router) { for _, r := range rr.routes { // GET requests have to be added to macaron routing using Get() // Otherwise HEAD requests will not be allowed. @@ -76,8 +116,6 @@ func (rr *routeRegister) Register(router Router) *macaron.Router { for _, g := range rr.groups { g.Register(router) } - - return &macaron.Router{} } func (rr *routeRegister) route(pattern, method string, handlers ...macaron.Handler) { @@ -89,6 +127,12 @@ func (rr *routeRegister) route(pattern, method string, handlers ...macaron.Handl h = append(h, rr.subfixHandlers...) h = append(h, handlers...) + for _, r := range rr.routes { + if r.pattern == rr.prefix+pattern && r.method == method { + panic("cannot add duplicate route") + } + } + rr.routes = append(rr.routes, route{ method: method, pattern: rr.prefix + pattern, diff --git a/pkg/api/route_register_test.go b/pkg/api/routing/route_register_test.go similarity index 70% rename from pkg/api/route_register_test.go rename to pkg/api/routing/route_register_test.go index f8a043c48df..62e8989ff92 100644 --- a/pkg/api/route_register_test.go +++ b/pkg/api/routing/route_register_test.go @@ -1,11 +1,11 @@ -package api +package routing import ( "net/http" "strconv" "testing" - macaron "gopkg.in/macaron.v1" + "gopkg.in/macaron.v1" ) type fakeRouter struct { @@ -33,7 +33,7 @@ func (fr *fakeRouter) Get(pattern string, handlers ...macaron.Handler) *macaron. } func emptyHandlers(n int) []macaron.Handler { - res := []macaron.Handler{} + var res []macaron.Handler for i := 1; n >= i; i++ { res = append(res, emptyHandler(strconv.Itoa(i))) } @@ -51,7 +51,7 @@ func TestRouteSimpleRegister(t *testing.T) { } // Setup - rr := newRouteRegister(func(name string) macaron.Handler { + rr := NewRouteRegister(func(name string) macaron.Handler { return emptyHandler(name) }) @@ -96,7 +96,7 @@ func TestRouteGroupedRegister(t *testing.T) { } // Setup - rr := newRouteRegister() + rr := NewRouteRegister() rr.Delete("/admin", emptyHandler("1")) rr.Get("/down", emptyHandler("1"), emptyHandler("2")) @@ -138,7 +138,78 @@ func TestRouteGroupedRegister(t *testing.T) { } } } +func TestRouteGroupInserting(t *testing.T) { + testTable := []route{ + {method: http.MethodGet, pattern: "/api/", handlers: emptyHandlers(1)}, + {method: http.MethodPost, pattern: "/api/group/endpoint", handlers: emptyHandlers(1)}, + {method: http.MethodGet, pattern: "/api/group/inserted", handlers: emptyHandlers(1)}, + {method: http.MethodDelete, pattern: "/api/inserted-endpoint", handlers: emptyHandlers(1)}, + } + + // Setup + rr := NewRouteRegister() + + rr.Group("/api", func(api RouteRegister) { + api.Get("/", emptyHandler("1")) + + api.Group("/group", func(group RouteRegister) { + group.Post("/endpoint", emptyHandler("1")) + }) + }) + + rr.Insert("/api", func(api RouteRegister) { + api.Delete("/inserted-endpoint", emptyHandler("1")) + }) + + rr.Insert("/api/group", func(group RouteRegister) { + group.Get("/inserted", emptyHandler("1")) + }) + + fr := &fakeRouter{} + rr.Register(fr) + + // Validation + if len(fr.route) != len(testTable) { + t.Fatalf("want %v routes, got %v", len(testTable), len(fr.route)) + } + + for i := range testTable { + if testTable[i].method != fr.route[i].method { + t.Errorf("want %s got %v", testTable[i].method, fr.route[i].method) + } + + if testTable[i].pattern != fr.route[i].pattern { + t.Errorf("want %s got %v", testTable[i].pattern, fr.route[i].pattern) + } + + if len(testTable[i].handlers) != len(fr.route[i].handlers) { + t.Errorf("want %d handlers got %d handlers \ntestcase: %v\nroute: %v\n", + len(testTable[i].handlers), + len(fr.route[i].handlers), + testTable[i], + fr.route[i]) + } + } +} + +func TestDuplicateRoutShouldPanic(t *testing.T) { + defer func() { + if recover() != "cannot add duplicate route" { + t.Errorf("Should cause panic if duplicate routes are added ") + } + }() + + rr := NewRouteRegister(func(name string) macaron.Handler { + return emptyHandler(name) + }) + + rr.Get("/api", emptyHandler("1")) + rr.Get("/api", emptyHandler("1")) + + fr := &fakeRouter{} + rr.Register(fr) +} func TestNamedMiddlewareRouteRegister(t *testing.T) { testTable := []route{ {method: "DELETE", pattern: "/admin", handlers: emptyHandlers(2)}, @@ -150,7 +221,7 @@ func TestNamedMiddlewareRouteRegister(t *testing.T) { } // Setup - rr := newRouteRegister(func(name string) macaron.Handler { + rr := NewRouteRegister(func(name string) macaron.Handler { return emptyHandler(name) }) diff --git a/pkg/api/search.go b/pkg/api/search.go index c8a0a5592bb..8c2b708d5a2 100644 --- a/pkg/api/search.go +++ b/pkg/api/search.go @@ -25,19 +25,19 @@ func Search(c *m.ReqContext) { permission = m.PERMISSION_EDIT } - dbids := make([]int64, 0) + dbIDs := make([]int64, 0) for _, id := range c.QueryStrings("dashboardIds") { - dashboardId, err := strconv.ParseInt(id, 10, 64) + dashboardID, err := strconv.ParseInt(id, 10, 64) if err == nil { - dbids = append(dbids, dashboardId) + dbIDs = append(dbIDs, dashboardID) } } - folderIds := make([]int64, 0) + folderIDs := make([]int64, 0) for _, id := range c.QueryStrings("folderIds") { - folderId, err := strconv.ParseInt(id, 10, 64) + folderID, err := strconv.ParseInt(id, 10, 64) if err == nil { - folderIds = append(folderIds, folderId) + folderIDs = append(folderIDs, folderID) } } @@ -48,9 +48,9 @@ func Search(c *m.ReqContext) { Limit: limit, IsStarred: starred == "true", OrgId: c.OrgId, - DashboardIds: dbids, + DashboardIds: dbIDs, Type: dashboardType, - FolderIds: folderIds, + FolderIds: folderIDs, Permission: permission, } diff --git a/pkg/api/signup.go b/pkg/api/signup.go index 838d2f9c0af..200a3ebc9d1 100644 --- a/pkg/api/signup.go +++ b/pkg/api/signup.go @@ -12,7 +12,7 @@ import ( // GET /api/user/signup/options func GetSignUpOptions(c *m.ReqContext) Response { - return Json(200, util.DynMap{ + return JSON(200, util.DynMap{ "verifyEmailEnabled": setting.VerifyEmailEnabled, "autoAssignOrg": setting.AutoAssignOrg, }) @@ -21,12 +21,12 @@ func GetSignUpOptions(c *m.ReqContext) Response { // POST /api/user/signup func SignUp(c *m.ReqContext, form dtos.SignUpForm) Response { if !setting.AllowUserSignUp { - return ApiError(401, "User signup is disabled", nil) + return Error(401, "User signup is disabled", nil) } existing := m.GetUserByLoginQuery{LoginOrEmail: form.Email} if err := bus.Dispatch(&existing); err == nil { - return ApiError(422, "User with same email address already exists", nil) + return Error(422, "User with same email address already exists", nil) } cmd := m.CreateTempUserCommand{} @@ -38,7 +38,7 @@ func SignUp(c *m.ReqContext, form dtos.SignUpForm) Response { cmd.RemoteAddr = c.Req.RemoteAddr if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to create signup", err) + return Error(500, "Failed to create signup", err) } bus.Publish(&events.SignUpStarted{ @@ -48,12 +48,12 @@ func SignUp(c *m.ReqContext, form dtos.SignUpForm) Response { metrics.M_Api_User_SignUpStarted.Inc() - return Json(200, util.DynMap{"status": "SignUpCreated"}) + return JSON(200, util.DynMap{"status": "SignUpCreated"}) } func SignUpStep2(c *m.ReqContext, form dtos.SignUpStep2Form) Response { if !setting.AllowUserSignUp { - return ApiError(401, "User signup is disabled", nil) + return Error(401, "User signup is disabled", nil) } createUserCmd := m.CreateUserCommand{ @@ -75,12 +75,12 @@ func SignUpStep2(c *m.ReqContext, form dtos.SignUpStep2Form) Response { // check if user exists existing := m.GetUserByLoginQuery{LoginOrEmail: form.Email} if err := bus.Dispatch(&existing); err == nil { - return ApiError(401, "User with same email address already exists", nil) + return Error(401, "User with same email address already exists", nil) } // dispatch create command if err := bus.Dispatch(&createUserCmd); err != nil { - return ApiError(500, "Failed to create user", err) + return Error(500, "Failed to create user", err) } // publish signup event @@ -98,7 +98,7 @@ func SignUpStep2(c *m.ReqContext, form dtos.SignUpStep2Form) Response { // check for pending invites invitesQuery := m.GetTempUsersQuery{Email: form.Email, Status: m.TmpUserInvitePending} if err := bus.Dispatch(&invitesQuery); err != nil { - return ApiError(500, "Failed to query database for invites", err) + return Error(500, "Failed to query database for invites", err) } apiResponse := util.DynMap{"message": "User sign up completed successfully", "code": "redirect-to-landing-page"} @@ -112,7 +112,7 @@ func SignUpStep2(c *m.ReqContext, form dtos.SignUpStep2Form) Response { loginUserWithUser(user, c) metrics.M_Api_User_SignUpCompleted.Inc() - return Json(200, apiResponse) + return JSON(200, apiResponse) } func verifyUserSignUpEmail(email string, code string) (bool, Response) { @@ -120,14 +120,14 @@ func verifyUserSignUpEmail(email string, code string) (bool, Response) { if err := bus.Dispatch(&query); err != nil { if err == m.ErrTempUserNotFound { - return false, ApiError(404, "Invalid email verification code", nil) + return false, Error(404, "Invalid email verification code", nil) } - return false, ApiError(500, "Failed to read temp user", err) + return false, Error(500, "Failed to read temp user", err) } tempUser := query.Result if tempUser.Email != email { - return false, ApiError(404, "Email verification code does not match email", nil) + return false, Error(404, "Email verification code does not match email", nil) } return true, nil diff --git a/pkg/api/stars.go b/pkg/api/stars.go index 5361f64eea6..2c55b95dfbe 100644 --- a/pkg/api/stars.go +++ b/pkg/api/stars.go @@ -7,20 +7,20 @@ import ( func StarDashboard(c *m.ReqContext) Response { if !c.IsSignedIn { - return ApiError(412, "You need to sign in to star dashboards", nil) + return Error(412, "You need to sign in to star dashboards", nil) } cmd := m.StarDashboardCommand{UserId: c.UserId, DashboardId: c.ParamsInt64(":id")} if cmd.DashboardId <= 0 { - return ApiError(400, "Missing dashboard id", nil) + return Error(400, "Missing dashboard id", nil) } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to star dashboard", err) + return Error(500, "Failed to star dashboard", err) } - return ApiSuccess("Dashboard starred!") + return Success("Dashboard starred!") } func UnstarDashboard(c *m.ReqContext) Response { @@ -28,12 +28,12 @@ func UnstarDashboard(c *m.ReqContext) Response { cmd := m.UnstarDashboardCommand{UserId: c.UserId, DashboardId: c.ParamsInt64(":id")} if cmd.DashboardId <= 0 { - return ApiError(400, "Missing dashboard id", nil) + return Error(400, "Missing dashboard id", nil) } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to unstar dashboard", err) + return Error(500, "Failed to unstar dashboard", err) } - return ApiSuccess("Dashboard unstarred") + return Success("Dashboard unstarred") } diff --git a/pkg/api/static/static.go b/pkg/api/static/static.go index 7a61c85b4f3..2a35dd11fa6 100644 --- a/pkg/api/static/static.go +++ b/pkg/api/static/static.go @@ -48,7 +48,7 @@ type StaticOptions struct { // Expires defines which user-defined function to use for producing a HTTP Expires Header // https://developers.google.com/speed/docs/insights/LeverageBrowserCaching AddHeaders func(ctx *macaron.Context) - // FileSystem is the interface for supporting any implmentation of file system. + // FileSystem is the interface for supporting any implementation of file system. FileSystem http.FileSystem } diff --git a/pkg/api/team.go b/pkg/api/team.go index 316adfc4e7c..32265e5d018 100644 --- a/pkg/api/team.go +++ b/pkg/api/team.go @@ -12,12 +12,12 @@ func CreateTeam(c *m.ReqContext, cmd m.CreateTeamCommand) Response { cmd.OrgId = c.OrgId if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrTeamNameTaken { - return ApiError(409, "Team name taken", err) + return Error(409, "Team name taken", err) } - return ApiError(500, "Failed to create Team", err) + return Error(500, "Failed to create Team", err) } - return Json(200, &util.DynMap{ + return JSON(200, &util.DynMap{ "teamId": cmd.Result.Id, "message": "Team created", }) @@ -29,23 +29,23 @@ func UpdateTeam(c *m.ReqContext, cmd m.UpdateTeamCommand) Response { cmd.Id = c.ParamsInt64(":teamId") if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrTeamNameTaken { - return ApiError(400, "Team name taken", err) + return Error(400, "Team name taken", err) } - return ApiError(500, "Failed to update Team", err) + return Error(500, "Failed to update Team", err) } - return ApiSuccess("Team updated") + return Success("Team updated") } // DELETE /api/teams/:teamId -func DeleteTeamById(c *m.ReqContext) Response { +func DeleteTeamByID(c *m.ReqContext) Response { if err := bus.Dispatch(&m.DeleteTeamCommand{OrgId: c.OrgId, Id: c.ParamsInt64(":teamId")}); err != nil { if err == m.ErrTeamNotFound { - return ApiError(404, "Failed to delete Team. ID not found", nil) + return Error(404, "Failed to delete Team. ID not found", nil) } - return ApiError(500, "Failed to update Team", err) + return Error(500, "Failed to update Team", err) } - return ApiSuccess("Team deleted") + return Success("Team deleted") } // GET /api/teams/search @@ -68,7 +68,7 @@ func SearchTeams(c *m.ReqContext) Response { } if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to search Teams", err) + return Error(500, "Failed to search Teams", err) } for _, team := range query.Result.Teams { @@ -78,20 +78,31 @@ func SearchTeams(c *m.ReqContext) Response { query.Result.Page = page query.Result.PerPage = perPage - return Json(200, query.Result) + return JSON(200, query.Result) } // GET /api/teams/:teamId -func GetTeamById(c *m.ReqContext) Response { +func GetTeamByID(c *m.ReqContext) Response { query := m.GetTeamByIdQuery{OrgId: c.OrgId, Id: c.ParamsInt64(":teamId")} if err := bus.Dispatch(&query); err != nil { if err == m.ErrTeamNotFound { - return ApiError(404, "Team not found", err) + return Error(404, "Team not found", err) } - return ApiError(500, "Failed to get Team", err) + return Error(500, "Failed to get Team", err) } - return Json(200, &query.Result) + query.Result.AvatarUrl = dtos.GetGravatarUrlWithDefault(query.Result.Email, query.Result.Name) + return JSON(200, &query.Result) +} + +// GET /api/teams/:teamId/preferences +func GetTeamPreferences(c *m.ReqContext) Response { + return getPreferencesFor(c.OrgId, 0, c.ParamsInt64(":teamId")) +} + +// PUT /api/teams/:teamId/preferences +func UpdateTeamPreferences(c *m.ReqContext, dtoCmd dtos.UpdatePrefsCmd) Response { + return updatePreferencesFor(c.OrgId, 0, c.ParamsInt64(":teamId"), &dtoCmd) } diff --git a/pkg/api/team_members.go b/pkg/api/team_members.go index 4fb05b016e3..5b5970de6ad 100644 --- a/pkg/api/team_members.go +++ b/pkg/api/team_members.go @@ -4,6 +4,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -12,14 +13,19 @@ func GetTeamMembers(c *m.ReqContext) Response { query := m.GetTeamMembersQuery{OrgId: c.OrgId, TeamId: c.ParamsInt64(":teamId")} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to get Team Members", err) + return Error(500, "Failed to get Team Members", err) } for _, member := range query.Result { member.AvatarUrl = dtos.GetGravatarUrl(member.Email) + member.Labels = []string{} + + if setting.IsEnterprise && setting.LdapEnabled && member.External { + member.Labels = append(member.Labels, "LDAP") + } } - return Json(200, query.Result) + return JSON(200, query.Result) } // POST /api/teams/:teamId/members @@ -29,17 +35,17 @@ func AddTeamMember(c *m.ReqContext, cmd m.AddTeamMemberCommand) Response { if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrTeamNotFound { - return ApiError(404, "Team not found", nil) + return Error(404, "Team not found", nil) } if err == m.ErrTeamMemberAlreadyAdded { - return ApiError(400, "User is already added to this team", nil) + return Error(400, "User is already added to this team", nil) } - return ApiError(500, "Failed to add Member to Team", err) + return Error(500, "Failed to add Member to Team", err) } - return Json(200, &util.DynMap{ + return JSON(200, &util.DynMap{ "message": "Member added to Team", }) } @@ -48,14 +54,14 @@ func AddTeamMember(c *m.ReqContext, cmd m.AddTeamMemberCommand) Response { func RemoveTeamMember(c *m.ReqContext) Response { if err := bus.Dispatch(&m.RemoveTeamMemberCommand{OrgId: c.OrgId, TeamId: c.ParamsInt64(":teamId"), UserId: c.ParamsInt64(":userId")}); err != nil { if err == m.ErrTeamNotFound { - return ApiError(404, "Team not found", nil) + return Error(404, "Team not found", nil) } if err == m.ErrTeamMemberNotFound { - return ApiError(404, "Team member not found", nil) + return Error(404, "Team member not found", nil) } - return ApiError(500, "Failed to remove Member from Team", err) + return Error(500, "Failed to remove Member from Team", err) } - return ApiSuccess("Team Member removed") + return Success("Team Member removed") } diff --git a/pkg/api/team_test.go b/pkg/api/team_test.go index 0bf06d723c8..a1984288870 100644 --- a/pkg/api/team_test.go +++ b/pkg/api/team_test.go @@ -13,7 +13,7 @@ import ( func TestTeamApiEndpoint(t *testing.T) { Convey("Given two teams", t, func() { mockResult := models.SearchTeamQueryResult{ - Teams: []*models.SearchTeamDto{ + Teams: []*models.TeamDTO{ {Name: "team1"}, {Name: "team2"}, }, diff --git a/pkg/api/user.go b/pkg/api/user.go index b8483316b9d..7116ad83f3f 100644 --- a/pkg/api/user.go +++ b/pkg/api/user.go @@ -14,21 +14,21 @@ func GetSignedInUser(c *m.ReqContext) Response { } // GET /api/users/:id -func GetUserById(c *m.ReqContext) Response { +func GetUserByID(c *m.ReqContext) Response { return getUserUserProfile(c.ParamsInt64(":id")) } -func getUserUserProfile(userId int64) Response { - query := m.GetUserProfileQuery{UserId: userId} +func getUserUserProfile(userID int64) Response { + query := m.GetUserProfileQuery{UserId: userID} if err := bus.Dispatch(&query); err != nil { if err == m.ErrUserNotFound { - return ApiError(404, m.ErrUserNotFound.Error(), nil) + return Error(404, m.ErrUserNotFound.Error(), nil) } - return ApiError(500, "Failed to get user", err) + return Error(500, "Failed to get user", err) } - return Json(200, query.Result) + return JSON(200, query.Result) } // GET /api/users/lookup @@ -36,9 +36,9 @@ func GetUserByLoginOrEmail(c *m.ReqContext) Response { query := m.GetUserByLoginQuery{LoginOrEmail: c.Query("loginOrEmail")} if err := bus.Dispatch(&query); err != nil { if err == m.ErrUserNotFound { - return ApiError(404, m.ErrUserNotFound.Error(), nil) + return Error(404, m.ErrUserNotFound.Error(), nil) } - return ApiError(500, "Failed to get user", err) + return Error(500, "Failed to get user", err) } user := query.Result result := m.UserProfileDTO{ @@ -50,17 +50,17 @@ func GetUserByLoginOrEmail(c *m.ReqContext) Response { IsGrafanaAdmin: user.IsAdmin, OrgId: user.OrgId, } - return Json(200, &result) + return JSON(200, &result) } // POST /api/user func UpdateSignedInUser(c *m.ReqContext, cmd m.UpdateUserCommand) Response { if setting.AuthProxyEnabled { if setting.AuthProxyHeaderProperty == "email" && cmd.Email != c.Email { - return ApiError(400, "Not allowed to change email when auth proxy is using email property", nil) + return Error(400, "Not allowed to change email when auth proxy is using email property", nil) } if setting.AuthProxyHeaderProperty == "username" && cmd.Login != c.Login { - return ApiError(400, "Not allowed to change username when auth proxy is using username property", nil) + return Error(400, "Not allowed to change username when auth proxy is using username property", nil) } } cmd.UserId = c.UserId @@ -75,35 +75,35 @@ func UpdateUser(c *m.ReqContext, cmd m.UpdateUserCommand) Response { //POST /api/users/:id/using/:orgId func UpdateUserActiveOrg(c *m.ReqContext) Response { - userId := c.ParamsInt64(":id") - orgId := c.ParamsInt64(":orgId") + userID := c.ParamsInt64(":id") + orgID := c.ParamsInt64(":orgId") - if !validateUsingOrg(userId, orgId) { - return ApiError(401, "Not a valid organization", nil) + if !validateUsingOrg(userID, orgID) { + return Error(401, "Not a valid organization", nil) } - cmd := m.SetUsingOrgCommand{UserId: userId, OrgId: orgId} + cmd := m.SetUsingOrgCommand{UserId: userID, OrgId: orgID} if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to change active organization", err) + return Error(500, "Failed to change active organization", err) } - return ApiSuccess("Active organization changed") + return Success("Active organization changed") } func handleUpdateUser(cmd m.UpdateUserCommand) Response { if len(cmd.Login) == 0 { cmd.Login = cmd.Email if len(cmd.Login) == 0 { - return ApiError(400, "Validation error, need to specify either username or email", nil) + return Error(400, "Validation error, need to specify either username or email", nil) } } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update user", err) + return Error(500, "Failed to update user", err) } - return ApiSuccess("User updated") + return Success("User updated") } // GET /api/user/orgs @@ -111,23 +111,38 @@ func GetSignedInUserOrgList(c *m.ReqContext) Response { return getUserOrgList(c.UserId) } +// GET /api/user/teams +func GetSignedInUserTeamList(c *m.ReqContext) Response { + query := m.GetTeamsByUserQuery{OrgId: c.OrgId, UserId: c.UserId} + + if err := bus.Dispatch(&query); err != nil { + return Error(500, "Failed to get user teams", err) + } + + for _, team := range query.Result { + team.AvatarUrl = dtos.GetGravatarUrlWithDefault(team.Email, team.Name) + } + + return JSON(200, query.Result) +} + // GET /api/user/:id/orgs func GetUserOrgList(c *m.ReqContext) Response { return getUserOrgList(c.ParamsInt64(":id")) } -func getUserOrgList(userId int64) Response { - query := m.GetUserOrgListQuery{UserId: userId} +func getUserOrgList(userID int64) Response { + query := m.GetUserOrgListQuery{UserId: userID} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to get user organizations", err) + return Error(500, "Failed to get user organizations", err) } - return Json(200, query.Result) + return JSON(200, query.Result) } -func validateUsingOrg(userId int64, orgId int64) bool { - query := m.GetUserOrgListQuery{UserId: userId} +func validateUsingOrg(userID int64, orgID int64) bool { + query := m.GetUserOrgListQuery{UserId: userID} if err := bus.Dispatch(&query); err != nil { return false @@ -136,7 +151,7 @@ func validateUsingOrg(userId int64, orgId int64) bool { // validate that the org id in the list valid := false for _, other := range query.Result { - if other.OrgId == orgId { + if other.OrgId == orgID { valid = true } } @@ -146,33 +161,33 @@ func validateUsingOrg(userId int64, orgId int64) bool { // POST /api/user/using/:id func UserSetUsingOrg(c *m.ReqContext) Response { - orgId := c.ParamsInt64(":id") + orgID := c.ParamsInt64(":id") - if !validateUsingOrg(c.UserId, orgId) { - return ApiError(401, "Not a valid organization", nil) + if !validateUsingOrg(c.UserId, orgID) { + return Error(401, "Not a valid organization", nil) } - cmd := m.SetUsingOrgCommand{UserId: c.UserId, OrgId: orgId} + cmd := m.SetUsingOrgCommand{UserId: c.UserId, OrgId: orgID} if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to change active organization", err) + return Error(500, "Failed to change active organization", err) } - return ApiSuccess("Active organization changed") + return Success("Active organization changed") } // GET /profile/switch-org/:id -func ChangeActiveOrgAndRedirectToHome(c *m.ReqContext) { - orgId := c.ParamsInt64(":id") +func (hs *HTTPServer) ChangeActiveOrgAndRedirectToHome(c *m.ReqContext) { + orgID := c.ParamsInt64(":id") - if !validateUsingOrg(c.UserId, orgId) { - NotFoundHandler(c) + if !validateUsingOrg(c.UserId, orgID) { + hs.NotFoundHandler(c) } - cmd := m.SetUsingOrgCommand{UserId: c.UserId, OrgId: orgId} + cmd := m.SetUsingOrgCommand{UserId: c.UserId, OrgId: orgID} if err := bus.Dispatch(&cmd); err != nil { - NotFoundHandler(c) + hs.NotFoundHandler(c) } c.Redirect(setting.AppSubUrl + "/") @@ -180,53 +195,53 @@ func ChangeActiveOrgAndRedirectToHome(c *m.ReqContext) { func ChangeUserPassword(c *m.ReqContext, cmd m.ChangeUserPasswordCommand) Response { if setting.LdapEnabled || setting.AuthProxyEnabled { - return ApiError(400, "Not allowed to change password when LDAP or Auth Proxy is enabled", nil) + return Error(400, "Not allowed to change password when LDAP or Auth Proxy is enabled", nil) } userQuery := m.GetUserByIdQuery{Id: c.UserId} if err := bus.Dispatch(&userQuery); err != nil { - return ApiError(500, "Could not read user from database", err) + return Error(500, "Could not read user from database", err) } passwordHashed := util.EncodePassword(cmd.OldPassword, userQuery.Result.Salt) if passwordHashed != userQuery.Result.Password { - return ApiError(401, "Invalid old password", nil) + return Error(401, "Invalid old password", nil) } password := m.Password(cmd.NewPassword) if password.IsWeak() { - return ApiError(400, "New password is too short", nil) + return Error(400, "New password is too short", nil) } cmd.UserId = c.UserId cmd.NewPassword = util.EncodePassword(cmd.NewPassword, userQuery.Result.Salt) if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to change user password", err) + return Error(500, "Failed to change user password", err) } - return ApiSuccess("User password changed") + return Success("User password changed") } // GET /api/users func SearchUsers(c *m.ReqContext) Response { query, err := searchUser(c) if err != nil { - return ApiError(500, "Failed to fetch users", err) + return Error(500, "Failed to fetch users", err) } - return Json(200, query.Result.Users) + return JSON(200, query.Result.Users) } // GET /api/users/search func SearchUsersWithPaging(c *m.ReqContext) Response { query, err := searchUser(c) if err != nil { - return ApiError(500, "Failed to fetch users", err) + return Error(500, "Failed to fetch users", err) } - return Json(200, query.Result) + return JSON(200, query.Result) } func searchUser(c *m.ReqContext) (*m.SearchUsersQuery, error) { @@ -269,10 +284,10 @@ func SetHelpFlag(c *m.ReqContext) Response { } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update help flag", err) + return Error(500, "Failed to update help flag", err) } - return Json(200, &util.DynMap{"message": "Help flag set", "helpFlags1": cmd.HelpFlags1}) + return JSON(200, &util.DynMap{"message": "Help flag set", "helpFlags1": cmd.HelpFlags1}) } func ClearHelpFlags(c *m.ReqContext) Response { @@ -282,8 +297,8 @@ func ClearHelpFlags(c *m.ReqContext) Response { } if err := bus.Dispatch(&cmd); err != nil { - return ApiError(500, "Failed to update help flag", err) + return Error(500, "Failed to update help flag", err) } - return Json(200, &util.DynMap{"message": "Help flag set", "helpFlags1": cmd.HelpFlags1}) + return JSON(200, &util.DynMap{"message": "Help flag set", "helpFlags1": cmd.HelpFlags1}) } diff --git a/pkg/bus/bus.go b/pkg/bus/bus.go index 59d4592766e..9cf930aeb82 100644 --- a/pkg/bus/bus.go +++ b/pkg/bus/bus.go @@ -2,7 +2,7 @@ package bus import ( "context" - "fmt" + "errors" "reflect" ) @@ -10,21 +10,44 @@ type HandlerFunc interface{} type CtxHandlerFunc func() type Msg interface{} +var ErrHandlerNotFound = errors.New("handler not found") + +type TransactionManager interface { + InTransaction(ctx context.Context, fn func(ctx context.Context) error) error +} + type Bus interface { Dispatch(msg Msg) error DispatchCtx(ctx context.Context, msg Msg) error Publish(msg Msg) error + // InTransaction starts a transaction and store it in the context. + // The caller can then pass a function with multiple DispatchCtx calls that + // all will be executed in the same transaction. InTransaction will rollback if the + // callback returns an error. + InTransaction(ctx context.Context, fn func(ctx context.Context) error) error + AddHandler(handler HandlerFunc) - AddCtxHandler(handler HandlerFunc) + AddHandlerCtx(handler HandlerFunc) AddEventListener(handler HandlerFunc) AddWildcardListener(handler HandlerFunc) + + // SetTransactionManager allows the user to replace the internal + // noop TransactionManager that is responsible for manageing + // transactions in `InTransaction` + SetTransactionManager(tm TransactionManager) +} + +func (b *InProcBus) InTransaction(ctx context.Context, fn func(ctx context.Context) error) error { + return b.txMng.InTransaction(ctx, fn) } type InProcBus struct { handlers map[string]HandlerFunc + handlersWithCtx map[string]HandlerFunc listeners map[string][]HandlerFunc wildcardListeners []HandlerFunc + txMng TransactionManager } // temp stuff, not sure how to handle bus instance, and init yet @@ -33,50 +56,70 @@ var globalBus = New() func New() Bus { bus := &InProcBus{} bus.handlers = make(map[string]HandlerFunc) + bus.handlersWithCtx = make(map[string]HandlerFunc) bus.listeners = make(map[string][]HandlerFunc) bus.wildcardListeners = make([]HandlerFunc, 0) + bus.txMng = &noopTransactionManager{} + return bus } +// Want to get rid of global bus +func GetBus() Bus { + return globalBus +} + +func (b *InProcBus) SetTransactionManager(tm TransactionManager) { + b.txMng = tm +} + func (b *InProcBus) DispatchCtx(ctx context.Context, msg Msg) error { var msgName = reflect.TypeOf(msg).Elem().Name() - var handler = b.handlers[msgName] + var handler = b.handlersWithCtx[msgName] if handler == nil { - return fmt.Errorf("handler not found for %s", msgName) + return ErrHandlerNotFound } - var params = make([]reflect.Value, 2) - params[0] = reflect.ValueOf(ctx) - params[1] = reflect.ValueOf(msg) + var params = []reflect.Value{} + params = append(params, reflect.ValueOf(ctx)) + params = append(params, reflect.ValueOf(msg)) ret := reflect.ValueOf(handler).Call(params) err := ret[0].Interface() if err == nil { return nil - } else { - return err.(error) } + return err.(error) } func (b *InProcBus) Dispatch(msg Msg) error { var msgName = reflect.TypeOf(msg).Elem().Name() - var handler = b.handlers[msgName] + var handler = b.handlersWithCtx[msgName] + withCtx := true + if handler == nil { - return fmt.Errorf("handler not found for %s", msgName) + withCtx = false + handler = b.handlers[msgName] } - var params = make([]reflect.Value, 1) - params[0] = reflect.ValueOf(msg) + if handler == nil { + return ErrHandlerNotFound + } + + var params = []reflect.Value{} + if withCtx { + params = append(params, reflect.ValueOf(context.Background())) + } + params = append(params, reflect.ValueOf(msg)) ret := reflect.ValueOf(handler).Call(params) err := ret[0].Interface() if err == nil { return nil - } else { - return err.(error) } + return err.(error) } func (b *InProcBus) Publish(msg Msg) error { @@ -115,10 +158,10 @@ func (b *InProcBus) AddHandler(handler HandlerFunc) { b.handlers[queryTypeName] = handler } -func (b *InProcBus) AddCtxHandler(handler HandlerFunc) { +func (b *InProcBus) AddHandlerCtx(handler HandlerFunc) { handlerType := reflect.TypeOf(handler) queryTypeName := handlerType.In(1).Elem().Name() - b.handlers[queryTypeName] = handler + b.handlersWithCtx[queryTypeName] = handler } func (b *InProcBus) AddEventListener(handler HandlerFunc) { @@ -137,8 +180,8 @@ func AddHandler(implName string, handler HandlerFunc) { } // Package level functions -func AddCtxHandler(implName string, handler HandlerFunc) { - globalBus.AddCtxHandler(handler) +func AddHandlerCtx(implName string, handler HandlerFunc) { + globalBus.AddHandlerCtx(handler) } // Package level functions @@ -162,6 +205,20 @@ func Publish(msg Msg) error { return globalBus.Publish(msg) } +// InTransaction starts a transaction and store it in the context. +// The caller can then pass a function with multiple DispatchCtx calls that +// all will be executed in the same transaction. InTransaction will rollback if the +// callback returns an error. +func InTransaction(ctx context.Context, fn func(ctx context.Context) error) error { + return globalBus.InTransaction(ctx, fn) +} + func ClearBusHandlers() { globalBus = New() } + +type noopTransactionManager struct{} + +func (*noopTransactionManager) InTransaction(ctx context.Context, fn func(ctx context.Context) error) error { + return fn(ctx) +} diff --git a/pkg/bus/bus_test.go b/pkg/bus/bus_test.go index 62e72f18308..9f41a5154df 100644 --- a/pkg/bus/bus_test.go +++ b/pkg/bus/bus_test.go @@ -1,24 +1,67 @@ package bus import ( + "context" "errors" "fmt" "testing" ) -type TestQuery struct { +type testQuery struct { Id int64 Resp string } +func TestDispatchCtxCanUseNormalHandlers(t *testing.T) { + bus := New() + + handlerWithCtxCallCount := 0 + handlerCallCount := 0 + + handlerWithCtx := func(ctx context.Context, query *testQuery) error { + handlerWithCtxCallCount++ + return nil + } + + handler := func(query *testQuery) error { + handlerCallCount++ + return nil + } + + err := bus.DispatchCtx(context.Background(), &testQuery{}) + if err != ErrHandlerNotFound { + t.Errorf("expected bus to return HandlerNotFound is no handler is registered") + } + + bus.AddHandler(handler) + + t.Run("when a normal handler is registered", func(t *testing.T) { + bus.Dispatch(&testQuery{}) + + if handlerCallCount != 1 { + t.Errorf("Expected normal handler to be called 1 time. was called %d", handlerCallCount) + } + + t.Run("when a ctx handler is registered", func(t *testing.T) { + bus.AddHandlerCtx(handlerWithCtx) + bus.Dispatch(&testQuery{}) + + if handlerWithCtxCallCount != 1 { + t.Errorf("Expected ctx handler to be called 1 time. was called %d", handlerWithCtxCallCount) + } + }) + }) + +} + func TestQueryHandlerReturnsError(t *testing.T) { bus := New() - bus.AddHandler(func(query *TestQuery) error { + bus.AddHandler(func(query *testQuery) error { return errors.New("handler error") }) - err := bus.Dispatch(&TestQuery{}) + err := bus.Dispatch(&testQuery{}) if err == nil { t.Fatal("Send query failed " + err.Error()) @@ -30,12 +73,12 @@ func TestQueryHandlerReturnsError(t *testing.T) { func TestQueryHandlerReturn(t *testing.T) { bus := New() - bus.AddHandler(func(q *TestQuery) error { + bus.AddHandler(func(q *testQuery) error { q.Resp = "hello from handler" return nil }) - query := &TestQuery{} + query := &testQuery{} err := bus.Dispatch(query) if err != nil { @@ -49,17 +92,17 @@ func TestEventListeners(t *testing.T) { bus := New() count := 0 - bus.AddEventListener(func(query *TestQuery) error { + bus.AddEventListener(func(query *testQuery) error { count += 1 return nil }) - bus.AddEventListener(func(query *TestQuery) error { + bus.AddEventListener(func(query *testQuery) error { count += 10 return nil }) - err := bus.Publish(&TestQuery{}) + err := bus.Publish(&testQuery{}) if err != nil { t.Fatal("Publish event failed " + err.Error()) diff --git a/pkg/cmd/grafana-cli/commands/commands.go b/pkg/cmd/grafana-cli/commands/commands.go index d8f01bbdcab..902fd415977 100644 --- a/pkg/cmd/grafana-cli/commands/commands.go +++ b/pkg/cmd/grafana-cli/commands/commands.go @@ -6,6 +6,7 @@ import ( "github.com/codegangsta/cli" "github.com/fatih/color" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" @@ -15,13 +16,17 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli return func(context *cli.Context) { cmd := &contextCommandLine{context} - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ Config: cmd.String("config"), HomePath: cmd.String("homepath"), Args: flag.Args(), }) - sqlstore.NewEngine() + engine := &sqlstore.SqlStore{} + engine.Cfg = cfg + engine.Bus = bus.GetBus() + engine.Init() if err := command(cmd); err != nil { logger.Errorf("\n%s: ", color.RedString("Error")) diff --git a/pkg/cmd/grafana-cli/commands/install_command.go b/pkg/cmd/grafana-cli/commands/install_command.go index f40bc9c081b..f88bb9bbfff 100644 --- a/pkg/cmd/grafana-cli/commands/install_command.go +++ b/pkg/cmd/grafana-cli/commands/install_command.go @@ -33,7 +33,7 @@ func validateInput(c CommandLine, pluginFolder string) error { fileInfo, err := os.Stat(pluginsDir) if err != nil { if err = os.MkdirAll(pluginsDir, os.ModePerm); err != nil { - return errors.New(fmt.Sprintf("pluginsDir (%s) is not a directory", pluginsDir)) + return fmt.Errorf("pluginsDir (%s) is not a writable directory", pluginsDir) } return nil } @@ -112,7 +112,7 @@ func SelectVersion(plugin m.Plugin, version string) (m.Version, error) { } } - return m.Version{}, errors.New("Could not find the version your looking for") + return m.Version{}, errors.New("Could not find the version you're looking for") } func RemoveGitBuildFromName(pluginName, filename string) string { @@ -152,7 +152,7 @@ func downloadFile(pluginName, filePath, url string) (err error) { return err } - r, err := zip.NewReader(bytes.NewReader(body), resp.ContentLength) + r, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) if err != nil { return err } diff --git a/pkg/cmd/grafana-cli/commands/ls_command.go b/pkg/cmd/grafana-cli/commands/ls_command.go index 7dcecb9d725..30745ce3172 100644 --- a/pkg/cmd/grafana-cli/commands/ls_command.go +++ b/pkg/cmd/grafana-cli/commands/ls_command.go @@ -24,7 +24,7 @@ var validateLsCommand = func(pluginDir string) error { return fmt.Errorf("error: %s", err) } - if pluginDirInfo.IsDir() == false { + if !pluginDirInfo.IsDir() { return errors.New("plugin path is not a directory") } diff --git a/pkg/cmd/grafana-cli/commands/remove_command.go b/pkg/cmd/grafana-cli/commands/remove_command.go index d5ed73def05..e51929dc95c 100644 --- a/pkg/cmd/grafana-cli/commands/remove_command.go +++ b/pkg/cmd/grafana-cli/commands/remove_command.go @@ -3,12 +3,11 @@ package commands import ( "errors" "fmt" - m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" - services "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" "strings" + + services "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" ) -var getPluginss func(path string) []m.InstalledPlugin = services.GetLocalPlugins var removePlugin func(pluginPath, id string) error = services.RemoveInstalledPlugin func removeCommand(c CommandLine) error { diff --git a/pkg/cmd/grafana-cli/commands/upgrade_all_command.go b/pkg/cmd/grafana-cli/commands/upgrade_all_command.go index 636292cce11..e01df2dab60 100644 --- a/pkg/cmd/grafana-cli/commands/upgrade_all_command.go +++ b/pkg/cmd/grafana-cli/commands/upgrade_all_command.go @@ -53,8 +53,7 @@ func upgradeAllCommand(c CommandLine) error { for _, p := range pluginsToUpgrade { logger.Infof("Updating %v \n", p.Id) - var err error - err = s.RemoveInstalledPlugin(pluginsDir, p.Id) + err := s.RemoveInstalledPlugin(pluginsDir, p.Id) if err != nil { return err } diff --git a/pkg/cmd/grafana-cli/commands/upgrade_command.go b/pkg/cmd/grafana-cli/commands/upgrade_command.go index 355ccab3d1c..396371d3577 100644 --- a/pkg/cmd/grafana-cli/commands/upgrade_command.go +++ b/pkg/cmd/grafana-cli/commands/upgrade_command.go @@ -16,7 +16,7 @@ func upgradeCommand(c CommandLine) error { return err } - v, err2 := s.GetPlugin(localPlugin.Id, c.RepoDirectory()) + v, err2 := s.GetPlugin(pluginName, c.RepoDirectory()) if err2 != nil { return err2 @@ -24,9 +24,9 @@ func upgradeCommand(c CommandLine) error { if ShouldUpgrade(localPlugin.Info.Version, v) { s.RemoveInstalledPlugin(pluginsDir, pluginName) - return InstallPlugin(localPlugin.Id, "", c) + return InstallPlugin(pluginName, "", c) } - logger.Infof("%s %s is up to date \n", color.GreenString("✔"), localPlugin.Id) + logger.Infof("%s %s is up to date \n", color.GreenString("✔"), pluginName) return nil } diff --git a/pkg/cmd/grafana-cli/services/services.go b/pkg/cmd/grafana-cli/services/services.go index d13e90d6a2f..338975bc130 100644 --- a/pkg/cmd/grafana-cli/services/services.go +++ b/pkg/cmd/grafana-cli/services/services.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "path" + "runtime" "time" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" @@ -42,7 +43,7 @@ func Init(version string, skipTLSVerify bool) { } HttpClient = http.Client{ - Timeout: time.Duration(10 * time.Second), + Timeout: 10 * time.Second, Transport: tr, } } @@ -62,7 +63,7 @@ func ListAllPlugins(repoUrl string) (m.PluginRepo, error) { var data m.PluginRepo err = json.Unmarshal(body, &data) if err != nil { - logger.Info("Failed to unmarshal graphite response error: %v", err) + logger.Info("Failed to unmarshal plugin repo response error:", err) return m.PluginRepo{}, err } @@ -139,7 +140,7 @@ func GetPlugin(pluginId, repoUrl string) (m.Plugin, error) { var data m.Plugin err = json.Unmarshal(body, &data) if err != nil { - logger.Info("Failed to unmarshal graphite response error: %v", err) + logger.Info("Failed to unmarshal plugin repo response error:", err) return m.Plugin{}, err } @@ -155,6 +156,8 @@ func sendRequest(repoUrl string, subPaths ...string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, u.String(), nil) req.Header.Set("grafana-version", grafanaVersion) + req.Header.Set("grafana-os", runtime.GOOS) + req.Header.Set("grafana-arch", runtime.GOARCH) req.Header.Set("User-Agent", "grafana "+grafanaVersion) if err != nil { diff --git a/pkg/cmd/grafana-cli/utils/grafana_path.go b/pkg/cmd/grafana-cli/utils/grafana_path.go index afb622bbb93..5f5c944f52b 100644 --- a/pkg/cmd/grafana-cli/utils/grafana_path.go +++ b/pkg/cmd/grafana-cli/utils/grafana_path.go @@ -42,6 +42,8 @@ func returnOsDefault(currentOs string) string { return "/usr/local/var/lib/grafana/plugins" case "freebsd": return "/var/db/grafana/plugins" + case "openbsd": + return "/var/grafana/plugins" default: //"linux" return "/var/lib/grafana/plugins" } diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index ab0e12f2d9f..c7c1ff3aff7 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -3,6 +3,8 @@ package main import ( "flag" "fmt" + "net/http" + _ "net/http/pprof" "os" "os/signal" "runtime" @@ -11,34 +13,32 @@ import ( "syscall" "time" - "net/http" - _ "net/http/pprof" - + extensions "github.com/grafana/grafana/pkg/extensions" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" - "github.com/grafana/grafana/pkg/setting" - _ "github.com/grafana/grafana/pkg/services/alerting/conditions" _ "github.com/grafana/grafana/pkg/services/alerting/notifiers" + "github.com/grafana/grafana/pkg/setting" _ "github.com/grafana/grafana/pkg/tsdb/cloudwatch" + _ "github.com/grafana/grafana/pkg/tsdb/elasticsearch" _ "github.com/grafana/grafana/pkg/tsdb/graphite" _ "github.com/grafana/grafana/pkg/tsdb/influxdb" _ "github.com/grafana/grafana/pkg/tsdb/mysql" _ "github.com/grafana/grafana/pkg/tsdb/opentsdb" _ "github.com/grafana/grafana/pkg/tsdb/postgres" _ "github.com/grafana/grafana/pkg/tsdb/prometheus" + _ "github.com/grafana/grafana/pkg/tsdb/stackdriver" _ "github.com/grafana/grafana/pkg/tsdb/testdata" ) var version = "5.0.0" var commit = "NA" +var buildBranch = "master" var buildstamp string -var build_date string var configFile = flag.String("config", "", "path to config file") var homePath = flag.String("homepath", "", "path to grafana install/home path, defaults to working directory") var pidFile = flag.String("pidfile", "", "path to pid file") -var exitChan = make(chan int) func main() { v := flag.Bool("v", false, "prints current version and exits") @@ -46,7 +46,7 @@ func main() { profilePort := flag.Int("profile-port", 6060, "Define custom port for profiling") flag.Parse() if *v { - fmt.Printf("Version %s (commit: %s)\n", version, commit) + fmt.Printf("Version %s (commit: %s, branch: %s)\n", version, commit, buildBranch) os.Exit(0) } @@ -77,45 +77,37 @@ func main() { setting.BuildVersion = version setting.BuildCommit = commit setting.BuildStamp = buildstampInt64 + setting.BuildBranch = buildBranch + setting.IsEnterprise = extensions.IsEnterprise + + metrics.SetBuildInformation(version, commit, buildBranch) - metrics.M_Grafana_Version.WithLabelValues(version).Set(1) - shutdownCompleted := make(chan int) server := NewGrafanaServer() - go listenToSystemSignals(server, shutdownCompleted) + go listenToSystemSignals(server) - go func() { - code := 0 - if err := server.Start(); err != nil { - log.Error2("Startup failed", "error", err) - code = 1 - } + err := server.Run() - exitChan <- code - }() - - code := <-shutdownCompleted - log.Info2("Grafana shutdown completed.", "code", code) + code := server.Exit(err) + trace.Stop() log.Close() + os.Exit(code) } -func listenToSystemSignals(server *GrafanaServerImpl, shutdownCompleted chan int) { +func listenToSystemSignals(server *GrafanaServerImpl) { signalChan := make(chan os.Signal, 1) - ignoreChan := make(chan os.Signal, 1) - code := 0 + sighupChan := make(chan os.Signal, 1) - signal.Notify(ignoreChan, syscall.SIGHUP) - signal.Notify(signalChan, os.Interrupt, os.Kill, syscall.SIGTERM) + signal.Notify(sighupChan, syscall.SIGHUP) + signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM) - select { - case sig := <-signalChan: - trace.Stop() // Stops trace if profiling has been enabled - server.Shutdown(0, fmt.Sprintf("system signal: %s", sig)) - shutdownCompleted <- 0 - case code = <-exitChan: - trace.Stop() // Stops trace if profiling has been enabled - server.Shutdown(code, "startup error") - shutdownCompleted <- code + for { + select { + case <-sighupChan: + log.Reload() + case sig := <-signalChan: + server.Shutdown(fmt.Sprintf("System signal: %s", sig)) + } } } diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go index 8ed3196e4ad..2c67a06a843 100644 --- a/pkg/cmd/grafana-server/server.go +++ b/pkg/cmd/grafana-server/server.go @@ -11,24 +11,33 @@ import ( "strconv" "time" - "github.com/grafana/grafana/pkg/services/provisioning" + "github.com/facebookgo/inject" + "github.com/grafana/grafana/pkg/api" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/login" + "github.com/grafana/grafana/pkg/middleware" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/social" "golang.org/x/sync/errgroup" - "github.com/grafana/grafana/pkg/api" "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/login" - "github.com/grafana/grafana/pkg/metrics" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/services/alerting" - "github.com/grafana/grafana/pkg/services/cleanup" - "github.com/grafana/grafana/pkg/services/notifications" - "github.com/grafana/grafana/pkg/services/search" - "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/services/cache" "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/social" - "github.com/grafana/grafana/pkg/tracing" + // self registering services + _ "github.com/grafana/grafana/pkg/extensions" + _ "github.com/grafana/grafana/pkg/metrics" + _ "github.com/grafana/grafana/pkg/plugins" + _ "github.com/grafana/grafana/pkg/services/alerting" + _ "github.com/grafana/grafana/pkg/services/cleanup" + _ "github.com/grafana/grafana/pkg/services/notifications" + _ "github.com/grafana/grafana/pkg/services/provisioning" + _ "github.com/grafana/grafana/pkg/services/rendering" + _ "github.com/grafana/grafana/pkg/services/search" + _ "github.com/grafana/grafana/pkg/services/sqlstore" + _ "github.com/grafana/grafana/pkg/tracing" ) func NewGrafanaServer() *GrafanaServerImpl { @@ -40,110 +49,143 @@ func NewGrafanaServer() *GrafanaServerImpl { shutdownFn: shutdownFn, childRoutines: childRoutines, log: log.New("server"), + cfg: setting.NewCfg(), } } type GrafanaServerImpl struct { - context context.Context - shutdownFn context.CancelFunc - childRoutines *errgroup.Group - log log.Logger + context context.Context + shutdownFn context.CancelFunc + childRoutines *errgroup.Group + log log.Logger + cfg *setting.Cfg + shutdownReason string + shutdownInProgress bool - httpServer *api.HttpServer + RouteRegister routing.RouteRegister `inject:""` + HttpServer *api.HTTPServer `inject:""` } -func (g *GrafanaServerImpl) Start() error { - g.initLogging() +func (g *GrafanaServerImpl) Run() error { + g.loadConfiguration() g.writePIDFile() - initSql() - - metrics.Init(setting.Cfg) - search.Init() login.Init() social.NewOAuthService() - pluginManager, err := plugins.NewPluginManager(g.context) - if err != nil { - return fmt.Errorf("Failed to start plugins. error: %v", err) - } - g.childRoutines.Go(func() error { return pluginManager.Run(g.context) }) + serviceGraph := inject.Graph{} + serviceGraph.Provide(&inject.Object{Value: bus.GetBus()}) + serviceGraph.Provide(&inject.Object{Value: g.cfg}) + serviceGraph.Provide(&inject.Object{Value: routing.NewRouteRegister(middleware.RequestMetrics, middleware.RequestTracing)}) + serviceGraph.Provide(&inject.Object{Value: cache.New(5*time.Minute, 10*time.Minute)}) - if err := provisioning.Init(g.context, setting.HomePath, setting.Cfg); err != nil { - return fmt.Errorf("Failed to provision Grafana from config. error: %v", err) + // self registered services + services := registry.GetServices() + + // Add all services to dependency graph + for _, service := range services { + serviceGraph.Provide(&inject.Object{Value: service.Instance}) } - tracingCloser, err := tracing.Init(setting.Cfg) - if err != nil { - return fmt.Errorf("Tracing settings is not valid. error: %v", err) - } - defer tracingCloser.Close() + serviceGraph.Provide(&inject.Object{Value: g}) - // init alerting - if setting.AlertingEnabled && setting.ExecuteAlerts { - engine := alerting.NewEngine() - g.childRoutines.Go(func() error { return engine.Run(g.context) }) + // Inject dependencies to services + if err := serviceGraph.Populate(); err != nil { + return fmt.Errorf("Failed to populate service dependency: %v", err) } - // cleanup service - cleanUpService := cleanup.NewCleanUpService() - g.childRoutines.Go(func() error { return cleanUpService.Run(g.context) }) + // Init & start services + for _, service := range services { + if registry.IsDisabled(service.Instance) { + continue + } - if err = notifications.Init(); err != nil { - return fmt.Errorf("Notification service failed to initialize. error: %v", err) + g.log.Info("Initializing " + service.Name) + + if err := service.Instance.Init(); err != nil { + return fmt.Errorf("Service init failed: %v", err) + } + } + + // Start background services + for _, srv := range services { + // variable needed for accessing loop variable in function callback + descriptor := srv + service, ok := srv.Instance.(registry.BackgroundService) + if !ok { + continue + } + + if registry.IsDisabled(descriptor.Instance) { + continue + } + + g.childRoutines.Go(func() error { + // Skip starting new service when shutting down + // Can happen when service stop/return during startup + if g.shutdownInProgress { + return nil + } + + err := service.Run(g.context) + + // If error is not canceled then the service crashed + if err != context.Canceled && err != nil { + g.log.Error("Stopped "+descriptor.Name, "reason", err) + } else { + g.log.Info("Stopped "+descriptor.Name, "reason", err) + } + + // Mark that we are in shutdown mode + // So more services are not started + g.shutdownInProgress = true + return err + }) } sendSystemdNotification("READY=1") - - return g.startHttpServer() + return g.childRoutines.Wait() } -func initSql() { - sqlstore.NewEngine() - sqlstore.EnsureAdminUser() -} - -func (g *GrafanaServerImpl) initLogging() { - err := setting.NewConfigContext(&setting.CommandLineArgs{ +func (g *GrafanaServerImpl) loadConfiguration() { + err := g.cfg.Load(&setting.CommandLineArgs{ Config: *configFile, HomePath: *homePath, Args: flag.Args(), }) if err != nil { - g.log.Error(err.Error()) + fmt.Fprintf(os.Stderr, "Failed to start grafana. error: %s\n", err.Error()) os.Exit(1) } - g.log.Info("Starting Grafana", "version", version, "commit", commit, "compiled", time.Unix(setting.BuildStamp, 0)) - setting.LogConfigurationInfo() + g.log.Info("Starting "+setting.ApplicationName, "version", version, "commit", commit, "branch", buildBranch, "compiled", time.Unix(setting.BuildStamp, 0)) + g.cfg.LogConfigSources() } -func (g *GrafanaServerImpl) startHttpServer() error { - g.httpServer = api.NewHttpServer() - - err := g.httpServer.Start(g.context) - - if err != nil { - return fmt.Errorf("Fail to start server. error: %v", err) - } - - return nil -} - -func (g *GrafanaServerImpl) Shutdown(code int, reason string) { - g.log.Info("Shutdown started", "code", code, "reason", reason) - - err := g.httpServer.Shutdown(g.context) - if err != nil { - g.log.Error("Failed to shutdown server", "error", err) - } +func (g *GrafanaServerImpl) Shutdown(reason string) { + g.log.Info("Shutdown started", "reason", reason) + g.shutdownReason = reason + g.shutdownInProgress = true + // call cancel func on root context g.shutdownFn() - err = g.childRoutines.Wait() - if err != nil && err != context.Canceled { - g.log.Error("Server shutdown completed with an error", "error", err) + + // wait for child routines + g.childRoutines.Wait() +} + +func (g *GrafanaServerImpl) Exit(reason error) int { + // default exit code is 1 + code := 1 + + if reason == context.Canceled && g.shutdownReason != "" { + reason = fmt.Errorf(g.shutdownReason) + code = 0 } + + g.log.Error("Server shutdown", "reason", reason) + return code } func (g *GrafanaServerImpl) writePIDFile() { diff --git a/pkg/components/apikeygen/apikeygen.go b/pkg/components/apikeygen/apikeygen.go index 310188a80ef..7824cf7667f 100644 --- a/pkg/components/apikeygen/apikeygen.go +++ b/pkg/components/apikeygen/apikeygen.go @@ -33,7 +33,7 @@ func New(orgId int64, name string) KeyGenResult { jsonString, _ := json.Marshal(jsonKey) - result.ClientSecret = base64.StdEncoding.EncodeToString([]byte(jsonString)) + result.ClientSecret = base64.StdEncoding.EncodeToString(jsonString) return result } @@ -44,7 +44,7 @@ func Decode(keyString string) (*ApiKeyJson, error) { } var keyObj ApiKeyJson - err = json.Unmarshal([]byte(jsonString), &keyObj) + err = json.Unmarshal(jsonString, &keyObj) if err != nil { return nil, ErrInvalidApiKey } diff --git a/pkg/components/dashdiffs/compare.go b/pkg/components/dashdiffs/compare.go index f5f2104cb92..ae940091ed1 100644 --- a/pkg/components/dashdiffs/compare.go +++ b/pkg/components/dashdiffs/compare.go @@ -6,7 +6,6 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" diff "github.com/yudai/gojsondiff" deltaFormatter "github.com/yudai/gojsondiff/formatter" @@ -15,11 +14,8 @@ import ( var ( // ErrUnsupportedDiffType occurs when an invalid diff type is used. ErrUnsupportedDiffType = errors.New("dashdiff: unsupported diff type") - // ErrNilDiff occurs when two compared interfaces are identical. ErrNilDiff = errors.New("dashdiff: diff is nil") - - diffLogger = log.New("dashdiffs") ) type DiffType int @@ -145,5 +141,9 @@ func getDiff(baseData, newData *simplejson.Json) (interface{}, diff.Diff, error) left := make(map[string]interface{}) err = json.Unmarshal(leftBytes, &left) + if err != nil { + return nil, nil, err + } + return left, jsonDiff, nil } diff --git a/pkg/components/dashdiffs/formatter_json.go b/pkg/components/dashdiffs/formatter_json.go index 3a9ddcc4ee3..488a345d492 100644 --- a/pkg/components/dashdiffs/formatter_json.go +++ b/pkg/components/dashdiffs/formatter_json.go @@ -22,7 +22,7 @@ const ( ) var ( - // changeTypeToSymbol is used for populating the terminating characer in + // changeTypeToSymbol is used for populating the terminating character in // the diff changeTypeToSymbol = map[ChangeType]string{ ChangeNil: "", diff --git a/pkg/components/dynmap/dynmap.go b/pkg/components/dynmap/dynmap.go index 797694845cd..96effb24332 100644 --- a/pkg/components/dynmap/dynmap.go +++ b/pkg/components/dynmap/dynmap.go @@ -134,9 +134,8 @@ func (v *Value) get(key string) (*Value, error) { child, ok := obj.Map()[key] if ok { return child, nil - } else { - return nil, KeyNotFoundError{key} } + return nil, KeyNotFoundError{key} } return nil, err @@ -174,17 +173,13 @@ func (v *Object) GetObject(keys ...string) (*Object, error) { if err != nil { return nil, err - } else { - - obj, err := child.Object() - - if err != nil { - return nil, err - } else { - return obj, nil - } - } + obj, err := child.Object() + + if err != nil { + return nil, err + } + return obj, nil } // Gets the value at key path and attempts to typecast the value into a string. @@ -196,18 +191,17 @@ func (v *Object) GetString(keys ...string) (string, error) { if err != nil { return "", err - } else { - return child.String() } + return child.String() } func (v *Object) MustGetString(path string, def string) string { keys := strings.Split(path, ".") - if str, err := v.GetString(keys...); err != nil { + str, err := v.GetString(keys...) + if err != nil { return def - } else { - return str } + return str } // Gets the value at key path and attempts to typecast the value into null. @@ -233,16 +227,13 @@ func (v *Object) GetNumber(keys ...string) (json.Number, error) { if err != nil { return "", err - } else { - - n, err := child.Number() - - if err != nil { - return "", err - } else { - return n, nil - } } + n, err := child.Number() + + if err != nil { + return "", err + } + return n, nil } // Gets the value at key path and attempts to typecast the value into a float64. @@ -254,16 +245,13 @@ func (v *Object) GetFloat64(keys ...string) (float64, error) { if err != nil { return 0, err - } else { - - n, err := child.Float64() - - if err != nil { - return 0, err - } else { - return n, nil - } } + n, err := child.Float64() + + if err != nil { + return 0, err + } + return n, nil } // Gets the value at key path and attempts to typecast the value into a float64. @@ -275,16 +263,13 @@ func (v *Object) GetInt64(keys ...string) (int64, error) { if err != nil { return 0, err - } else { - - n, err := child.Int64() - - if err != nil { - return 0, err - } else { - return n, nil - } } + n, err := child.Int64() + + if err != nil { + return 0, err + } + return n, nil } // Gets the value at key path and attempts to typecast the value into a float64. @@ -296,9 +281,8 @@ func (v *Object) GetInterface(keys ...string) (interface{}, error) { if err != nil { return nil, err - } else { - return child.Interface(), nil } + return child.Interface(), nil } // Gets the value at key path and attempts to typecast the value into a bool. @@ -311,7 +295,6 @@ func (v *Object) GetBoolean(keys ...string) (bool, error) { if err != nil { return false, err } - return child.Boolean() } @@ -328,11 +311,8 @@ func (v *Object) GetValueArray(keys ...string) ([]*Value, error) { if err != nil { return nil, err - } else { - - return child.Array() - } + return child.Array() } // Gets the value at key path and attempts to typecast the value into an array of objects. @@ -347,30 +327,24 @@ func (v *Object) GetObjectArray(keys ...string) ([]*Object, error) { if err != nil { return nil, err - } else { + } + array, err := child.Array() - array, err := child.Array() + if err != nil { + return nil, err + } + typedArray := make([]*Object, len(array)) + + for index, arrayItem := range array { + typedArrayItem, err := arrayItem. + Object() if err != nil { return nil, err - } else { - - typedArray := make([]*Object, len(array)) - - for index, arrayItem := range array { - typedArrayItem, err := arrayItem. - Object() - - if err != nil { - return nil, err - } else { - typedArray[index] = typedArrayItem - } - - } - return typedArray, nil } + typedArray[index] = typedArrayItem } + return typedArray, nil } // Gets the value at key path and attempts to typecast the value into an array of string. @@ -387,29 +361,23 @@ func (v *Object) GetStringArray(keys ...string) ([]string, error) { if err != nil { return nil, err - } else { + } + array, err := child.Array() - array, err := child.Array() + if err != nil { + return nil, err + } + typedArray := make([]string, len(array)) + + for index, arrayItem := range array { + typedArrayItem, err := arrayItem.String() if err != nil { return nil, err - } else { - - typedArray := make([]string, len(array)) - - for index, arrayItem := range array { - typedArrayItem, err := arrayItem.String() - - if err != nil { - return nil, err - } else { - typedArray[index] = typedArrayItem - } - - } - return typedArray, nil } + typedArray[index] = typedArrayItem } + return typedArray, nil } // Gets the value at key path and attempts to typecast the value into an array of numbers. @@ -424,29 +392,23 @@ func (v *Object) GetNumberArray(keys ...string) ([]json.Number, error) { if err != nil { return nil, err - } else { + } + array, err := child.Array() - array, err := child.Array() + if err != nil { + return nil, err + } + typedArray := make([]json.Number, len(array)) + + for index, arrayItem := range array { + typedArrayItem, err := arrayItem.Number() if err != nil { return nil, err - } else { - - typedArray := make([]json.Number, len(array)) - - for index, arrayItem := range array { - typedArrayItem, err := arrayItem.Number() - - if err != nil { - return nil, err - } else { - typedArray[index] = typedArrayItem - } - - } - return typedArray, nil } + typedArray[index] = typedArrayItem } + return typedArray, nil } // Gets the value at key path and attempts to typecast the value into an array of floats. @@ -456,29 +418,23 @@ func (v *Object) GetFloat64Array(keys ...string) ([]float64, error) { if err != nil { return nil, err - } else { + } + array, err := child.Array() - array, err := child.Array() + if err != nil { + return nil, err + } + typedArray := make([]float64, len(array)) + + for index, arrayItem := range array { + typedArrayItem, err := arrayItem.Float64() if err != nil { return nil, err - } else { - - typedArray := make([]float64, len(array)) - - for index, arrayItem := range array { - typedArrayItem, err := arrayItem.Float64() - - if err != nil { - return nil, err - } else { - typedArray[index] = typedArrayItem - } - - } - return typedArray, nil } + typedArray[index] = typedArrayItem } + return typedArray, nil } // Gets the value at key path and attempts to typecast the value into an array of ints. @@ -488,29 +444,23 @@ func (v *Object) GetInt64Array(keys ...string) ([]int64, error) { if err != nil { return nil, err - } else { + } + array, err := child.Array() - array, err := child.Array() + if err != nil { + return nil, err + } + typedArray := make([]int64, len(array)) + + for index, arrayItem := range array { + typedArrayItem, err := arrayItem.Int64() if err != nil { return nil, err - } else { - - typedArray := make([]int64, len(array)) - - for index, arrayItem := range array { - typedArrayItem, err := arrayItem.Int64() - - if err != nil { - return nil, err - } else { - typedArray[index] = typedArrayItem - } - - } - return typedArray, nil } + typedArray[index] = typedArrayItem } + return typedArray, nil } // Gets the value at key path and attempts to typecast the value into an array of bools. @@ -520,29 +470,23 @@ func (v *Object) GetBooleanArray(keys ...string) ([]bool, error) { if err != nil { return nil, err - } else { + } + array, err := child.Array() - array, err := child.Array() + if err != nil { + return nil, err + } + typedArray := make([]bool, len(array)) + + for index, arrayItem := range array { + typedArrayItem, err := arrayItem.Boolean() if err != nil { return nil, err - } else { - - typedArray := make([]bool, len(array)) - - for index, arrayItem := range array { - typedArrayItem, err := arrayItem.Boolean() - - if err != nil { - return nil, err - } else { - typedArray[index] = typedArrayItem - } - - } - return typedArray, nil } + typedArray[index] = typedArrayItem } + return typedArray, nil } // Gets the value at key path and attempts to typecast the value into an array of nulls. @@ -552,29 +496,23 @@ func (v *Object) GetNullArray(keys ...string) (int64, error) { if err != nil { return 0, err - } else { + } + array, err := child.Array() - array, err := child.Array() + if err != nil { + return 0, err + } + var length int64 = 0 + + for _, arrayItem := range array { + err := arrayItem.Null() if err != nil { return 0, err - } else { - - var length int64 = 0 - - for _, arrayItem := range array { - err := arrayItem.Null() - - if err != nil { - return 0, err - } else { - length++ - } - - } - return length, nil } + length++ } + return length, nil } // Returns an error if the value is not actually null @@ -585,15 +523,12 @@ func (v *Value) Null() error { switch v.data.(type) { case nil: valid = v.exists // Valid only if j also exists, since other values could possibly also be nil - break } if valid { return nil } - return ErrNotNull - } // Attempts to typecast the current value into an array. @@ -607,24 +542,19 @@ func (v *Value) Array() ([]*Value, error) { switch v.data.(type) { case []interface{}: valid = true - break } // Unsure if this is a good way to use slices, it's probably not var slice []*Value if valid { - for _, element := range v.data.([]interface{}) { child := Value{element, true} slice = append(slice, &child) } - return slice, nil } - return slice, ErrNotArray - } // Attempts to typecast the current value into a number. @@ -638,7 +568,6 @@ func (v *Value) Number() (json.Number, error) { switch v.data.(type) { case json.Number: valid = true - break } if valid { @@ -687,7 +616,6 @@ func (v *Value) Boolean() (bool, error) { switch v.data.(type) { case bool: valid = true - break } if valid { @@ -709,7 +637,6 @@ func (v *Value) Object() (*Object, error) { switch v.data.(type) { case map[string]interface{}: valid = true - break } if valid { @@ -746,7 +673,6 @@ func (v *Value) ObjectArray() ([]*Object, error) { switch v.data.(type) { case []interface{}: valid = true - break } // Unsure if this is a good way to use slices, it's probably not @@ -782,7 +708,6 @@ func (v *Value) String() (string, error) { switch v.data.(type) { case string: valid = true - break } if valid { diff --git a/pkg/components/dynmap/dynmap_test.go b/pkg/components/dynmap/dynmap_test.go index cc002ea06e0..62d356bd67d 100644 --- a/pkg/components/dynmap/dynmap_test.go +++ b/pkg/components/dynmap/dynmap_test.go @@ -21,7 +21,7 @@ func NewAssert(t *testing.T) *Assert { } func (assert *Assert) True(value bool, message string) { - if value == false { + if !value { log.Panicln("Assert: ", message) } } @@ -60,6 +60,7 @@ func TestFirst(t *testing.T) { }` j, err := NewObjectFromBytes([]byte(testJSON)) + assert.True(err == nil, "failed to create new object from bytes") a, err := j.GetObject("address") assert.True(a != nil && err == nil, "failed to create json from string") @@ -76,10 +77,10 @@ func TestFirst(t *testing.T) { assert.True(s == "fallback", "must get string return fallback") s, err = j.GetString("name") - assert.True(s == "anton" && err == nil, "name shoud match") + assert.True(s == "anton" && err == nil, "name should match") s, err = j.GetString("address", "street") - assert.True(s == "Street 42" && err == nil, "street shoud match") + assert.True(s == "Street 42" && err == nil, "street should match") //log.Println("s: ", s.String()) _, err = j.GetNumber("age") @@ -108,6 +109,7 @@ func TestFirst(t *testing.T) { //log.Println("address: ", address) s, err = address.GetString("street") + assert.True(s == "Street 42" && err == nil, "street mismatching") addressAsString, err := j.GetString("address") assert.True(addressAsString == "" && err != nil, "address should not be an string") @@ -119,13 +121,13 @@ func TestFirst(t *testing.T) { assert.True(s == "" && err != nil, "nonexistent string fail") b, err := j.GetBoolean("true") - assert.True(b == true && err == nil, "bool true test") + assert.True(b && err == nil, "bool true test") b, err = j.GetBoolean("false") - assert.True(b == false && err == nil, "bool false test") + assert.True(!b && err == nil, "bool false test") b, err = j.GetBoolean("invalid_field") - assert.True(b == false && err != nil, "bool invalid test") + assert.True(!b && err != nil, "bool invalid test") list, err := j.GetValueArray("list") assert.True(list != nil && err == nil, "list should be an array") @@ -148,6 +150,7 @@ func TestFirst(t *testing.T) { //assert.True(element.IsObject() == true, "first fail") element, err := elementValue.Object() + assert.True(err == nil, "create element fail") s, err = element.GetString("street") assert.True(s == "Street 42" && err == nil, "second fail") @@ -232,6 +235,7 @@ func TestSecond(t *testing.T) { assert.True(fromName == "Tom Brady" && err == nil, "fromName mismatch") actions, err := dataItem.GetObjectArray("actions") + assert.True(err == nil, "get object from array failed") for index, action := range actions { diff --git a/pkg/components/imguploader/azureblobuploader.go b/pkg/components/imguploader/azureblobuploader.go index 40d2de836be..b37763931c8 100644 --- a/pkg/components/imguploader/azureblobuploader.go +++ b/pkg/components/imguploader/azureblobuploader.go @@ -52,7 +52,7 @@ func (az *AzureBlobUploader) Upload(ctx context.Context, imageDiskPath string) ( } randomFileName := util.GetRandomString(30) + ".png" // upload image - az.log.Debug("Uploading image to azure_blob", "conatiner_name", az.container_name, "blob_name", randomFileName) + az.log.Debug("Uploading image to azure_blob", "container_name", az.container_name, "blob_name", randomFileName) resp, err := blob.FileUpload(az.container_name, randomFileName, file) if err != nil { return "", err @@ -127,8 +127,6 @@ type xmlError struct { const ms_date_layout = "Mon, 02 Jan 2006 15:04:05 GMT" const version = "2017-04-17" -var client = &http.Client{} - type StorageClient struct { Auth *Auth Transport http.RoundTripper @@ -225,7 +223,7 @@ func (a *Auth) SignRequest(req *http.Request) { ) decodedKey, _ := base64.StdEncoding.DecodeString(a.Key) - sha256 := hmac.New(sha256.New, []byte(decodedKey)) + sha256 := hmac.New(sha256.New, decodedKey) sha256.Write([]byte(strToSign)) signature := base64.StdEncoding.EncodeToString(sha256.Sum(nil)) @@ -274,10 +272,10 @@ func (a *Auth) canonicalizedHeaders(req *http.Request) string { } } - splitted := strings.Split(buffer.String(), "\n") - sort.Strings(splitted) + split := strings.Split(buffer.String(), "\n") + sort.Strings(split) - return strings.Join(splitted, "\n") + return strings.Join(split, "\n") } /* @@ -313,8 +311,8 @@ func (a *Auth) canonicalizedResource(req *http.Request) string { buffer.WriteString(fmt.Sprintf("\n%s:%s", key, strings.Join(values, ","))) } - splitted := strings.Split(buffer.String(), "\n") - sort.Strings(splitted) + split := strings.Split(buffer.String(), "\n") + sort.Strings(split) - return strings.Join(splitted, "\n") + return strings.Join(split, "\n") } diff --git a/pkg/components/imguploader/azureblobuploader_test.go b/pkg/components/imguploader/azureblobuploader_test.go index 570e105b321..c0c7889a155 100644 --- a/pkg/components/imguploader/azureblobuploader_test.go +++ b/pkg/components/imguploader/azureblobuploader_test.go @@ -10,9 +10,11 @@ import ( func TestUploadToAzureBlob(t *testing.T) { SkipConvey("[Integration test] for external_image_store.azure_blob", t, func() { - err := setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + err := cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) + So(err, ShouldBeNil) uploader, _ := NewImageUploader() diff --git a/pkg/components/imguploader/gcsuploader_test.go b/pkg/components/imguploader/gcsuploader_test.go index bdc21084dbf..58cb21c184c 100644 --- a/pkg/components/imguploader/gcsuploader_test.go +++ b/pkg/components/imguploader/gcsuploader_test.go @@ -10,7 +10,8 @@ import ( func TestUploadToGCS(t *testing.T) { SkipConvey("[Integration test] for external_image_store.gcs", t, func() { - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) diff --git a/pkg/components/imguploader/imguploader.go b/pkg/components/imguploader/imguploader.go index 52a31f9f606..93f69cadd46 100644 --- a/pkg/components/imguploader/imguploader.go +++ b/pkg/components/imguploader/imguploader.go @@ -3,9 +3,10 @@ package imguploader import ( "context" "fmt" - "github.com/grafana/grafana/pkg/log" "regexp" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/setting" ) @@ -24,7 +25,7 @@ func NewImageUploader() (ImageUploader, error) { switch setting.ImageUploadProvider { case "s3": - s3sec, err := setting.Cfg.GetSection("external_image_storage.s3") + s3sec, err := setting.Raw.GetSection("external_image_storage.s3") if err != nil { return nil, err } @@ -51,7 +52,7 @@ func NewImageUploader() (ImageUploader, error) { return NewS3Uploader(region, bucket, path, "public-read", accessKey, secretKey), nil case "webdav": - webdavSec, err := setting.Cfg.GetSection("external_image_storage.webdav") + webdavSec, err := setting.Raw.GetSection("external_image_storage.webdav") if err != nil { return nil, err } @@ -67,7 +68,7 @@ func NewImageUploader() (ImageUploader, error) { return NewWebdavImageUploader(url, username, password, public_url) case "gcs": - gcssec, err := setting.Cfg.GetSection("external_image_storage.gcs") + gcssec, err := setting.Raw.GetSection("external_image_storage.gcs") if err != nil { return nil, err } @@ -78,7 +79,7 @@ func NewImageUploader() (ImageUploader, error) { return NewGCSUploader(keyFile, bucketName, path), nil case "azure_blob": - azureBlobSec, err := setting.Cfg.GetSection("external_image_storage.azure_blob") + azureBlobSec, err := setting.Raw.GetSection("external_image_storage.azure_blob") if err != nil { return nil, err } diff --git a/pkg/components/imguploader/imguploader_test.go b/pkg/components/imguploader/imguploader_test.go index b0311dac975..570e36a47e3 100644 --- a/pkg/components/imguploader/imguploader_test.go +++ b/pkg/components/imguploader/imguploader_test.go @@ -11,14 +11,16 @@ import ( func TestImageUploaderFactory(t *testing.T) { Convey("Can create image uploader for ", t, func() { Convey("S3ImageUploader config", func() { - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) setting.ImageUploadProvider = "s3" Convey("with bucket url https://foo.bar.baz.s3-us-east-2.amazonaws.com", func() { - s3sec, err := setting.Cfg.GetSection("external_image_storage.s3") + s3sec, err := setting.Raw.GetSection("external_image_storage.s3") + So(err, ShouldBeNil) s3sec.NewKey("bucket_url", "https://foo.bar.baz.s3-us-east-2.amazonaws.com") s3sec.NewKey("access_key", "access_key") s3sec.NewKey("secret_key", "secret_key") @@ -36,7 +38,8 @@ func TestImageUploaderFactory(t *testing.T) { }) Convey("with bucket url https://s3.amazonaws.com/mybucket", func() { - s3sec, err := setting.Cfg.GetSection("external_image_storage.s3") + s3sec, err := setting.Raw.GetSection("external_image_storage.s3") + So(err, ShouldBeNil) s3sec.NewKey("bucket_url", "https://s3.amazonaws.com/my.bucket.com") s3sec.NewKey("access_key", "access_key") s3sec.NewKey("secret_key", "secret_key") @@ -54,16 +57,16 @@ func TestImageUploaderFactory(t *testing.T) { }) Convey("with bucket url https://s3-us-west-2.amazonaws.com/mybucket", func() { - s3sec, err := setting.Cfg.GetSection("external_image_storage.s3") + s3sec, err := setting.Raw.GetSection("external_image_storage.s3") + So(err, ShouldBeNil) s3sec.NewKey("bucket_url", "https://s3-us-west-2.amazonaws.com/my.bucket.com") s3sec.NewKey("access_key", "access_key") s3sec.NewKey("secret_key", "secret_key") uploader, err := NewImageUploader() - So(err, ShouldBeNil) - original, ok := uploader.(*S3Uploader) + original, ok := uploader.(*S3Uploader) So(ok, ShouldBeTrue) So(original.region, ShouldEqual, "us-west-2") So(original.bucket, ShouldEqual, "my.bucket.com") @@ -75,13 +78,15 @@ func TestImageUploaderFactory(t *testing.T) { Convey("Webdav uploader", func() { var err error - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) setting.ImageUploadProvider = "webdav" - webdavSec, err := setting.Cfg.GetSection("external_image_storage.webdav") + webdavSec, err := cfg.Raw.GetSection("external_image_storage.webdav") + So(err, ShouldBeNil) webdavSec.NewKey("url", "webdavUrl") webdavSec.NewKey("username", "username") webdavSec.NewKey("password", "password") @@ -100,43 +105,45 @@ func TestImageUploaderFactory(t *testing.T) { Convey("GCS uploader", func() { var err error - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) setting.ImageUploadProvider = "gcs" - gcpSec, err := setting.Cfg.GetSection("external_image_storage.gcs") + gcpSec, err := cfg.Raw.GetSection("external_image_storage.gcs") + So(err, ShouldBeNil) gcpSec.NewKey("key_file", "/etc/secrets/project-79a52befa3f6.json") gcpSec.NewKey("bucket", "project-grafana-east") uploader, err := NewImageUploader() - So(err, ShouldBeNil) - original, ok := uploader.(*GCSUploader) + original, ok := uploader.(*GCSUploader) So(ok, ShouldBeTrue) So(original.keyFile, ShouldEqual, "/etc/secrets/project-79a52befa3f6.json") So(original.bucket, ShouldEqual, "project-grafana-east") }) Convey("AzureBlobUploader config", func() { - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) setting.ImageUploadProvider = "azure_blob" Convey("with container name", func() { - azureBlobSec, err := setting.Cfg.GetSection("external_image_storage.azure_blob") + azureBlobSec, err := cfg.Raw.GetSection("external_image_storage.azure_blob") + So(err, ShouldBeNil) azureBlobSec.NewKey("account_name", "account_name") azureBlobSec.NewKey("account_key", "account_key") azureBlobSec.NewKey("container_name", "container_name") uploader, err := NewImageUploader() - So(err, ShouldBeNil) - original, ok := uploader.(*AzureBlobUploader) + original, ok := uploader.(*AzureBlobUploader) So(ok, ShouldBeTrue) So(original.account_name, ShouldEqual, "account_name") So(original.account_key, ShouldEqual, "account_key") @@ -147,7 +154,8 @@ func TestImageUploaderFactory(t *testing.T) { Convey("Local uploader", func() { var err error - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) diff --git a/pkg/components/imguploader/s3uploader.go b/pkg/components/imguploader/s3uploader.go index 62196357c61..9c8af21e39e 100644 --- a/pkg/components/imguploader/s3uploader.go +++ b/pkg/components/imguploader/s3uploader.go @@ -2,12 +2,15 @@ package imguploader import ( "context" + "fmt" "os" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" + "github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" @@ -50,7 +53,7 @@ func (u *S3Uploader) Upload(ctx context.Context, imageDiskPath string) (string, SecretAccessKey: u.secretKey, }}, &credentials.EnvProvider{}, - &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess), ExpiryWindow: 5 * time.Minute}, + remoteCredProvider(sess), }) cfg := &aws.Config{ Region: aws.String(u.region), @@ -60,7 +63,7 @@ func (u *S3Uploader) Upload(ctx context.Context, imageDiskPath string) (string, s3_endpoint, _ := endpoints.DefaultResolver().EndpointFor("s3", u.region) key := u.path + util.GetRandomString(20) + ".png" image_url := s3_endpoint.URL + "/" + u.bucket + "/" + key - log.Debug("Uploading image to s3", "url = ", image_url) + log.Debug("Uploading image to s3. url = %s", image_url) file, err := os.Open(imageDiskPath) if err != nil { @@ -85,3 +88,27 @@ func (u *S3Uploader) Upload(ctx context.Context, imageDiskPath string) (string, } return image_url, nil } + +func remoteCredProvider(sess *session.Session) credentials.Provider { + ecsCredURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + + if len(ecsCredURI) > 0 { + return ecsCredProvider(sess, ecsCredURI) + } + return ec2RoleProvider(sess) +} + +func ecsCredProvider(sess *session.Session, uri string) credentials.Provider { + const host = `169.254.170.2` + + d := defaults.Get() + return endpointcreds.NewProviderClient( + *d.Config, + d.Handlers, + fmt.Sprintf("http://%s%s", host, uri), + func(p *endpointcreds.Provider) { p.ExpiryWindow = 5 * time.Minute }) +} + +func ec2RoleProvider(sess *session.Session) credentials.Provider { + return &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess), ExpiryWindow: 5 * time.Minute} +} diff --git a/pkg/components/imguploader/s3uploader_test.go b/pkg/components/imguploader/s3uploader_test.go index b02d4676b5e..0e43740ef9b 100644 --- a/pkg/components/imguploader/s3uploader_test.go +++ b/pkg/components/imguploader/s3uploader_test.go @@ -10,7 +10,8 @@ import ( func TestUploadToS3(t *testing.T) { SkipConvey("[Integration test] for external_image_store.s3", t, func() { - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) diff --git a/pkg/components/imguploader/webdavuploader.go b/pkg/components/imguploader/webdavuploader.go index 53d75247c76..ed6b14725c0 100644 --- a/pkg/components/imguploader/webdavuploader.go +++ b/pkg/components/imguploader/webdavuploader.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "path" + "strings" "time" "github.com/grafana/grafana/pkg/util" @@ -35,20 +36,36 @@ var netClient = &http.Client{ Transport: netTransport, } +func (u *WebdavUploader) PublicURL(filename string) string { + if strings.Contains(u.public_url, "${file}") { + return strings.Replace(u.public_url, "${file}", filename, -1) + } else { + publicURL, _ := url.Parse(u.public_url) + publicURL.Path = path.Join(publicURL.Path, filename) + return publicURL.String() + } +} + func (u *WebdavUploader) Upload(ctx context.Context, pa string) (string, error) { url, _ := url.Parse(u.url) filename := util.GetRandomString(20) + ".png" url.Path = path.Join(url.Path, filename) imgData, err := ioutil.ReadFile(pa) + if err != nil { + return "", err + } + req, err := http.NewRequest("PUT", url.String(), bytes.NewReader(imgData)) + if err != nil { + return "", err + } if u.username != "" { req.SetBasicAuth(u.username, u.password) } res, err := netClient.Do(req) - if err != nil { return "", err } @@ -59,9 +76,7 @@ func (u *WebdavUploader) Upload(ctx context.Context, pa string) (string, error) } if u.public_url != "" { - publicURL, _ := url.Parse(u.public_url) - publicURL.Path = path.Join(publicURL.Path, filename) - return publicURL.String(), nil + return u.PublicURL(filename), nil } return url.String(), nil diff --git a/pkg/components/imguploader/webdavuploader_test.go b/pkg/components/imguploader/webdavuploader_test.go index 5a8abd0542d..0178c9cda6c 100644 --- a/pkg/components/imguploader/webdavuploader_test.go +++ b/pkg/components/imguploader/webdavuploader_test.go @@ -2,6 +2,7 @@ package imguploader import ( "context" + "net/url" "testing" . "github.com/smartystreets/goconvey/convey" @@ -26,3 +27,15 @@ func TestUploadToWebdav(t *testing.T) { So(path, ShouldStartWith, "http://publicurl:8888/webdav/") }) } + +func TestPublicURL(t *testing.T) { + Convey("Given a public URL with parameters, and no template", t, func() { + webdavUploader, _ := NewWebdavImageUploader("http://localhost:8888/webdav/", "test", "test", "http://cloudycloud.me/s/DOIFDOMV/download?files=") + parsed, _ := url.Parse(webdavUploader.PublicURL("fileyfile.png")) + So(parsed.Path, ShouldEndWith, "fileyfile.png") + }) + Convey("Given a public URL with parameters, and a template", t, func() { + webdavUploader, _ := NewWebdavImageUploader("http://localhost:8888/webdav/", "test", "test", "http://cloudycloud.me/s/DOIFDOMV/download?files=${file}") + So(webdavUploader.PublicURL("fileyfile.png"), ShouldEndWith, "fileyfile.png") + }) +} diff --git a/pkg/components/null/float.go b/pkg/components/null/float.go index 1e78946e878..9082c831084 100644 --- a/pkg/components/null/float.go +++ b/pkg/components/null/float.go @@ -8,6 +8,10 @@ import ( "strconv" ) +const ( + nullString = "null" +) + // Float is a nullable float64. // It does not consider zero values to be null. // It will decode to null, not zero, if null. @@ -50,7 +54,7 @@ func (f *Float) UnmarshalJSON(data []byte) error { } switch x := v.(type) { case float64: - f.Float64 = float64(x) + f.Float64 = x case map[string]interface{}: err = json.Unmarshal(data, &f.NullFloat64) case nil: @@ -68,7 +72,7 @@ func (f *Float) UnmarshalJSON(data []byte) error { // It will return an error if the input is not an integer, blank, or "null". func (f *Float) UnmarshalText(text []byte) error { str := string(text) - if str == "" || str == "null" { + if str == "" || str == nullString { f.Valid = false return nil } @@ -82,7 +86,7 @@ func (f *Float) UnmarshalText(text []byte) error { // It will encode null if this Float is null. func (f Float) MarshalJSON() ([]byte, error) { if !f.Valid { - return []byte("null"), nil + return []byte(nullString), nil } return []byte(strconv.FormatFloat(f.Float64, 'f', -1, 64)), nil } @@ -100,12 +104,21 @@ func (f Float) MarshalText() ([]byte, error) { // It will encode a blank string if this Float is null. func (f Float) String() string { if !f.Valid { - return "null" + return nullString } return fmt.Sprintf("%1.3f", f.Float64) } +// FullString returns float as string in full precision +func (f Float) FullString() string { + if !f.Valid { + return nullString + } + + return fmt.Sprintf("%f", f.Float64) +} + // SetValid changes this Float's value and also sets it to be non-null. func (f *Float) SetValid(n float64) { f.Float64 = n diff --git a/pkg/components/renderer/renderer.go b/pkg/components/renderer/renderer.go deleted file mode 100644 index 26751ddd5c7..00000000000 --- a/pkg/components/renderer/renderer.go +++ /dev/null @@ -1,161 +0,0 @@ -package renderer - -import ( - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "runtime" - "time" - - "strconv" - - "strings" - - "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/middleware" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" -) - -type RenderOpts struct { - Path string - Width string - Height string - Timeout string - OrgId int64 - UserId int64 - OrgRole models.RoleType - Timezone string - IsAlertContext bool - Encoding string -} - -var ErrTimeout = errors.New("Timeout error. You can set timeout in seconds with &timeout url parameter") -var rendererLog log.Logger = log.New("png-renderer") - -func isoTimeOffsetToPosixTz(isoOffset string) string { - // invert offset - if strings.HasPrefix(isoOffset, "UTC+") { - return strings.Replace(isoOffset, "UTC+", "UTC-", 1) - } - if strings.HasPrefix(isoOffset, "UTC-") { - return strings.Replace(isoOffset, "UTC-", "UTC+", 1) - } - return isoOffset -} - -func appendEnviron(baseEnviron []string, name string, value string) []string { - results := make([]string, 0) - prefix := fmt.Sprintf("%s=", name) - for _, v := range baseEnviron { - if !strings.HasPrefix(v, prefix) { - results = append(results, v) - } - } - return append(results, fmt.Sprintf("%s=%s", name, value)) -} - -func RenderToPng(params *RenderOpts) (string, error) { - rendererLog.Info("Rendering", "path", params.Path) - - var executable = "phantomjs" - if runtime.GOOS == "windows" { - executable = executable + ".exe" - } - - localDomain := "localhost" - if setting.HttpAddr != setting.DEFAULT_HTTP_ADDR { - localDomain = setting.HttpAddr - } - - // &render=1 signals to the legacy redirect layer to - // avoid redirect these requests. - url := fmt.Sprintf("%s://%s:%s/%s&render=1", setting.Protocol, localDomain, setting.HttpPort, params.Path) - - binPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, executable)) - scriptPath, _ := filepath.Abs(filepath.Join(setting.PhantomDir, "render.js")) - pngPath, _ := filepath.Abs(filepath.Join(setting.ImagesDir, util.GetRandomString(20))) - pngPath = pngPath + ".png" - - orgRole := params.OrgRole - if params.IsAlertContext { - orgRole = models.ROLE_ADMIN - } - renderKey := middleware.AddRenderAuthKey(params.OrgId, params.UserId, orgRole) - defer middleware.RemoveRenderAuthKey(renderKey) - - timeout, err := strconv.Atoi(params.Timeout) - if err != nil { - timeout = 15 - } - - phantomDebugArg := "--debug=false" - if log.GetLogLevelFor("png-renderer") >= log.LvlDebug { - phantomDebugArg = "--debug=true" - } - - cmdArgs := []string{ - "--ignore-ssl-errors=true", - "--web-security=false", - phantomDebugArg, - scriptPath, - "url=" + url, - "width=" + params.Width, - "height=" + params.Height, - "png=" + pngPath, - "domain=" + localDomain, - "timeout=" + strconv.Itoa(timeout), - "renderKey=" + renderKey, - } - - if params.Encoding != "" { - cmdArgs = append([]string{fmt.Sprintf("--output-encoding=%s", params.Encoding)}, cmdArgs...) - } - - cmd := exec.Command(binPath, cmdArgs...) - output, err := cmd.StdoutPipe() - - if err != nil { - rendererLog.Error("Could not acquire stdout pipe", err) - return "", err - } - cmd.Stderr = cmd.Stdout - - if params.Timezone != "" { - baseEnviron := os.Environ() - cmd.Env = appendEnviron(baseEnviron, "TZ", isoTimeOffsetToPosixTz(params.Timezone)) - } - - err = cmd.Start() - if err != nil { - rendererLog.Error("Could not start command", err) - return "", err - } - - logWriter := log.NewLogWriter(rendererLog, log.LvlDebug, "[phantom] ") - go io.Copy(logWriter, output) - - done := make(chan error) - go func() { - if err := cmd.Wait(); err != nil { - rendererLog.Error("failed to render an image", "error", err) - } - close(done) - }() - - select { - case <-time.After(time.Duration(timeout) * time.Second): - if err := cmd.Process.Kill(); err != nil { - rendererLog.Error("failed to kill", "error", err) - } - return "", ErrTimeout - case <-done: - } - - rendererLog.Debug("Image rendered", "path", pngPath) - return pngPath, nil -} diff --git a/pkg/components/renderer/renderer_test.go b/pkg/components/renderer/renderer_test.go deleted file mode 100644 index 5ee42b784be..00000000000 --- a/pkg/components/renderer/renderer_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package renderer - -// -// import ( -// "io/ioutil" -// "os" -// "testing" -// -// . "github.com/smartystreets/goconvey/convey" -// ) -// -// func TestPhantomRender(t *testing.T) { -// -// Convey("Can render url", t, func() { -// tempDir, _ := ioutil.TempDir("", "img") -// ipng, err := RenderToPng("http://www.google.com") -// So(err, ShouldBeNil) -// So(exists(png), ShouldEqual, true) -// -// //_, err = os.Stat(store.getFilePathForDashboard("hello")) -// //So(err, ShouldBeNil) -// }) -// -// } -// -// func exists(path string) bool { -// _, err := os.Stat(path) -// if err == nil { -// return true -// } -// if os.IsNotExist(err) { -// return false -// } -// return false -// } diff --git a/pkg/components/simplejson/simplejson.go b/pkg/components/simplejson/simplejson.go index 85e2f955943..35e305eb414 100644 --- a/pkg/components/simplejson/simplejson.go +++ b/pkg/components/simplejson/simplejson.go @@ -256,7 +256,7 @@ func (j *Json) StringArray() ([]string, error) { // MustArray guarantees the return of a `[]interface{}` (with optional default) // -// useful when you want to interate over array values in a succinct manner: +// useful when you want to iterate over array values in a succinct manner: // for i, v := range js.Get("results").MustArray() { // fmt.Println(i, v) // } @@ -281,7 +281,7 @@ func (j *Json) MustArray(args ...[]interface{}) []interface{} { // MustMap guarantees the return of a `map[string]interface{}` (with optional default) // -// useful when you want to interate over map values in a succinct manner: +// useful when you want to iterate over map values in a succinct manner: // for k, v := range js.Get("dictionary").MustMap() { // fmt.Println(k, v) // } @@ -329,7 +329,7 @@ func (j *Json) MustString(args ...string) string { // MustStringArray guarantees the return of a `[]string` (with optional default) // -// useful when you want to interate over array values in a succinct manner: +// useful when you want to iterate over array values in a succinct manner: // for i, s := range js.Get("results").MustStringArray() { // fmt.Println(i, s) // } diff --git a/pkg/extensions/main.go b/pkg/extensions/main.go new file mode 100644 index 00000000000..1d8bbce03f3 --- /dev/null +++ b/pkg/extensions/main.go @@ -0,0 +1,7 @@ +package extensions + +import ( + _ "gopkg.in/square/go-jose.v2" +) + +var IsEnterprise bool = false diff --git a/pkg/log/file.go b/pkg/log/file.go index 721db1e55b3..b8430dc6086 100644 --- a/pkg/log/file.go +++ b/pkg/log/file.go @@ -99,10 +99,7 @@ func (w *FileLogWriter) StartLogger() error { return err } w.mw.SetFd(fd) - if err = w.initFd(); err != nil { - return err - } - return nil + return w.initFd() } func (w *FileLogWriter) docheck(size int) { @@ -239,3 +236,20 @@ func (w *FileLogWriter) Close() { func (w *FileLogWriter) Flush() { w.mw.fd.Sync() } + +// Reload file logger +func (w *FileLogWriter) Reload() { + // block Logger's io.Writer + w.mw.Lock() + defer w.mw.Unlock() + + // Close + fd := w.mw.fd + fd.Close() + + // Open again + err := w.StartLogger() + if err != nil { + fmt.Fprintf(os.Stderr, "Reload StartLogger: %s\n", err) + } +} diff --git a/pkg/log/file_test.go b/pkg/log/file_test.go index 3e98e0786cc..97a3b8fe82f 100644 --- a/pkg/log/file_test.go +++ b/pkg/log/file_test.go @@ -32,7 +32,9 @@ func TestLogFile(t *testing.T) { Convey("Logging should add lines", func() { err := fileLogWrite.WriteLine("test1\n") + So(err, ShouldBeNil) err = fileLogWrite.WriteLine("test2\n") + So(err, ShouldBeNil) err = fileLogWrite.WriteLine("test3\n") So(err, ShouldBeNil) So(fileLogWrite.maxlines_curlines, ShouldEqual, 3) diff --git a/pkg/log/handlers.go b/pkg/log/handlers.go index 14a96fdcdb4..804d8fcbd70 100644 --- a/pkg/log/handlers.go +++ b/pkg/log/handlers.go @@ -3,3 +3,7 @@ package log type DisposableHandler interface { Close() } + +type ReloadableHandler interface { + Reload() +} diff --git a/pkg/log/log.go b/pkg/log/log.go index 0e6874e1b4b..8f0522748ef 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -21,10 +21,12 @@ import ( var Root log15.Logger var loggersToClose []DisposableHandler +var loggersToReload []ReloadableHandler var filters map[string]log15.Lvl func init() { loggersToClose = make([]DisposableHandler, 0) + loggersToReload = make([]ReloadableHandler, 0) Root = log15.Root() Root.SetHandler(log15.DiscardHandler()) } @@ -103,7 +105,7 @@ func Critical(skip int, format string, v ...interface{}) { } func Fatal(skip int, format string, v ...interface{}) { - Root.Crit(fmt.Sprintf(format, v)) + Root.Crit(fmt.Sprintf(format, v...)) Close() os.Exit(1) } @@ -115,6 +117,12 @@ func Close() { loggersToClose = make([]DisposableHandler, 0) } +func Reload() { + for _, logger := range loggersToReload { + logger.Reload() + } +} + func GetLogLevelFor(name string) Lvl { if level, ok := filters[name]; ok { switch level { @@ -230,6 +238,7 @@ func ReadLoggingConfig(modes []string, logsPath string, cfg *ini.File) { fileHandler.Init() loggersToClose = append(loggersToClose, fileHandler) + loggersToReload = append(loggersToReload, fileHandler) handler = fileHandler case "syslog": sysLogHandler := NewSyslog(sec, format) diff --git a/pkg/login/auth.go b/pkg/login/auth.go index 5527c7271d6..b766d963328 100644 --- a/pkg/login/auth.go +++ b/pkg/login/auth.go @@ -8,27 +8,31 @@ import ( ) var ( - ErrInvalidCredentials = errors.New("Invalid Username or Password") - ErrTooManyLoginAttempts = errors.New("Too many consecutive incorrect login attempts for user. Login for user temporarily blocked") + ErrEmailNotAllowed = errors.New("Required email domain not fulfilled") + ErrInvalidCredentials = errors.New("Invalid Username or Password") + ErrNoEmail = errors.New("Login provider didn't return an email address") + ErrProviderDeniedRequest = errors.New("Login provider denied login request") + ErrSignUpNotAllowed = errors.New("Signup is not allowed for this adapter") + ErrTooManyLoginAttempts = errors.New("Too many consecutive incorrect login attempts for user. Login for user temporarily blocked") + ErrPasswordEmpty = errors.New("No password provided.") + ErrUsersQuotaReached = errors.New("Users quota reached") + ErrGettingUserQuota = errors.New("Error getting user quota") ) -type LoginUserQuery struct { - Username string - Password string - User *m.User - IpAddress string -} - func Init() { bus.AddHandler("auth", AuthenticateUser) loadLdapConfig() } -func AuthenticateUser(query *LoginUserQuery) error { +func AuthenticateUser(query *m.LoginUserQuery) error { if err := validateLoginAttempts(query.Username); err != nil { return err } + if err := validatePasswordSet(query.Password); err != nil { + return err + } + err := loginUsingGrafanaDB(query) if err == nil || (err != m.ErrUserNotFound && err != ErrInvalidCredentials) { return err @@ -53,3 +57,10 @@ func AuthenticateUser(query *LoginUserQuery) error { return err } +func validatePasswordSet(password string) error { + if len(password) == 0 { + return ErrPasswordEmpty + } + + return nil +} diff --git a/pkg/login/auth_test.go b/pkg/login/auth_test.go index 59d3c8f2b33..a4cd8284cdd 100644 --- a/pkg/login/auth_test.go +++ b/pkg/login/auth_test.go @@ -10,6 +10,24 @@ import ( func TestAuthenticateUser(t *testing.T) { Convey("Authenticate user", t, func() { + authScenario("When a user authenticates without setting a password", func(sc *authScenarioContext) { + mockLoginAttemptValidation(nil, sc) + mockLoginUsingGrafanaDB(nil, sc) + mockLoginUsingLdap(false, nil, sc) + + loginQuery := m.LoginUserQuery{ + Username: "user", + Password: "", + } + err := AuthenticateUser(&loginQuery) + + Convey("login should fail", func() { + So(sc.grafanaLoginWasCalled, ShouldBeFalse) + So(sc.ldapLoginWasCalled, ShouldBeFalse) + So(err, ShouldEqual, ErrPasswordEmpty) + }) + }) + authScenario("When a user authenticates having too many login attempts", func(sc *authScenarioContext) { mockLoginAttemptValidation(ErrTooManyLoginAttempts, sc) mockLoginUsingGrafanaDB(nil, sc) @@ -151,7 +169,7 @@ func TestAuthenticateUser(t *testing.T) { } type authScenarioContext struct { - loginUserQuery *LoginUserQuery + loginUserQuery *m.LoginUserQuery grafanaLoginWasCalled bool ldapLoginWasCalled bool loginAttemptValidationWasCalled bool @@ -161,14 +179,14 @@ type authScenarioContext struct { type authScenarioFunc func(sc *authScenarioContext) func mockLoginUsingGrafanaDB(err error, sc *authScenarioContext) { - loginUsingGrafanaDB = func(query *LoginUserQuery) error { + loginUsingGrafanaDB = func(query *m.LoginUserQuery) error { sc.grafanaLoginWasCalled = true return err } } func mockLoginUsingLdap(enabled bool, err error, sc *authScenarioContext) { - loginUsingLdap = func(query *LoginUserQuery) (bool, error) { + loginUsingLdap = func(query *m.LoginUserQuery) (bool, error) { sc.ldapLoginWasCalled = true return enabled, err } @@ -182,7 +200,7 @@ func mockLoginAttemptValidation(err error, sc *authScenarioContext) { } func mockSaveInvalidLoginAttempt(sc *authScenarioContext) { - saveInvalidLoginAttempt = func(query *LoginUserQuery) { + saveInvalidLoginAttempt = func(query *m.LoginUserQuery) { sc.saveInvalidLoginAttemptWasCalled = true } } @@ -195,7 +213,7 @@ func authScenario(desc string, fn authScenarioFunc) { origSaveInvalidLoginAttempt := saveInvalidLoginAttempt sc := &authScenarioContext{ - loginUserQuery: &LoginUserQuery{ + loginUserQuery: &m.LoginUserQuery{ Username: "user", Password: "pwd", IpAddress: "192.168.1.1:56433", diff --git a/pkg/login/brute_force_login_protection.go b/pkg/login/brute_force_login_protection.go index 2ea93979c7a..d524c420540 100644 --- a/pkg/login/brute_force_login_protection.go +++ b/pkg/login/brute_force_login_protection.go @@ -9,8 +9,8 @@ import ( ) var ( - maxInvalidLoginAttempts int64 = 5 - loginAttemptsWindow time.Duration = time.Minute * 5 + maxInvalidLoginAttempts int64 = 5 + loginAttemptsWindow = time.Minute * 5 ) var validateLoginAttempts = func(username string) error { @@ -34,7 +34,7 @@ var validateLoginAttempts = func(username string) error { return nil } -var saveInvalidLoginAttempt = func(query *LoginUserQuery) { +var saveInvalidLoginAttempt = func(query *m.LoginUserQuery) { if setting.DisableBruteForceLoginProtection { return } diff --git a/pkg/login/brute_force_login_protection_test.go b/pkg/login/brute_force_login_protection_test.go index 5375134ba88..aca100760c7 100644 --- a/pkg/login/brute_force_login_protection_test.go +++ b/pkg/login/brute_force_login_protection_test.go @@ -50,7 +50,7 @@ func TestLoginAttemptsValidation(t *testing.T) { return nil }) - saveInvalidLoginAttempt(&LoginUserQuery{ + saveInvalidLoginAttempt(&m.LoginUserQuery{ Username: "user", Password: "pwd", IpAddress: "192.168.1.1:56433", @@ -103,7 +103,7 @@ func TestLoginAttemptsValidation(t *testing.T) { return nil }) - saveInvalidLoginAttempt(&LoginUserQuery{ + saveInvalidLoginAttempt(&m.LoginUserQuery{ Username: "user", Password: "pwd", IpAddress: "192.168.1.1:56433", diff --git a/pkg/login/ext_user.go b/pkg/login/ext_user.go new file mode 100644 index 00000000000..1262c1cc44f --- /dev/null +++ b/pkg/login/ext_user.go @@ -0,0 +1,207 @@ +package login + +import ( + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/quota" +) + +func init() { + bus.AddHandler("auth", UpsertUser) +} + +func UpsertUser(cmd *m.UpsertUserCommand) error { + extUser := cmd.ExternalUser + + userQuery := &m.GetUserByAuthInfoQuery{ + AuthModule: extUser.AuthModule, + AuthId: extUser.AuthId, + UserId: extUser.UserId, + Email: extUser.Email, + Login: extUser.Login, + } + + err := bus.Dispatch(userQuery) + if err != m.ErrUserNotFound && err != nil { + return err + } + + if err != nil { + if !cmd.SignupAllowed { + log.Warn("Not allowing %s login, user not found in internal user database and allow signup = false", extUser.AuthModule) + return ErrInvalidCredentials + } + + limitReached, err := quota.QuotaReached(cmd.ReqContext, "user") + if err != nil { + log.Warn("Error getting user quota. error: %v", err) + return ErrGettingUserQuota + } + if limitReached { + return ErrUsersQuotaReached + } + + cmd.Result, err = createUser(extUser) + if err != nil { + return err + } + + if extUser.AuthModule != "" && extUser.AuthId != "" { + cmd2 := &m.SetAuthInfoCommand{ + UserId: cmd.Result.Id, + AuthModule: extUser.AuthModule, + AuthId: extUser.AuthId, + } + if err := bus.Dispatch(cmd2); err != nil { + return err + } + } + + } else { + cmd.Result = userQuery.Result + + err = updateUser(cmd.Result, extUser) + if err != nil { + return err + } + } + + err = syncOrgRoles(cmd.Result, extUser) + if err != nil { + return err + } + + // Sync isGrafanaAdmin permission + if extUser.IsGrafanaAdmin != nil && *extUser.IsGrafanaAdmin != cmd.Result.IsAdmin { + if err := bus.Dispatch(&m.UpdateUserPermissionsCommand{UserId: cmd.Result.Id, IsGrafanaAdmin: *extUser.IsGrafanaAdmin}); err != nil { + return err + } + } + + err = bus.Dispatch(&m.SyncTeamsCommand{ + User: cmd.Result, + ExternalUser: extUser, + }) + + if err == bus.ErrHandlerNotFound { + return nil + } + + return err +} + +func createUser(extUser *m.ExternalUserInfo) (*m.User, error) { + cmd := &m.CreateUserCommand{ + Login: extUser.Login, + Email: extUser.Email, + Name: extUser.Name, + SkipOrgSetup: len(extUser.OrgRoles) > 0, + } + + if err := bus.Dispatch(cmd); err != nil { + return nil, err + } + + return &cmd.Result, nil +} + +func updateUser(user *m.User, extUser *m.ExternalUserInfo) error { + // sync user info + updateCmd := &m.UpdateUserCommand{ + UserId: user.Id, + } + + needsUpdate := false + if extUser.Login != "" && extUser.Login != user.Login { + updateCmd.Login = extUser.Login + user.Login = extUser.Login + needsUpdate = true + } + + if extUser.Email != "" && extUser.Email != user.Email { + updateCmd.Email = extUser.Email + user.Email = extUser.Email + needsUpdate = true + } + + if extUser.Name != "" && extUser.Name != user.Name { + updateCmd.Name = extUser.Name + user.Name = extUser.Name + needsUpdate = true + } + + if !needsUpdate { + return nil + } + + log.Debug2("Syncing user info", "id", user.Id, "update", updateCmd) + return bus.Dispatch(updateCmd) +} + +func syncOrgRoles(user *m.User, extUser *m.ExternalUserInfo) error { + // don't sync org roles if none are specified + if len(extUser.OrgRoles) == 0 { + return nil + } + + orgsQuery := &m.GetUserOrgListQuery{UserId: user.Id} + if err := bus.Dispatch(orgsQuery); err != nil { + return err + } + + handledOrgIds := map[int64]bool{} + deleteOrgIds := []int64{} + + // update existing org roles + for _, org := range orgsQuery.Result { + handledOrgIds[org.OrgId] = true + + if extUser.OrgRoles[org.OrgId] == "" { + deleteOrgIds = append(deleteOrgIds, org.OrgId) + } else if extUser.OrgRoles[org.OrgId] != org.Role { + // update role + cmd := &m.UpdateOrgUserCommand{OrgId: org.OrgId, UserId: user.Id, Role: extUser.OrgRoles[org.OrgId]} + if err := bus.Dispatch(cmd); err != nil { + return err + } + } + } + + // add any new org roles + for orgId, orgRole := range extUser.OrgRoles { + if _, exists := handledOrgIds[orgId]; exists { + continue + } + + // add role + cmd := &m.AddOrgUserCommand{UserId: user.Id, Role: orgRole, OrgId: orgId} + err := bus.Dispatch(cmd) + if err != nil && err != m.ErrOrgNotFound { + return err + } + } + + // delete any removed org roles + for _, orgId := range deleteOrgIds { + cmd := &m.RemoveOrgUserCommand{OrgId: orgId, UserId: user.Id} + if err := bus.Dispatch(cmd); err != nil { + return err + } + } + + // update user's default org if needed + if _, ok := extUser.OrgRoles[user.OrgId]; !ok { + for orgId := range extUser.OrgRoles { + user.OrgId = orgId + break + } + + return bus.Dispatch(&m.SetUsingOrgCommand{ + UserId: user.Id, + OrgId: user.OrgId, + }) + } + + return nil +} diff --git a/pkg/login/grafana_login.go b/pkg/login/grafana_login.go index 677ba776e4f..e8594fdd190 100644 --- a/pkg/login/grafana_login.go +++ b/pkg/login/grafana_login.go @@ -17,7 +17,7 @@ var validatePassword = func(providedPassword string, userPassword string, userSa return nil } -var loginUsingGrafanaDB = func(query *LoginUserQuery) error { +var loginUsingGrafanaDB = func(query *m.LoginUserQuery) error { userQuery := m.GetUserByLoginQuery{LoginOrEmail: query.Username} if err := bus.Dispatch(&userQuery); err != nil { diff --git a/pkg/login/grafana_login_test.go b/pkg/login/grafana_login_test.go index 88e52224113..90422678fd2 100644 --- a/pkg/login/grafana_login_test.go +++ b/pkg/login/grafana_login_test.go @@ -66,7 +66,7 @@ func TestGrafanaLogin(t *testing.T) { } type grafanaLoginScenarioContext struct { - loginUserQuery *LoginUserQuery + loginUserQuery *m.LoginUserQuery validatePasswordCalled bool } @@ -77,7 +77,7 @@ func grafanaLoginScenario(desc string, fn grafanaLoginScenarioFunc) { origValidatePassword := validatePassword sc := &grafanaLoginScenarioContext{ - loginUserQuery: &LoginUserQuery{ + loginUserQuery: &m.LoginUserQuery{ Username: "user", Password: "pwd", IpAddress: "192.168.1.1:56433", diff --git a/pkg/login/ldap.go b/pkg/login/ldap.go index be3babac02e..d4e81d2bd46 100644 --- a/pkg/login/ldap.go +++ b/pkg/login/ldap.go @@ -24,10 +24,9 @@ type ILdapConn interface { } type ILdapAuther interface { - Login(query *LoginUserQuery) error - SyncSignedInUser(signedInUser *m.SignedInUser) error - GetGrafanaUserFor(ldapUser *LdapUserInfo) (*m.User, error) - SyncOrgRoles(user *m.User, ldapUser *LdapUserInfo) error + Login(query *m.LoginUserQuery) error + SyncUser(query *m.LoginUserQuery) error + GetGrafanaUserFor(ctx *m.ReqContext, ldapUser *LdapUserInfo) (*m.User, error) } type ldapAuther struct { @@ -51,13 +50,20 @@ func (a *ldapAuther) Dial() error { if a.server.RootCACert != "" { certPool = x509.NewCertPool() for _, caCertFile := range strings.Split(a.server.RootCACert, " ") { - if pem, err := ioutil.ReadFile(caCertFile); err != nil { + pem, err := ioutil.ReadFile(caCertFile) + if err != nil { return err - } else { - if !certPool.AppendCertsFromPEM(pem) { - return errors.New("Failed to append CA certificate " + caCertFile) - } } + if !certPool.AppendCertsFromPEM(pem) { + return errors.New("Failed to append CA certificate " + caCertFile) + } + } + } + var clientCert tls.Certificate + if a.server.ClientCert != "" && a.server.ClientKey != "" { + clientCert, err = tls.LoadX509KeyPair(a.server.ClientCert, a.server.ClientKey) + if err != nil { + return err } } for _, host := range strings.Split(a.server.Host, " ") { @@ -68,6 +74,9 @@ func (a *ldapAuther) Dial() error { ServerName: host, RootCAs: certPool, } + if len(clientCert.Certificate) > 0 { + tlsCfg.Certificates = append(tlsCfg.Certificates, clientCert) + } if a.server.StartTLS { a.conn, err = ldap.Dial("tcp", address) if err == nil { @@ -89,7 +98,8 @@ func (a *ldapAuther) Dial() error { return err } -func (a *ldapAuther) Login(query *LoginUserQuery) error { +func (a *ldapAuther) Login(query *m.LoginUserQuery) error { + // connect to ldap server if err := a.Dial(); err != nil { return err } @@ -101,206 +111,110 @@ func (a *ldapAuther) Login(query *LoginUserQuery) error { } // find user entry & attributes - if ldapUser, err := a.searchForUser(query.Username); err != nil { + ldapUser, err := a.searchForUser(query.Username) + if err != nil { return err - } else { - a.log.Debug("Ldap User found", "info", spew.Sdump(ldapUser)) + } - // check if a second user bind is needed - if a.requireSecondBind { - if err := a.secondBind(ldapUser, query.Password); err != nil { - return err - } - } + a.log.Debug("Ldap User found", "info", spew.Sdump(ldapUser)) - if grafanaUser, err := a.GetGrafanaUserFor(ldapUser); err != nil { + // check if a second user bind is needed + if a.requireSecondBind { + err = a.secondBind(ldapUser, query.Password) + if err != nil { return err - } else { - if syncErr := a.syncInfoAndOrgRoles(grafanaUser, ldapUser); syncErr != nil { - return syncErr - } - query.User = grafanaUser - return nil } } + + grafanaUser, err := a.GetGrafanaUserFor(query.ReqContext, ldapUser) + if err != nil { + return err + } + + query.User = grafanaUser + return nil } -func (a *ldapAuther) SyncSignedInUser(signedInUser *m.SignedInUser) error { - grafanaUser := m.User{ - Id: signedInUser.UserId, - Login: signedInUser.Login, - Email: signedInUser.Email, - Name: signedInUser.Name, - } - - if err := a.Dial(); err != nil { +func (a *ldapAuther) SyncUser(query *m.LoginUserQuery) error { + // connect to ldap server + err := a.Dial() + if err != nil { return err } - defer a.conn.Close() - if err := a.serverBind(); err != nil { + + err = a.serverBind() + if err != nil { return err } - if ldapUser, err := a.searchForUser(signedInUser.Login); err != nil { + // find user entry & attributes + ldapUser, err := a.searchForUser(query.Username) + if err != nil { a.log.Error("Failed searching for user in ldap", "error", err) - return err - } else { - if err := a.syncInfoAndOrgRoles(&grafanaUser, ldapUser); err != nil { - return err + } + + a.log.Debug("Ldap User found", "info", spew.Sdump(ldapUser)) + + grafanaUser, err := a.GetGrafanaUserFor(query.ReqContext, ldapUser) + if err != nil { + return err + } + + query.User = grafanaUser + return nil +} + +func (a *ldapAuther) GetGrafanaUserFor(ctx *m.ReqContext, ldapUser *LdapUserInfo) (*m.User, error) { + extUser := &m.ExternalUserInfo{ + AuthModule: "ldap", + AuthId: ldapUser.DN, + Name: fmt.Sprintf("%s %s", ldapUser.FirstName, ldapUser.LastName), + Login: ldapUser.Username, + Email: ldapUser.Email, + Groups: ldapUser.MemberOf, + OrgRoles: map[int64]m.RoleType{}, + } + + for _, group := range a.server.LdapGroups { + // only use the first match for each org + if extUser.OrgRoles[group.OrgId] != "" { + continue } - a.log.Debug("Got Ldap User Info", "user", spew.Sdump(ldapUser)) + if ldapUser.isMemberOf(group.GroupDN) { + extUser.OrgRoles[group.OrgId] = group.OrgRole + if extUser.IsGrafanaAdmin == nil || !*extUser.IsGrafanaAdmin { + extUser.IsGrafanaAdmin = group.IsGrafanaAdmin + } + } } - return nil -} - -// Sync info for ldap user and grafana user -func (a *ldapAuther) syncInfoAndOrgRoles(user *m.User, ldapUser *LdapUserInfo) error { - // sync user details - if err := a.syncUserInfo(user, ldapUser); err != nil { - return err - } - // sync org roles - if err := a.SyncOrgRoles(user, ldapUser); err != nil { - return err - } - - return nil -} - -func (a *ldapAuther) GetGrafanaUserFor(ldapUser *LdapUserInfo) (*m.User, error) { // validate that the user has access // if there are no ldap group mappings access is true // otherwise a single group must match - access := len(a.server.LdapGroups) == 0 - for _, ldapGroup := range a.server.LdapGroups { - if ldapUser.isMemberOf(ldapGroup.GroupDN) { - access = true - break - } - } - - if !access { - a.log.Info("Ldap Auth: user does not belong in any of the specified ldap groups", "username", ldapUser.Username, "groups", ldapUser.MemberOf) + if len(a.server.LdapGroups) > 0 && len(extUser.OrgRoles) < 1 { + a.log.Info( + "Ldap Auth: user does not belong in any of the specified ldap groups", + "username", ldapUser.Username, + "groups", ldapUser.MemberOf) return nil, ErrInvalidCredentials } - // get user from grafana db - userQuery := m.GetUserByLoginQuery{LoginOrEmail: ldapUser.Username} - if err := bus.Dispatch(&userQuery); err != nil { - if err == m.ErrUserNotFound && setting.LdapAllowSignup { - return a.createGrafanaUser(ldapUser) - } else if err == m.ErrUserNotFound { - a.log.Warn("Not allowing LDAP login, user not found in internal user database, and ldap allow signup = false") - return nil, ErrInvalidCredentials - } else { - return nil, err - } + // add/update user in grafana + upsertUserCmd := &m.UpsertUserCommand{ + ReqContext: ctx, + ExternalUser: extUser, + SignupAllowed: setting.LdapAllowSignup, } - return userQuery.Result, nil - -} -func (a *ldapAuther) createGrafanaUser(ldapUser *LdapUserInfo) (*m.User, error) { - cmd := m.CreateUserCommand{ - Login: ldapUser.Username, - Email: ldapUser.Email, - Name: fmt.Sprintf("%s %s", ldapUser.FirstName, ldapUser.LastName), - } - - if err := bus.Dispatch(&cmd); err != nil { + err := bus.Dispatch(upsertUserCmd) + if err != nil { return nil, err } - return &cmd.Result, nil -} - -func (a *ldapAuther) syncUserInfo(user *m.User, ldapUser *LdapUserInfo) error { - var name = fmt.Sprintf("%s %s", ldapUser.FirstName, ldapUser.LastName) - if user.Email == ldapUser.Email && user.Name == name { - return nil - } - - a.log.Debug("Syncing user info", "username", ldapUser.Username) - updateCmd := m.UpdateUserCommand{} - updateCmd.UserId = user.Id - updateCmd.Login = user.Login - updateCmd.Email = ldapUser.Email - updateCmd.Name = fmt.Sprintf("%s %s", ldapUser.FirstName, ldapUser.LastName) - return bus.Dispatch(&updateCmd) -} - -func (a *ldapAuther) SyncOrgRoles(user *m.User, ldapUser *LdapUserInfo) error { - if len(a.server.LdapGroups) == 0 { - a.log.Warn("No group mappings defined") - return nil - } - - orgsQuery := m.GetUserOrgListQuery{UserId: user.Id} - if err := bus.Dispatch(&orgsQuery); err != nil { - return err - } - - handledOrgIds := map[int64]bool{} - - // update or remove org roles - for _, org := range orgsQuery.Result { - match := false - handledOrgIds[org.OrgId] = true - - for _, group := range a.server.LdapGroups { - if org.OrgId != group.OrgId { - continue - } - - if ldapUser.isMemberOf(group.GroupDN) { - match = true - if org.Role != group.OrgRole { - // update role - cmd := m.UpdateOrgUserCommand{OrgId: org.OrgId, UserId: user.Id, Role: group.OrgRole} - if err := bus.Dispatch(&cmd); err != nil { - return err - } - } - // ignore subsequent ldap group mapping matches - break - } - } - - // remove role if no mappings match - if !match { - cmd := m.RemoveOrgUserCommand{OrgId: org.OrgId, UserId: user.Id} - if err := bus.Dispatch(&cmd); err != nil { - return err - } - } - } - - // add missing org roles - for _, group := range a.server.LdapGroups { - if !ldapUser.isMemberOf(group.GroupDN) { - continue - } - - if _, exists := handledOrgIds[group.OrgId]; exists { - continue - } - - // add role - cmd := m.AddOrgUserCommand{UserId: user.Id, Role: group.OrgRole, OrgId: group.OrgId} - err := bus.Dispatch(&cmd) - if err != nil && err != m.ErrOrgNotFound { - return err - } - - // mark this group has handled so we do not process it again - handledOrgIds[group.OrgId] = true - } - - return nil + return upsertUserCmd.Result, nil } func (a *ldapAuther) serverBind() error { @@ -404,23 +318,29 @@ func (a *ldapAuther) searchForUser(username string) (*LdapUserInfo, error) { var groupSearchResult *ldap.SearchResult for _, groupSearchBase := range a.server.GroupSearchBaseDNs { var filter_replace string - filter_replace = getLdapAttr(a.server.GroupSearchFilterUserAttribute, searchResult) if a.server.GroupSearchFilterUserAttribute == "" { filter_replace = getLdapAttr(a.server.Attr.Username, searchResult) + } else { + filter_replace = getLdapAttr(a.server.GroupSearchFilterUserAttribute, searchResult) } + filter := strings.Replace(a.server.GroupSearchFilter, "%s", ldap.EscapeFilter(filter_replace), -1) a.log.Info("Searching for user's groups", "filter", filter) + // support old way of reading settings + groupIdAttribute := a.server.Attr.MemberOf + // but prefer dn attribute if default settings are used + if groupIdAttribute == "" || groupIdAttribute == "memberOf" { + groupIdAttribute = "dn" + } + groupSearchReq := ldap.SearchRequest{ BaseDN: groupSearchBase, Scope: ldap.ScopeWholeSubtree, DerefAliases: ldap.NeverDerefAliases, - Attributes: []string{ - // Here MemberOf would be the thing that identifies the group, which is normally 'cn' - a.server.Attr.MemberOf, - }, - Filter: filter, + Attributes: []string{groupIdAttribute}, + Filter: filter, } groupSearchResult, err = a.conn.Search(&groupSearchReq) @@ -430,7 +350,7 @@ func (a *ldapAuther) searchForUser(username string) (*LdapUserInfo, error) { if len(groupSearchResult.Entries) > 0 { for i := range groupSearchResult.Entries { - memberOf = append(memberOf, getLdapAttrN(a.server.Attr.MemberOf, groupSearchResult, i)) + memberOf = append(memberOf, getLdapAttrN(groupIdAttribute, groupSearchResult, i)) } break } @@ -448,6 +368,9 @@ func (a *ldapAuther) searchForUser(username string) (*LdapUserInfo, error) { } func getLdapAttrN(name string, result *ldap.SearchResult, n int) string { + if strings.ToLower(name) == "dn" { + return result.Entries[n].DN + } for _, attr := range result.Entries[n].Attributes { if attr.Name == name { if len(attr.Values) > 0 { @@ -470,7 +393,3 @@ func getLdapAttrArray(name string, result *ldap.SearchResult) []string { } return []string{} } - -func createUserFromLdapInfo() error { - return nil -} diff --git a/pkg/login/ldap_login.go b/pkg/login/ldap_login.go index b74b69db036..5974e19d691 100644 --- a/pkg/login/ldap_login.go +++ b/pkg/login/ldap_login.go @@ -1,10 +1,11 @@ package login import ( + m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" ) -var loginUsingLdap = func(query *LoginUserQuery) (bool, error) { +var loginUsingLdap = func(query *m.LoginUserQuery) (bool, error) { if !setting.LdapEnabled { return false, nil } diff --git a/pkg/login/ldap_login_test.go b/pkg/login/ldap_login_test.go index 6af125566e8..6067a063795 100644 --- a/pkg/login/ldap_login_test.go +++ b/pkg/login/ldap_login_test.go @@ -79,7 +79,7 @@ func TestLdapLogin(t *testing.T) { ldapLoginScenario("When login", func(sc *ldapLoginScenarioContext) { sc.withLoginResult(false) - enabled, err := loginUsingLdap(&LoginUserQuery{ + enabled, err := loginUsingLdap(&m.LoginUserQuery{ Username: "user", Password: "pwd", }) @@ -117,7 +117,7 @@ type mockLdapAuther struct { loginCalled bool } -func (a *mockLdapAuther) Login(query *LoginUserQuery) error { +func (a *mockLdapAuther) Login(query *m.LoginUserQuery) error { a.loginCalled = true if !a.validLogin { @@ -127,20 +127,16 @@ func (a *mockLdapAuther) Login(query *LoginUserQuery) error { return nil } -func (a *mockLdapAuther) SyncSignedInUser(signedInUser *m.SignedInUser) error { +func (a *mockLdapAuther) SyncUser(query *m.LoginUserQuery) error { return nil } -func (a *mockLdapAuther) GetGrafanaUserFor(ldapUser *LdapUserInfo) (*m.User, error) { +func (a *mockLdapAuther) GetGrafanaUserFor(ctx *m.ReqContext, ldapUser *LdapUserInfo) (*m.User, error) { return nil, nil } -func (a *mockLdapAuther) SyncOrgRoles(user *m.User, ldapUser *LdapUserInfo) error { - return nil -} - type ldapLoginScenarioContext struct { - loginUserQuery *LoginUserQuery + loginUserQuery *m.LoginUserQuery ldapAuthenticatorMock *mockLdapAuther } @@ -151,7 +147,7 @@ func ldapLoginScenario(desc string, fn ldapLoginScenarioFunc) { origNewLdapAuthenticator := NewLdapAuthenticator sc := &ldapLoginScenarioContext{ - loginUserQuery: &LoginUserQuery{ + loginUserQuery: &m.LoginUserQuery{ Username: "user", Password: "pwd", IpAddress: "192.168.1.1:56433", diff --git a/pkg/login/ldap_settings.go b/pkg/login/ldap_settings.go index 497d8725e29..40791a509db 100644 --- a/pkg/login/ldap_settings.go +++ b/pkg/login/ldap_settings.go @@ -21,6 +21,8 @@ type LdapServerConf struct { StartTLS bool `toml:"start_tls"` SkipVerifySSL bool `toml:"ssl_skip_verify"` RootCACert string `toml:"root_ca_cert"` + ClientCert string `toml:"client_cert"` + ClientKey string `toml:"client_key"` BindDN string `toml:"bind_dn"` BindPassword string `toml:"bind_password"` Attr LdapAttributeMap `toml:"attributes"` @@ -44,9 +46,10 @@ type LdapAttributeMap struct { } type LdapGroupToOrgRole struct { - GroupDN string `toml:"group_dn"` - OrgId int64 `toml:"org_id"` - OrgRole m.RoleType `toml:"org_role"` + GroupDN string `toml:"group_dn"` + OrgId int64 `toml:"org_id"` + IsGrafanaAdmin *bool `toml:"grafana_admin"` // This is a pointer to know if it was set or not (for backwards compatibility) + OrgRole m.RoleType `toml:"org_role"` } var LdapCfg LdapConfig diff --git a/pkg/login/ldap_test.go b/pkg/login/ldap_test.go index 8677bbeae42..1cf98bd1e14 100644 --- a/pkg/login/ldap_test.go +++ b/pkg/login/ldap_test.go @@ -1,6 +1,7 @@ package login import ( + "context" "crypto/tls" "testing" @@ -14,17 +15,23 @@ func TestLdapAuther(t *testing.T) { Convey("When translating ldap user to grafana user", t, func() { + var user1 = &m.User{} + + bus.AddHandlerCtx("test", func(ctx context.Context, cmd *m.UpsertUserCommand) error { + cmd.Result = user1 + cmd.Result.Login = "torkelo" + return nil + }) + Convey("Given no ldap group map match", func() { ldapAuther := NewLdapAuthenticator(&LdapServerConf{ LdapGroups: []*LdapGroupToOrgRole{{}}, }) - _, err := ldapAuther.GetGrafanaUserFor(&LdapUserInfo{}) + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{}) So(err, ShouldEqual, ErrInvalidCredentials) }) - var user1 = &m.User{} - ldapAutherScenario("Given wildcard group match", func(sc *scenarioContext) { ldapAuther := NewLdapAuthenticator(&LdapServerConf{ LdapGroups: []*LdapGroupToOrgRole{ @@ -34,7 +41,7 @@ func TestLdapAuther(t *testing.T) { sc.userQueryReturns(user1) - result, err := ldapAuther.GetGrafanaUserFor(&LdapUserInfo{}) + result, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{}) So(err, ShouldBeNil) So(result, ShouldEqual, user1) }) @@ -48,7 +55,21 @@ func TestLdapAuther(t *testing.T) { sc.userQueryReturns(user1) - result, err := ldapAuther.GetGrafanaUserFor(&LdapUserInfo{MemberOf: []string{"cn=users"}}) + result, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{MemberOf: []string{"cn=users"}}) + So(err, ShouldBeNil) + So(result, ShouldEqual, user1) + }) + + ldapAutherScenario("Given group match with different case", func(sc *scenarioContext) { + ldapAuther := NewLdapAuthenticator(&LdapServerConf{ + LdapGroups: []*LdapGroupToOrgRole{ + {GroupDN: "cn=users", OrgRole: "Admin"}, + }, + }) + + sc.userQueryReturns(user1) + + result, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{MemberOf: []string{"CN=users"}}) So(err, ShouldBeNil) So(result, ShouldEqual, user1) }) @@ -64,7 +85,8 @@ func TestLdapAuther(t *testing.T) { sc.userQueryReturns(nil) - result, err := ldapAuther.GetGrafanaUserFor(&LdapUserInfo{ + result, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ + DN: "torkelo", Username: "torkelo", Email: "my@email.com", MemberOf: []string{"cn=editor"}, @@ -72,21 +94,19 @@ func TestLdapAuther(t *testing.T) { So(err, ShouldBeNil) - Convey("Should create new user", func() { - So(sc.createUserCmd.Login, ShouldEqual, "torkelo") - So(sc.createUserCmd.Email, ShouldEqual, "my@email.com") - }) - Convey("Should return new user", func() { So(result.Login, ShouldEqual, "torkelo") }) + Convey("Should set isGrafanaAdmin to false by default", func() { + So(result.IsAdmin, ShouldBeFalse) + }) + }) }) Convey("When syncing ldap groups to grafana org roles", t, func() { - ldapAutherScenario("given no current user orgs", func(sc *scenarioContext) { ldapAuther := NewLdapAuthenticator(&LdapServerConf{ LdapGroups: []*LdapGroupToOrgRole{ @@ -95,7 +115,7 @@ func TestLdapAuther(t *testing.T) { }) sc.userOrgsQueryReturns([]*m.UserOrgDTO{}) - err := ldapAuther.SyncOrgRoles(&m.User{}, &LdapUserInfo{ + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ MemberOf: []string{"cn=users"}, }) @@ -114,7 +134,7 @@ func TestLdapAuther(t *testing.T) { }) sc.userOrgsQueryReturns([]*m.UserOrgDTO{{OrgId: 1, Role: m.ROLE_EDITOR}}) - err := ldapAuther.SyncOrgRoles(&m.User{}, &LdapUserInfo{ + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ MemberOf: []string{"cn=users"}, }) @@ -122,24 +142,29 @@ func TestLdapAuther(t *testing.T) { So(err, ShouldBeNil) So(sc.updateOrgUserCmd, ShouldNotBeNil) So(sc.updateOrgUserCmd.Role, ShouldEqual, m.ROLE_ADMIN) + So(sc.setUsingOrgCmd.OrgId, ShouldEqual, 1) }) }) ldapAutherScenario("given current org role is removed in ldap", func(sc *scenarioContext) { ldapAuther := NewLdapAuthenticator(&LdapServerConf{ LdapGroups: []*LdapGroupToOrgRole{ - {GroupDN: "cn=users", OrgId: 1, OrgRole: "Admin"}, + {GroupDN: "cn=users", OrgId: 2, OrgRole: "Admin"}, }, }) - sc.userOrgsQueryReturns([]*m.UserOrgDTO{{OrgId: 1, Role: m.ROLE_EDITOR}}) - err := ldapAuther.SyncOrgRoles(&m.User{}, &LdapUserInfo{ - MemberOf: []string{"cn=other"}, + sc.userOrgsQueryReturns([]*m.UserOrgDTO{ + {OrgId: 1, Role: m.ROLE_EDITOR}, + {OrgId: 2, Role: m.ROLE_EDITOR}, + }) + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ + MemberOf: []string{"cn=users"}, }) Convey("Should remove org role", func() { So(err, ShouldBeNil) So(sc.removeOrgUserCmd, ShouldNotBeNil) + So(sc.setUsingOrgCmd.OrgId, ShouldEqual, 2) }) }) @@ -152,7 +177,7 @@ func TestLdapAuther(t *testing.T) { }) sc.userOrgsQueryReturns([]*m.UserOrgDTO{{OrgId: 1, Role: m.ROLE_EDITOR}}) - err := ldapAuther.SyncOrgRoles(&m.User{}, &LdapUserInfo{ + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ MemberOf: []string{"cn=users"}, }) @@ -160,6 +185,7 @@ func TestLdapAuther(t *testing.T) { So(err, ShouldBeNil) So(sc.removeOrgUserCmd, ShouldBeNil) So(sc.updateOrgUserCmd, ShouldNotBeNil) + So(sc.setUsingOrgCmd.OrgId, ShouldEqual, 1) }) }) @@ -172,13 +198,14 @@ func TestLdapAuther(t *testing.T) { }) sc.userOrgsQueryReturns([]*m.UserOrgDTO{{OrgId: 1, Role: m.ROLE_ADMIN}}) - err := ldapAuther.SyncOrgRoles(&m.User{}, &LdapUserInfo{ + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ MemberOf: []string{"cn=admins"}, }) Convey("Should take first match, and ignore subsequent matches", func() { So(err, ShouldBeNil) So(sc.updateOrgUserCmd, ShouldBeNil) + So(sc.setUsingOrgCmd.OrgId, ShouldEqual, 1) }) }) @@ -191,19 +218,44 @@ func TestLdapAuther(t *testing.T) { }) sc.userOrgsQueryReturns([]*m.UserOrgDTO{}) - err := ldapAuther.SyncOrgRoles(&m.User{}, &LdapUserInfo{ + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ MemberOf: []string{"cn=admins"}, }) Convey("Should take first match, and ignore subsequent matches", func() { So(err, ShouldBeNil) So(sc.addOrgUserCmd.Role, ShouldEqual, m.ROLE_ADMIN) + So(sc.setUsingOrgCmd.OrgId, ShouldEqual, 1) + }) + + Convey("Should not update permissions unless specified", func() { + So(err, ShouldBeNil) + So(sc.updateUserPermissionsCmd, ShouldBeNil) }) }) + ldapAutherScenario("given ldap groups with grafana_admin=true", func(sc *scenarioContext) { + trueVal := true + + ldapAuther := NewLdapAuthenticator(&LdapServerConf{ + LdapGroups: []*LdapGroupToOrgRole{ + {GroupDN: "cn=admins", OrgId: 1, OrgRole: "Admin", IsGrafanaAdmin: &trueVal}, + }, + }) + + sc.userOrgsQueryReturns([]*m.UserOrgDTO{}) + _, err := ldapAuther.GetGrafanaUserFor(nil, &LdapUserInfo{ + MemberOf: []string{"cn=admins"}, + }) + + Convey("Should create user with admin set to true", func() { + So(err, ShouldBeNil) + So(sc.updateUserPermissionsCmd.IsGrafanaAdmin, ShouldBeTrue) + }) + }) }) - Convey("When calling SyncSignedInUser", t, func() { + Convey("When calling SyncUser", t, func() { mockLdapConnection := &mockLdapConn{} ldapAuther := NewLdapAuthenticator( @@ -243,17 +295,20 @@ func TestLdapAuther(t *testing.T) { ldapAutherScenario("When ldapUser found call syncInfo and orgRoles", func(sc *scenarioContext) { // arrange - signedInUser := &m.SignedInUser{ - Email: "roel@test.net", - UserId: 1, - Name: "Roel Gerrits", - Login: "roelgerrits", + query := &m.LoginUserQuery{ + Username: "roelgerrits", } + sc.userQueryReturns(&m.User{ + Id: 1, + Email: "roel@test.net", + Name: "Roel Gerrits", + Login: "roelgerrits", + }) sc.userOrgsQueryReturns([]*m.UserOrgDTO{}) // act - syncErrResult := ldapAuther.SyncSignedInUser(signedInUser) + syncErrResult := ldapAuther.SyncUser(query) // assert So(dialCalled, ShouldBeTrue) @@ -299,6 +354,28 @@ func ldapAutherScenario(desc string, fn scenarioFunc) { sc := &scenarioContext{} + bus.AddHandler("test", UpsertUser) + + bus.AddHandlerCtx("test", func(ctx context.Context, cmd *m.SyncTeamsCommand) error { + return nil + }) + + bus.AddHandlerCtx("test", func(ctx context.Context, cmd *m.UpdateUserPermissionsCommand) error { + sc.updateUserPermissionsCmd = cmd + return nil + }) + + bus.AddHandler("test", func(cmd *m.GetUserByAuthInfoQuery) error { + sc.getUserByAuthInfoQuery = cmd + sc.getUserByAuthInfoQuery.Result = &m.User{Login: cmd.Login} + return nil + }) + + bus.AddHandler("test", func(cmd *m.GetUserOrgListQuery) error { + sc.getUserOrgListQuery = cmd + return nil + }) + bus.AddHandler("test", func(cmd *m.CreateUserCommand) error { sc.createUserCmd = cmd sc.createUserCmd.Result = m.User{Login: cmd.Login} @@ -325,26 +402,37 @@ func ldapAutherScenario(desc string, fn scenarioFunc) { return nil }) + bus.AddHandler("test", func(cmd *m.SetUsingOrgCommand) error { + sc.setUsingOrgCmd = cmd + return nil + }) + fn(sc) }) } type scenarioContext struct { - createUserCmd *m.CreateUserCommand - addOrgUserCmd *m.AddOrgUserCommand - updateOrgUserCmd *m.UpdateOrgUserCommand - removeOrgUserCmd *m.RemoveOrgUserCommand - updateUserCmd *m.UpdateUserCommand + getUserByAuthInfoQuery *m.GetUserByAuthInfoQuery + getUserOrgListQuery *m.GetUserOrgListQuery + createUserCmd *m.CreateUserCommand + addOrgUserCmd *m.AddOrgUserCommand + updateOrgUserCmd *m.UpdateOrgUserCommand + removeOrgUserCmd *m.RemoveOrgUserCommand + updateUserCmd *m.UpdateUserCommand + setUsingOrgCmd *m.SetUsingOrgCommand + updateUserPermissionsCmd *m.UpdateUserPermissionsCommand } func (sc *scenarioContext) userQueryReturns(user *m.User) { - bus.AddHandler("test", func(query *m.GetUserByLoginQuery) error { + bus.AddHandler("test", func(query *m.GetUserByAuthInfoQuery) error { if user == nil { return m.ErrUserNotFound - } else { - query.Result = user - return nil } + query.Result = user + return nil + }) + bus.AddHandler("test", func(query *m.SetAuthInfoCommand) error { + return nil }) } diff --git a/pkg/login/ldap_user.go b/pkg/login/ldap_user.go index 9f1cf3c96b6..3651d9e5e23 100644 --- a/pkg/login/ldap_user.go +++ b/pkg/login/ldap_user.go @@ -1,5 +1,9 @@ package login +import ( + "strings" +) + type LdapUserInfo struct { DN string FirstName string @@ -15,7 +19,7 @@ func (u *LdapUserInfo) isMemberOf(group string) bool { } for _, member := range u.MemberOf { - if member == group { + if strings.EqualFold(member, group) { return true } } diff --git a/pkg/metrics/graphitebridge/graphite.go b/pkg/metrics/graphitebridge/graphite.go index 68fb544fc7c..5b61f078e6c 100644 --- a/pkg/metrics/graphitebridge/graphite.go +++ b/pkg/metrics/graphitebridge/graphite.go @@ -55,7 +55,7 @@ const ( AbortOnError ) -var metricCategoryPrefix []string = []string{ +var metricCategoryPrefix = []string{ "proxy_", "api_", "page_", @@ -66,7 +66,7 @@ var metricCategoryPrefix []string = []string{ "go_", "process_"} -var trimMetricPrefix []string = []string{"grafana_"} +var trimMetricPrefix = []string{"grafana_"} // Config defines the Graphite bridge config. type Config struct { @@ -295,11 +295,7 @@ func writeMetric(buf *bufio.Writer, m model.Metric, mf *dto.MetricFamily) error } } - if err = addExtentionConventionForRollups(buf, mf, m); err != nil { - return err - } - - return nil + return addExtentionConventionForRollups(buf, mf, m) } func addExtentionConventionForRollups(buf *bufio.Writer, mf *dto.MetricFamily, m model.Metric) error { diff --git a/pkg/metrics/init.go b/pkg/metrics/init.go deleted file mode 100644 index 833b148d319..00000000000 --- a/pkg/metrics/init.go +++ /dev/null @@ -1,38 +0,0 @@ -package metrics - -import ( - "context" - - ini "gopkg.in/ini.v1" - - "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/metrics/graphitebridge" -) - -var metricsLogger log.Logger = log.New("metrics") - -type logWrapper struct { - logger log.Logger -} - -func (lw *logWrapper) Println(v ...interface{}) { - lw.logger.Info("graphite metric bridge", v...) -} - -func Init(file *ini.File) { - cfg := ReadSettings(file) - internalInit(cfg) -} - -func internalInit(settings *MetricSettings) { - initMetricVars(settings) - - if settings.GraphiteBridgeConfig != nil { - bridge, err := graphitebridge.NewBridge(settings.GraphiteBridgeConfig) - if err != nil { - metricsLogger.Error("failed to create graphite bridge", "error", err) - } else { - go bridge.Run(context.Background()) - } - } -} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4d4a11d0faa..5709e3e3213 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -44,6 +44,7 @@ var ( M_Alerting_Notification_Sent *prometheus.CounterVec M_Aws_CloudWatch_GetMetricStatistics prometheus.Counter M_Aws_CloudWatch_ListMetrics prometheus.Counter + M_Aws_CloudWatch_GetMetricData prometheus.Counter M_DB_DataSource_QueryById prometheus.Counter // Timers @@ -54,11 +55,36 @@ var ( M_Alerting_Active_Alerts prometheus.Gauge M_StatTotal_Dashboards prometheus.Gauge M_StatTotal_Users prometheus.Gauge + M_StatActive_Users prometheus.Gauge M_StatTotal_Orgs prometheus.Gauge M_StatTotal_Playlists prometheus.Gauge - M_Grafana_Version *prometheus.GaugeVec + + // M_Grafana_Version is a gauge that contains build info about this binary + // + // Deprecated: use M_Grafana_Build_Version instead. + M_Grafana_Version *prometheus.GaugeVec + + // grafanaBuildVersion is a gauge that contains build info about this binary + grafanaBuildVersion *prometheus.GaugeVec ) +func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec { + counter := prometheus.NewCounterVec(opts, labels) + + for _, label := range labelValues { + counter.WithLabelValues(label).Add(0) + } + + return counter +} + +func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter { + counter := prometheus.NewCounter(opts) + counter.Add(0) + + return counter +} + func init() { M_Instance_Start = prometheus.NewCounter(prometheus.CounterOpts{ Name: "instance_start_total", @@ -66,32 +92,27 @@ func init() { Namespace: exporterName, }) - M_Page_Status = prometheus.NewCounterVec( + httpStatusCodes := []string{"200", "404", "500", "unknown"} + M_Page_Status = newCounterVecStartingAtZero( prometheus.CounterOpts{ Name: "page_response_status_total", Help: "page http response status", Namespace: exporterName, - }, - []string{"code"}, - ) + }, []string{"code"}, httpStatusCodes...) - M_Api_Status = prometheus.NewCounterVec( + M_Api_Status = newCounterVecStartingAtZero( prometheus.CounterOpts{ Name: "api_response_status_total", Help: "api http response status", Namespace: exporterName, - }, - []string{"code"}, - ) + }, []string{"code"}, httpStatusCodes...) - M_Proxy_Status = prometheus.NewCounterVec( + M_Proxy_Status = newCounterVecStartingAtZero( prometheus.CounterOpts{ Name: "proxy_response_status_total", Help: "proxy http response status", Namespace: exporterName, - }, - []string{"code"}, - ) + }, []string{"code"}, httpStatusCodes...) M_Http_Request_Total = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -109,19 +130,19 @@ func init() { []string{"handler", "statuscode", "method"}, ) - M_Api_User_SignUpStarted = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_User_SignUpStarted = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_user_signup_started_total", Help: "amount of users who started the signup flow", Namespace: exporterName, }) - M_Api_User_SignUpCompleted = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_User_SignUpCompleted = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_user_signup_completed_total", Help: "amount of users who completed the signup flow", Namespace: exporterName, }) - M_Api_User_SignUpInvite = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_User_SignUpInvite = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_user_signup_invite_total", Help: "amount of users who have been invited", Namespace: exporterName, @@ -145,49 +166,49 @@ func init() { Namespace: exporterName, }) - M_Api_Admin_User_Create = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Admin_User_Create = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_admin_user_created_total", Help: "api admin user created counter", Namespace: exporterName, }) - M_Api_Login_Post = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Login_Post = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_login_post_total", Help: "api login post counter", Namespace: exporterName, }) - M_Api_Login_OAuth = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Login_OAuth = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_login_oauth_total", Help: "api login oauth counter", Namespace: exporterName, }) - M_Api_Org_Create = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Org_Create = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_org_create_total", Help: "api org created counter", Namespace: exporterName, }) - M_Api_Dashboard_Snapshot_Create = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Dashboard_Snapshot_Create = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_dashboard_snapshot_create_total", Help: "dashboard snapshots created", Namespace: exporterName, }) - M_Api_Dashboard_Snapshot_External = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Dashboard_Snapshot_External = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_dashboard_snapshot_external_total", Help: "external dashboard snapshots created", Namespace: exporterName, }) - M_Api_Dashboard_Snapshot_Get = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Dashboard_Snapshot_Get = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_dashboard_snapshot_get_total", Help: "loaded dashboards", Namespace: exporterName, }) - M_Api_Dashboard_Insert = prometheus.NewCounter(prometheus.CounterOpts{ + M_Api_Dashboard_Insert = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_models_dashboard_insert_total", Help: "dashboards inserted ", Namespace: exporterName, @@ -205,19 +226,25 @@ func init() { Namespace: exporterName, }, []string{"type"}) - M_Aws_CloudWatch_GetMetricStatistics = prometheus.NewCounter(prometheus.CounterOpts{ + M_Aws_CloudWatch_GetMetricStatistics = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "aws_cloudwatch_get_metric_statistics_total", Help: "counter for getting metric statistics from aws", Namespace: exporterName, }) - M_Aws_CloudWatch_ListMetrics = prometheus.NewCounter(prometheus.CounterOpts{ + M_Aws_CloudWatch_ListMetrics = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "aws_cloudwatch_list_metrics_total", Help: "counter for getting list of metrics from aws", Namespace: exporterName, }) - M_DB_DataSource_QueryById = prometheus.NewCounter(prometheus.CounterOpts{ + M_Aws_CloudWatch_GetMetricData = newCounterStartingAtZero(prometheus.CounterOpts{ + Name: "aws_cloudwatch_get_metric_data_total", + Help: "counter for getting metric data time series from aws", + Namespace: exporterName, + }) + + M_DB_DataSource_QueryById = newCounterStartingAtZero(prometheus.CounterOpts{ Name: "db_datasource_query_by_id_total", Help: "counter for getting datasource by id", Namespace: exporterName, @@ -253,6 +280,12 @@ func init() { Namespace: exporterName, }) + M_StatActive_Users = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "stat_active_users", + Help: "number of active users", + Namespace: exporterName, + }) + M_StatTotal_Orgs = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "stat_total_orgs", Help: "total amount of orgs", @@ -267,13 +300,28 @@ func init() { M_Grafana_Version = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "info", - Help: "Information about the Grafana", + Help: "Information about the Grafana. This metric is deprecated. please use `grafana_build_info`", Namespace: exporterName, }, []string{"version"}) + grafanaBuildVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "build_info", + Help: "A metric with a constant '1' value labeled by version, revision, branch, and goversion from which Grafana was built.", + Namespace: exporterName, + }, []string{"version", "revision", "branch", "goversion"}) } -func initMetricVars(settings *MetricSettings) { +// SetBuildInformation sets the build information for this binary +func SetBuildInformation(version, revision, branch string) { + // We export this info twice for backwards compability. + // Once this have been released for some time we should be able to remote `M_Grafana_Version` + // The reason we added a new one is that its common practice in the prometheus community + // to name this metric `*_build_info` so its easy to do aggregation on all programs. + M_Grafana_Version.WithLabelValues(version).Set(1) + grafanaBuildVersion.WithLabelValues(version, revision, branch, runtime.Version()).Set(1) +} + +func initMetricVars() { prometheus.MustRegister( M_Instance_Start, M_Page_Status, @@ -301,52 +349,44 @@ func initMetricVars(settings *MetricSettings) { M_Alerting_Notification_Sent, M_Aws_CloudWatch_GetMetricStatistics, M_Aws_CloudWatch_ListMetrics, + M_Aws_CloudWatch_GetMetricData, M_DB_DataSource_QueryById, M_Alerting_Active_Alerts, M_StatTotal_Dashboards, M_StatTotal_Users, + M_StatActive_Users, M_StatTotal_Orgs, M_StatTotal_Playlists, - M_Grafana_Version) + M_Grafana_Version, + grafanaBuildVersion) - go instrumentationLoop(settings) } -func instrumentationLoop(settings *MetricSettings) chan struct{} { - M_Instance_Start.Inc() - - onceEveryDayTick := time.NewTicker(time.Hour * 24) - secondTicker := time.NewTicker(time.Second * time.Duration(settings.IntervalSeconds)) - - for { - select { - case <-onceEveryDayTick.C: - sendUsageStats() - case <-secondTicker.C: - updateTotalStats() - } - } -} - -var metricPublishCounter int64 = 0 - func updateTotalStats() { - metricPublishCounter++ - if metricPublishCounter == 1 || metricPublishCounter%10 == 0 { - statsQuery := models.GetSystemStatsQuery{} - if err := bus.Dispatch(&statsQuery); err != nil { - metricsLogger.Error("Failed to get system stats", "error", err) - return - } + statsQuery := models.GetSystemStatsQuery{} + if err := bus.Dispatch(&statsQuery); err != nil { + metricsLogger.Error("Failed to get system stats", "error", err) + return + } - M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards)) - M_StatTotal_Users.Set(float64(statsQuery.Result.Users)) - M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists)) - M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs)) + M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards)) + M_StatTotal_Users.Set(float64(statsQuery.Result.Users)) + M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers)) + M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists)) + M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs)) +} + +var usageStatsURL = "https://stats.grafana.org/grafana-usage-report" + +func getEdition() string { + if setting.IsEnterprise { + return "enterprise" + } else { + return "oss" } } -func sendUsageStats() { +func sendUsageStats(oauthProviders map[string]bool) { if !setting.ReportingEnabled { return } @@ -361,6 +401,7 @@ func sendUsageStats() { "metrics": metrics, "os": runtime.GOOS, "arch": runtime.GOARCH, + "edition": getEdition(), } statsQuery := models.GetSystemStatsQuery{} @@ -380,6 +421,12 @@ func sendUsageStats() { metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers metrics["stats.datasources.count"] = statsQuery.Result.Datasources metrics["stats.stars.count"] = statsQuery.Result.Stars + metrics["stats.folders.count"] = statsQuery.Result.Folders + metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions + metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions + metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards + metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots + metrics["stats.teams.count"] = statsQuery.Result.Teams dsStats := models.GetDataSourceStatsQuery{} if err := bus.Dispatch(&dsStats); err != nil { @@ -400,9 +447,66 @@ func sendUsageStats() { } metrics["stats.ds.other.count"] = dsOtherCount + dsAccessStats := models.GetDataSourceAccessStatsQuery{} + if err := bus.Dispatch(&dsAccessStats); err != nil { + metricsLogger.Error("Failed to get datasource access stats", "error", err) + return + } + + // send access counters for each data source + // but ignore any custom data sources + // as sending that name could be sensitive information + dsAccessOtherCount := make(map[string]int64) + for _, dsAccessStat := range dsAccessStats.Result { + if dsAccessStat.Access == "" { + continue + } + + access := strings.ToLower(dsAccessStat.Access) + + if models.IsKnownDataSourcePlugin(dsAccessStat.Type) { + metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count + } else { + old := dsAccessOtherCount[access] + dsAccessOtherCount[access] = old + dsAccessStat.Count + } + } + + for access, count := range dsAccessOtherCount { + metrics["stats.ds_access.other."+access+".count"] = count + } + + anStats := models.GetAlertNotifierUsageStatsQuery{} + if err := bus.Dispatch(&anStats); err != nil { + metricsLogger.Error("Failed to get alert notification stats", "error", err) + return + } + + for _, stats := range anStats.Result { + metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count + } + + authTypes := map[string]bool{} + authTypes["anonymous"] = setting.AnonymousEnabled + authTypes["basic_auth"] = setting.BasicAuthEnabled + authTypes["ldap"] = setting.LdapEnabled + authTypes["auth_proxy"] = setting.AuthProxyEnabled + + for provider, enabled := range oauthProviders { + authTypes["oauth_"+provider] = enabled + } + + for authType, enabled := range authTypes { + enabledValue := 0 + if enabled { + enabledValue = 1 + } + metrics["stats.auth_enabled."+authType+".count"] = enabledValue + } + out, _ := json.MarshalIndent(report, "", " ") data := bytes.NewBuffer(out) - client := http.Client{Timeout: time.Duration(5 * time.Second)} - go client.Post("https://stats.grafana.org/grafana-usage-report", "application/json", data) + client := http.Client{Timeout: 5 * time.Second} + go client.Post(usageStatsURL, "application/json", data) } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 00000000000..43739221f1e --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,267 @@ +package metrics + +import ( + "bytes" + "io/ioutil" + "runtime" + "sync" + "testing" + "time" + + "net/http" + "net/http/httptest" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" +) + +func TestMetrics(t *testing.T) { + Convey("Test send usage stats", t, func() { + var getSystemStatsQuery *models.GetSystemStatsQuery + bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error { + query.Result = &models.SystemStats{ + Dashboards: 1, + Datasources: 2, + Users: 3, + ActiveUsers: 4, + Orgs: 5, + Playlists: 6, + Alerts: 7, + Stars: 8, + Folders: 9, + DashboardPermissions: 10, + FolderPermissions: 11, + ProvisionedDashboards: 12, + Snapshots: 13, + Teams: 14, + } + getSystemStatsQuery = query + return nil + }) + + var getDataSourceStatsQuery *models.GetDataSourceStatsQuery + bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error { + query.Result = []*models.DataSourceStats{ + { + Type: models.DS_ES, + Count: 9, + }, + { + Type: models.DS_PROMETHEUS, + Count: 10, + }, + { + Type: "unknown_ds", + Count: 11, + }, + { + Type: "unknown_ds2", + Count: 12, + }, + } + getDataSourceStatsQuery = query + return nil + }) + + var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery + bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error { + query.Result = []*models.DataSourceAccessStats{ + { + Type: models.DS_ES, + Access: "direct", + Count: 1, + }, + { + Type: models.DS_ES, + Access: "proxy", + Count: 2, + }, + { + Type: models.DS_PROMETHEUS, + Access: "proxy", + Count: 3, + }, + { + Type: "unknown_ds", + Access: "proxy", + Count: 4, + }, + { + Type: "unknown_ds2", + Access: "", + Count: 5, + }, + { + Type: "unknown_ds3", + Access: "direct", + Count: 6, + }, + { + Type: "unknown_ds4", + Access: "direct", + Count: 7, + }, + { + Type: "unknown_ds5", + Access: "proxy", + Count: 8, + }, + } + getDataSourceAccessStatsQuery = query + return nil + }) + + var getAlertNotifierUsageStatsQuery *models.GetAlertNotifierUsageStatsQuery + bus.AddHandler("test", func(query *models.GetAlertNotifierUsageStatsQuery) error { + query.Result = []*models.NotifierUsageStats{ + { + Type: "slack", + Count: 1, + }, + { + Type: "webhook", + Count: 2, + }, + } + + getAlertNotifierUsageStatsQuery = query + + return nil + }) + + var wg sync.WaitGroup + var responseBuffer *bytes.Buffer + var req *http.Request + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + req = r + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("Failed to read response body, err=%v", err) + } + responseBuffer = bytes.NewBuffer(buf) + wg.Done() + })) + usageStatsURL = ts.URL + + oauthProviders := map[string]bool{ + "github": true, + "gitlab": true, + "google": true, + "generic_oauth": true, + "grafana_com": true, + } + + sendUsageStats(oauthProviders) + + Convey("Given reporting not enabled and sending usage stats", func() { + setting.ReportingEnabled = false + sendUsageStats(oauthProviders) + + Convey("Should not gather stats or call http endpoint", func() { + So(getSystemStatsQuery, ShouldBeNil) + So(getDataSourceStatsQuery, ShouldBeNil) + So(getDataSourceAccessStatsQuery, ShouldBeNil) + So(req, ShouldBeNil) + }) + }) + + Convey("Given reporting enabled and sending usage stats", func() { + setting.ReportingEnabled = true + setting.BuildVersion = "5.0.0" + setting.AnonymousEnabled = true + setting.BasicAuthEnabled = true + setting.LdapEnabled = true + setting.AuthProxyEnabled = true + + wg.Add(1) + sendUsageStats(oauthProviders) + + Convey("Should gather stats and call http endpoint", func() { + if waitTimeout(&wg, 2*time.Second) { + t.Fatalf("Timed out waiting for http request") + } + + So(getSystemStatsQuery, ShouldNotBeNil) + So(getDataSourceStatsQuery, ShouldNotBeNil) + So(getDataSourceAccessStatsQuery, ShouldNotBeNil) + So(getAlertNotifierUsageStatsQuery, ShouldNotBeNil) + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.Header.Get("Content-Type"), ShouldEqual, "application/json") + + So(responseBuffer, ShouldNotBeNil) + + j, err := simplejson.NewFromReader(responseBuffer) + So(err, ShouldBeNil) + + So(j.Get("version").MustString(), ShouldEqual, "5_0_0") + So(j.Get("os").MustString(), ShouldEqual, runtime.GOOS) + So(j.Get("arch").MustString(), ShouldEqual, runtime.GOARCH) + + metrics := j.Get("metrics") + So(metrics.Get("stats.dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Dashboards) + So(metrics.Get("stats.users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Users) + So(metrics.Get("stats.orgs.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Orgs) + So(metrics.Get("stats.playlist.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Playlists) + So(metrics.Get("stats.plugins.apps.count").MustInt(), ShouldEqual, len(plugins.Apps)) + So(metrics.Get("stats.plugins.panels.count").MustInt(), ShouldEqual, len(plugins.Panels)) + So(metrics.Get("stats.plugins.datasources.count").MustInt(), ShouldEqual, len(plugins.DataSources)) + So(metrics.Get("stats.alerts.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Alerts) + So(metrics.Get("stats.active_users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ActiveUsers) + So(metrics.Get("stats.datasources.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Datasources) + So(metrics.Get("stats.stars.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Stars) + So(metrics.Get("stats.folders.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Folders) + So(metrics.Get("stats.dashboard_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.DashboardPermissions) + So(metrics.Get("stats.folder_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.FolderPermissions) + So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards) + So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots) + So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams) + + So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9) + So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10) + So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12) + + So(metrics.Get("stats.ds_access."+models.DS_ES+".direct.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.ds_access."+models.DS_ES+".proxy.count").MustInt(), ShouldEqual, 2) + So(metrics.Get("stats.ds_access."+models.DS_PROMETHEUS+".proxy.count").MustInt(), ShouldEqual, 3) + So(metrics.Get("stats.ds_access.other.direct.count").MustInt(), ShouldEqual, 6+7) + So(metrics.Get("stats.ds_access.other.proxy.count").MustInt(), ShouldEqual, 4+8) + + So(metrics.Get("stats.alert_notifiers.slack.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.alert_notifiers.webhook.count").MustInt(), ShouldEqual, 2) + + So(metrics.Get("stats.auth_enabled.anonymous.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.basic_auth.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.ldap.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.auth_proxy.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.oauth_github.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.oauth_gitlab.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.oauth_google.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.oauth_generic_oauth.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.auth_enabled.oauth_grafana_com.count").MustInt(), ShouldEqual, 1) + }) + }) + + Reset(func() { + ts.Close() + }) + }) +} + +func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() + }() + select { + case <-c: + return false // completed normally + case <-time.After(timeout): + return true // timed out + } +} diff --git a/pkg/metrics/service.go b/pkg/metrics/service.go new file mode 100644 index 00000000000..d2c0c815da9 --- /dev/null +++ b/pkg/metrics/service.go @@ -0,0 +1,71 @@ +package metrics + +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics/graphitebridge" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/setting" +) + +var metricsLogger log.Logger = log.New("metrics") + +type logWrapper struct { + logger log.Logger +} + +func (lw *logWrapper) Println(v ...interface{}) { + lw.logger.Info("graphite metric bridge", v...) +} + +func init() { + registry.RegisterService(&InternalMetricsService{}) + initMetricVars() +} + +type InternalMetricsService struct { + Cfg *setting.Cfg `inject:""` + + intervalSeconds int64 + graphiteCfg *graphitebridge.Config + oauthProviders map[string]bool +} + +func (im *InternalMetricsService) Init() error { + return im.readSettings() +} + +func (im *InternalMetricsService) Run(ctx context.Context) error { + // Start Graphite Bridge + if im.graphiteCfg != nil { + bridge, err := graphitebridge.NewBridge(im.graphiteCfg) + if err != nil { + metricsLogger.Error("failed to create graphite bridge", "error", err) + } else { + go bridge.Run(ctx) + } + } + + M_Instance_Start.Inc() + + // set the total stats gauges before we publishing metrics + updateTotalStats() + + onceEveryDayTick := time.NewTicker(time.Hour * 24) + everyMinuteTicker := time.NewTicker(time.Minute) + defer onceEveryDayTick.Stop() + defer everyMinuteTicker.Stop() + + for { + select { + case <-onceEveryDayTick.C: + sendUsageStats(im.oauthProviders) + case <-everyMinuteTicker.C: + updateTotalStats() + case <-ctx.Done(): + return ctx.Err() + } + } +} diff --git a/pkg/metrics/settings.go b/pkg/metrics/settings.go index 5e51f85768a..18b9e78d6ff 100644 --- a/pkg/metrics/settings.go +++ b/pkg/metrics/settings.go @@ -1,67 +1,52 @@ package metrics import ( + "fmt" "strings" "time" + "github.com/grafana/grafana/pkg/social" + "github.com/grafana/grafana/pkg/metrics/graphitebridge" "github.com/grafana/grafana/pkg/setting" "github.com/prometheus/client_golang/prometheus" - ini "gopkg.in/ini.v1" ) -type MetricSettings struct { - Enabled bool - IntervalSeconds int64 - GraphiteBridgeConfig *graphitebridge.Config +func (im *InternalMetricsService) readSettings() error { + var section, err = im.Cfg.Raw.GetSection("metrics") + if err != nil { + return fmt.Errorf("Unable to find metrics config section %v", err) + } + + im.intervalSeconds = section.Key("interval_seconds").MustInt64(10) + + if err := im.parseGraphiteSettings(); err != nil { + return fmt.Errorf("Unable to parse metrics graphite section, %v", err) + } + + im.oauthProviders = social.GetOAuthProviders(im.Cfg) + + return nil } -func ReadSettings(file *ini.File) *MetricSettings { - var settings = &MetricSettings{ - Enabled: false, - } +func (im *InternalMetricsService) parseGraphiteSettings() error { + graphiteSection, err := im.Cfg.Raw.GetSection("metrics.graphite") - var section, err = file.GetSection("metrics") if err != nil { - metricsLogger.Crit("Unable to find metrics config section", "error", err) return nil } - settings.Enabled = section.Key("enabled").MustBool(false) - settings.IntervalSeconds = section.Key("interval_seconds").MustInt64(10) - - if !settings.Enabled { - return settings - } - - cfg, err := parseGraphiteSettings(settings, file) - if err != nil { - metricsLogger.Crit("Unable to parse metrics graphite section", "error", err) - return nil - } - - settings.GraphiteBridgeConfig = cfg - - return settings -} - -func parseGraphiteSettings(settings *MetricSettings, file *ini.File) (*graphitebridge.Config, error) { - graphiteSection, err := setting.Cfg.GetSection("metrics.graphite") - if err != nil { - return nil, nil - } - address := graphiteSection.Key("address").String() if address == "" { - return nil, nil + return nil } - cfg := &graphitebridge.Config{ + bridgeCfg := &graphitebridge.Config{ URL: address, Prefix: graphiteSection.Key("prefix").MustString("prod.grafana.%(instance_name)s"), CountersAsDelta: true, Gatherer: prometheus.DefaultGatherer, - Interval: time.Duration(settings.IntervalSeconds) * time.Second, + Interval: time.Duration(im.intervalSeconds) * time.Second, Timeout: 10 * time.Second, Logger: &logWrapper{logger: metricsLogger}, ErrorHandling: graphitebridge.ContinueOnError, @@ -74,6 +59,8 @@ func parseGraphiteSettings(settings *MetricSettings, file *ini.File) (*graphiteb prefix = "prod.grafana.%(instance_name)s." } - cfg.Prefix = strings.Replace(prefix, "%(instance_name)s", safeInstanceName, -1) - return cfg, nil + bridgeCfg.Prefix = strings.Replace(prefix, "%(instance_name)s", safeInstanceName, -1) + + im.graphiteCfg = bridgeCfg + return nil } diff --git a/pkg/middleware/auth.go b/pkg/middleware/auth.go index d6c377bc9ac..5faee1e3fa7 100644 --- a/pkg/middleware/auth.go +++ b/pkg/middleware/auth.go @@ -9,6 +9,7 @@ import ( m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" ) type AuthOptions struct { @@ -17,10 +18,10 @@ type AuthOptions struct { } func getRequestUserId(c *m.ReqContext) int64 { - userId := c.Session.Get(session.SESS_KEY_USERID) + userID := c.Session.Get(session.SESS_KEY_USERID) - if userId != nil { - return userId.(int64) + if userID != nil { + return userID.(int64) } return 0 @@ -34,6 +35,11 @@ func getApiKey(c *m.ReqContext) string { return key } + username, password, err := util.DecodeBasicAuthHeader(header) + if err == nil && username == "api_key" { + return password + } + return "" } diff --git a/pkg/middleware/auth_proxy.go b/pkg/middleware/auth_proxy.go index 4d2a7a98908..29bd305b336 100644 --- a/pkg/middleware/auth_proxy.go +++ b/pkg/middleware/auth_proxy.go @@ -1,8 +1,10 @@ package middleware import ( - "errors" "fmt" + "net" + "net/mail" + "reflect" "strings" "time" @@ -14,7 +16,9 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -func initContextWithAuthProxy(ctx *m.ReqContext, orgId int64) bool { +var AUTH_PROXY_SESSION_VAR = "authProxyHeaderValue" + +func initContextWithAuthProxy(ctx *m.ReqContext, orgID int64) bool { if !setting.AuthProxyEnabled { return false } @@ -25,68 +29,133 @@ func initContextWithAuthProxy(ctx *m.ReqContext, orgId int64) bool { } // if auth proxy ip(s) defined, check if request comes from one of those - if err := checkAuthenticationProxy(ctx, proxyHeaderValue); err != nil { + if err := checkAuthenticationProxy(ctx.Req.RemoteAddr, proxyHeaderValue); err != nil { ctx.Handle(407, "Proxy authentication required", err) return true } - query := getSignedInUserQueryForProxyAuth(proxyHeaderValue) - query.OrgId = orgId - if err := bus.Dispatch(query); err != nil { - if err != m.ErrUserNotFound { - ctx.Handle(500, "Failed to find user specified in auth proxy header", err) + // initialize session + if err := ctx.Session.Start(ctx.Context); err != nil { + log.Error(3, "Failed to start session. error %v", err) + return false + } + + query := &m.GetSignedInUserQuery{OrgId: orgID} + + // if this session has already been authenticated by authProxy just load the user + sessProxyValue := ctx.Session.Get(AUTH_PROXY_SESSION_VAR) + if sessProxyValue != nil && sessProxyValue.(string) == proxyHeaderValue && getRequestUserId(ctx) > 0 { + // if we're using ldap, sync user periodically + if setting.LdapEnabled { + syncQuery := &m.LoginUserQuery{ + ReqContext: ctx, + Username: proxyHeaderValue, + } + + if err := syncGrafanaUserWithLdapUser(syncQuery); err != nil { + if err == login.ErrInvalidCredentials { + ctx.Handle(500, "Unable to authenticate user", err) + return false + } + + ctx.Handle(500, "Failed to sync user", err) + return false + } + } + + query.UserId = getRequestUserId(ctx) + // if we're using ldap, pass authproxy login name to ldap user sync + } else if setting.LdapEnabled { + ctx.Session.Delete(session.SESS_KEY_LASTLDAPSYNC) + + syncQuery := &m.LoginUserQuery{ + ReqContext: ctx, + Username: proxyHeaderValue, + } + + if err := syncGrafanaUserWithLdapUser(syncQuery); err != nil { + if err == login.ErrInvalidCredentials { + ctx.Handle(500, "Unable to authenticate user", err) + return false + } + + ctx.Handle(500, "Failed to sync user", err) + return false + } + + if syncQuery.User == nil { + ctx.Handle(500, "Failed to sync user", nil) + return false + } + + query.UserId = syncQuery.User.Id + // no ldap, just use the info we have + } else { + extUser := &m.ExternalUserInfo{ + AuthModule: "authproxy", + AuthId: proxyHeaderValue, + } + + if setting.AuthProxyHeaderProperty == "username" { + extUser.Login = proxyHeaderValue + + // only set Email if it can be parsed as an email address + emailAddr, emailErr := mail.ParseAddress(proxyHeaderValue) + if emailErr == nil { + extUser.Email = emailAddr.Address + } + } else if setting.AuthProxyHeaderProperty == "email" { + extUser.Email = proxyHeaderValue + extUser.Login = proxyHeaderValue + } else { + ctx.Handle(500, "Auth proxy header property invalid", nil) return true } - if setting.AuthProxyAutoSignUp { - cmd := getCreateUserCommandForProxyAuth(proxyHeaderValue) - if setting.LdapEnabled { - cmd.SkipOrgSetup = true + for _, field := range []string{"Name", "Email", "Login"} { + if setting.AuthProxyHeaders[field] == "" { + continue } - if err := bus.Dispatch(cmd); err != nil { - ctx.Handle(500, "Failed to create user specified in auth proxy header", err) - return true + if val := ctx.Req.Header.Get(setting.AuthProxyHeaders[field]); val != "" { + reflect.ValueOf(extUser).Elem().FieldByName(field).SetString(val) } - query = &m.GetSignedInUserQuery{UserId: cmd.Result.Id, OrgId: orgId} - if err := bus.Dispatch(query); err != nil { - ctx.Handle(500, "Failed find user after creation", err) - return true - } - } else { - return false } + + // add/update user in grafana + cmd := &m.UpsertUserCommand{ + ReqContext: ctx, + ExternalUser: extUser, + SignupAllowed: setting.AuthProxyAutoSignUp, + } + err := bus.Dispatch(cmd) + if err != nil { + ctx.Handle(500, "Failed to login as user specified in auth proxy header", err) + return true + } + + query.UserId = cmd.Result.Id } - // initialize session - if err := ctx.Session.Start(ctx.Context); err != nil { - log.Error(3, "Failed to start session", err) - return false + if err := bus.Dispatch(query); err != nil { + ctx.Handle(500, "Failed to find user", err) + return true } // Make sure that we cannot share a session between different users! if getRequestUserId(ctx) > 0 && getRequestUserId(ctx) != query.Result.UserId { // remove session if err := ctx.Session.Destory(ctx.Context); err != nil { - log.Error(3, "Failed to destroy session, err") + log.Error(3, "Failed to destroy session. error: %v", err) } // initialize a new session if err := ctx.Session.Start(ctx.Context); err != nil { - log.Error(3, "Failed to start session", err) + log.Error(3, "Failed to start session. error: %v", err) } } - // When ldap is enabled, sync userinfo and org roles - if err := syncGrafanaUserWithLdapUser(ctx, query); err != nil { - if err == login.ErrInvalidCredentials { - ctx.Handle(500, "Unable to authenticate user", err) - return false - } - - ctx.Handle(500, "Failed to sync user", err) - return false - } + ctx.Session.Set(AUTH_PROXY_SESSION_VAR, proxyHeaderValue) ctx.SignedInUser = query.Result ctx.IsSignedIn = true @@ -95,78 +164,51 @@ func initContextWithAuthProxy(ctx *m.ReqContext, orgId int64) bool { return true } -var syncGrafanaUserWithLdapUser = func(ctx *m.ReqContext, query *m.GetSignedInUserQuery) error { - if setting.LdapEnabled { - expireEpoch := time.Now().Add(time.Duration(-setting.AuthProxyLdapSyncTtl) * time.Minute).Unix() +var syncGrafanaUserWithLdapUser = func(query *m.LoginUserQuery) error { + expireEpoch := time.Now().Add(time.Duration(-setting.AuthProxyLdapSyncTtl) * time.Minute).Unix() - var lastLdapSync int64 - if lastLdapSyncInSession := ctx.Session.Get(session.SESS_KEY_LASTLDAPSYNC); lastLdapSyncInSession != nil { - lastLdapSync = lastLdapSyncInSession.(int64) + var lastLdapSync int64 + if lastLdapSyncInSession := query.ReqContext.Session.Get(session.SESS_KEY_LASTLDAPSYNC); lastLdapSyncInSession != nil { + lastLdapSync = lastLdapSyncInSession.(int64) + } + + if lastLdapSync < expireEpoch { + ldapCfg := login.LdapCfg + + if len(ldapCfg.Servers) < 1 { + return fmt.Errorf("No LDAP servers available") } - if lastLdapSync < expireEpoch { - ldapCfg := login.LdapCfg - - for _, server := range ldapCfg.Servers { - author := login.NewLdapAuthenticator(server) - if err := author.SyncSignedInUser(query.Result); err != nil { - return err - } + for _, server := range ldapCfg.Servers { + author := login.NewLdapAuthenticator(server) + if err := author.SyncUser(query); err != nil { + return err } - - ctx.Session.Set(session.SESS_KEY_LASTLDAPSYNC, time.Now().Unix()) } + + query.ReqContext.Session.Set(session.SESS_KEY_LASTLDAPSYNC, time.Now().Unix()) } return nil } -func checkAuthenticationProxy(ctx *m.ReqContext, proxyHeaderValue string) error { - if len(strings.TrimSpace(setting.AuthProxyWhitelist)) > 0 { - proxies := strings.Split(setting.AuthProxyWhitelist, ",") - remoteAddrSplit := strings.Split(ctx.Req.RemoteAddr, ":") - sourceIP := remoteAddrSplit[0] +func checkAuthenticationProxy(remoteAddr string, proxyHeaderValue string) error { + if len(strings.TrimSpace(setting.AuthProxyWhitelist)) == 0 { + return nil + } - found := false - for _, proxyIP := range proxies { - if sourceIP == strings.TrimSpace(proxyIP) { - found = true - break - } - } + proxies := strings.Split(setting.AuthProxyWhitelist, ",") + sourceIP, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return err + } - if !found { - msg := fmt.Sprintf("Request for user (%s) is not from the authentication proxy", proxyHeaderValue) - err := errors.New(msg) - return err + // Compare allowed IP addresses to actual address + for _, proxyIP := range proxies { + if sourceIP == strings.TrimSpace(proxyIP) { + return nil } } - return nil -} - -func getSignedInUserQueryForProxyAuth(headerVal string) *m.GetSignedInUserQuery { - query := m.GetSignedInUserQuery{} - if setting.AuthProxyHeaderProperty == "username" { - query.Login = headerVal - } else if setting.AuthProxyHeaderProperty == "email" { - query.Email = headerVal - } else { - panic("Auth proxy header property invalid") - } - return &query -} - -func getCreateUserCommandForProxyAuth(headerVal string) *m.CreateUserCommand { - cmd := m.CreateUserCommand{} - if setting.AuthProxyHeaderProperty == "username" { - cmd.Login = headerVal - cmd.Email = headerVal - } else if setting.AuthProxyHeaderProperty == "email" { - cmd.Email = headerVal - cmd.Login = headerVal - } else { - panic("Auth proxy header property invalid") - } - return &cmd + return fmt.Errorf("Request for user (%s) from %s is not from the authentication proxy", proxyHeaderValue, sourceIP) } diff --git a/pkg/middleware/auth_proxy_test.go b/pkg/middleware/auth_proxy_test.go index b3c011bd870..47ed2f71a79 100644 --- a/pkg/middleware/auth_proxy_test.go +++ b/pkg/middleware/auth_proxy_test.go @@ -26,57 +26,71 @@ func TestAuthProxyWithLdapEnabled(t *testing.T) { return &mockLdapAuther } - signedInUser := m.SignedInUser{} - query := m.GetSignedInUserQuery{Result: &signedInUser} - - Convey("When session variable lastLdapSync not set, call syncSignedInUser and set lastLdapSync", func() { + Convey("When user logs in, call SyncUser", func() { // arrange - sess := mockSession{} + sess := newMockSession() ctx := m.ReqContext{Session: &sess} So(sess.Get(session.SESS_KEY_LASTLDAPSYNC), ShouldBeNil) // act - syncGrafanaUserWithLdapUser(&ctx, &query) + syncGrafanaUserWithLdapUser(&m.LoginUserQuery{ + ReqContext: &ctx, + Username: "test", + }) // assert - So(mockLdapAuther.syncSignedInUserCalled, ShouldBeTrue) + So(mockLdapAuther.syncUserCalled, ShouldBeTrue) So(sess.Get(session.SESS_KEY_LASTLDAPSYNC), ShouldBeGreaterThan, 0) }) Convey("When session variable not expired, don't sync and don't change session var", func() { // arrange - sess := mockSession{} + sess := newMockSession() ctx := m.ReqContext{Session: &sess} now := time.Now().Unix() sess.Set(session.SESS_KEY_LASTLDAPSYNC, now) + sess.Set(AUTH_PROXY_SESSION_VAR, "test") // act - syncGrafanaUserWithLdapUser(&ctx, &query) + syncGrafanaUserWithLdapUser(&m.LoginUserQuery{ + ReqContext: &ctx, + Username: "test", + }) // assert So(sess.Get(session.SESS_KEY_LASTLDAPSYNC), ShouldEqual, now) - So(mockLdapAuther.syncSignedInUserCalled, ShouldBeFalse) + So(mockLdapAuther.syncUserCalled, ShouldBeFalse) }) Convey("When lastldapsync is expired, session variable should be updated", func() { // arrange - sess := mockSession{} + sess := newMockSession() ctx := m.ReqContext{Session: &sess} expiredTime := time.Now().Add(time.Duration(-120) * time.Minute).Unix() sess.Set(session.SESS_KEY_LASTLDAPSYNC, expiredTime) + sess.Set(AUTH_PROXY_SESSION_VAR, "test") // act - syncGrafanaUserWithLdapUser(&ctx, &query) + syncGrafanaUserWithLdapUser(&m.LoginUserQuery{ + ReqContext: &ctx, + Username: "test", + }) // assert So(sess.Get(session.SESS_KEY_LASTLDAPSYNC), ShouldBeGreaterThan, expiredTime) - So(mockLdapAuther.syncSignedInUserCalled, ShouldBeTrue) + So(mockLdapAuther.syncUserCalled, ShouldBeTrue) }) }) } type mockSession struct { - value interface{} + value map[interface{}]interface{} +} + +func newMockSession() mockSession { + session := mockSession{} + session.value = make(map[interface{}]interface{}) + return session } func (s *mockSession) Start(c *macaron.Context) error { @@ -84,15 +98,16 @@ func (s *mockSession) Start(c *macaron.Context) error { } func (s *mockSession) Set(k interface{}, v interface{}) error { - s.value = v + s.value[k] = v return nil } func (s *mockSession) Get(k interface{}) interface{} { - return s.value + return s.value[k] } func (s *mockSession) Delete(k interface{}) interface{} { + delete(s.value, k) return nil } @@ -113,21 +128,18 @@ func (s *mockSession) RegenerateId(c *macaron.Context) error { } type mockLdapAuthenticator struct { - syncSignedInUserCalled bool + syncUserCalled bool } -func (a *mockLdapAuthenticator) Login(query *login.LoginUserQuery) error { +func (a *mockLdapAuthenticator) Login(query *m.LoginUserQuery) error { return nil } -func (a *mockLdapAuthenticator) SyncSignedInUser(signedInUser *m.SignedInUser) error { - a.syncSignedInUserCalled = true +func (a *mockLdapAuthenticator) SyncUser(query *m.LoginUserQuery) error { + a.syncUserCalled = true return nil } -func (a *mockLdapAuthenticator) GetGrafanaUserFor(ldapUser *login.LdapUserInfo) (*m.User, error) { +func (a *mockLdapAuthenticator) GetGrafanaUserFor(ctx *m.ReqContext, ldapUser *login.LdapUserInfo) (*m.User, error) { return nil, nil } -func (a *mockLdapAuthenticator) SyncOrgRoles(user *m.User, ldapUser *login.LdapUserInfo) error { - return nil -} diff --git a/pkg/middleware/dashboard_redirect.go b/pkg/middleware/dashboard_redirect.go index 7c2af548a8f..2edf04d543e 100644 --- a/pkg/middleware/dashboard_redirect.go +++ b/pkg/middleware/dashboard_redirect.go @@ -10,8 +10,8 @@ import ( "gopkg.in/macaron.v1" ) -func getDashboardUrlBySlug(orgId int64, slug string) (string, error) { - query := m.GetDashboardQuery{Slug: slug, OrgId: orgId} +func getDashboardURLBySlug(orgID int64, slug string) (string, error) { + query := m.GetDashboardQuery{Slug: slug, OrgId: orgID} if err := bus.Dispatch(&query); err != nil { return "", m.ErrDashboardNotFound @@ -20,12 +20,12 @@ func getDashboardUrlBySlug(orgId int64, slug string) (string, error) { return m.GetDashboardUrl(query.Result.Uid, query.Result.Slug), nil } -func RedirectFromLegacyDashboardUrl() macaron.Handler { +func RedirectFromLegacyDashboardURL() macaron.Handler { return func(c *m.ReqContext) { slug := c.Params("slug") if slug != "" { - if url, err := getDashboardUrlBySlug(c.OrgId, slug); err == nil { + if url, err := getDashboardURLBySlug(c.OrgId, slug); err == nil { url = fmt.Sprintf("%s?%s", url, c.Req.URL.RawQuery) c.Redirect(url, 301) return @@ -34,13 +34,13 @@ func RedirectFromLegacyDashboardUrl() macaron.Handler { } } -func RedirectFromLegacyDashboardSoloUrl() macaron.Handler { +func RedirectFromLegacyDashboardSoloURL() macaron.Handler { return func(c *m.ReqContext) { slug := c.Params("slug") renderRequest := c.QueryBool("render") if slug != "" { - if url, err := getDashboardUrlBySlug(c.OrgId, slug); err == nil { + if url, err := getDashboardURLBySlug(c.OrgId, slug); err == nil { if renderRequest && strings.Contains(url, setting.AppSubUrl) { url = strings.Replace(url, setting.AppSubUrl, "", 1) } diff --git a/pkg/middleware/dashboard_redirect_test.go b/pkg/middleware/dashboard_redirect_test.go index 0af06347ed0..1e4d8293cae 100644 --- a/pkg/middleware/dashboard_redirect_test.go +++ b/pkg/middleware/dashboard_redirect_test.go @@ -13,8 +13,8 @@ import ( func TestMiddlewareDashboardRedirect(t *testing.T) { Convey("Given the dashboard redirect middleware", t, func() { bus.ClearBusHandlers() - redirectFromLegacyDashboardUrl := RedirectFromLegacyDashboardUrl() - redirectFromLegacyDashboardSoloUrl := RedirectFromLegacyDashboardSoloUrl() + redirectFromLegacyDashboardUrl := RedirectFromLegacyDashboardURL() + redirectFromLegacyDashboardSoloUrl := RedirectFromLegacyDashboardSoloURL() fakeDash := m.NewDashboard("Child dash") fakeDash.Id = 1 @@ -34,9 +34,9 @@ func TestMiddlewareDashboardRedirect(t *testing.T) { Convey("Should redirect to new dashboard url with a 301 Moved Permanently", func() { So(sc.resp.Code, ShouldEqual, 301) - redirectUrl, _ := sc.resp.Result().Location() - So(redirectUrl.Path, ShouldEqual, m.GetDashboardUrl(fakeDash.Uid, fakeDash.Slug)) - So(len(redirectUrl.Query()), ShouldEqual, 2) + redirectURL, _ := sc.resp.Result().Location() + So(redirectURL.Path, ShouldEqual, m.GetDashboardUrl(fakeDash.Uid, fakeDash.Slug)) + So(len(redirectURL.Query()), ShouldEqual, 2) }) }) @@ -47,11 +47,11 @@ func TestMiddlewareDashboardRedirect(t *testing.T) { Convey("Should redirect to new dashboard url with a 301 Moved Permanently", func() { So(sc.resp.Code, ShouldEqual, 301) - redirectUrl, _ := sc.resp.Result().Location() - expectedUrl := m.GetDashboardUrl(fakeDash.Uid, fakeDash.Slug) - expectedUrl = strings.Replace(expectedUrl, "/d/", "/d-solo/", 1) - So(redirectUrl.Path, ShouldEqual, expectedUrl) - So(len(redirectUrl.Query()), ShouldEqual, 2) + redirectURL, _ := sc.resp.Result().Location() + expectedURL := m.GetDashboardUrl(fakeDash.Uid, fakeDash.Slug) + expectedURL = strings.Replace(expectedURL, "/d/", "/d-solo/", 1) + So(redirectURL.Path, ShouldEqual, expectedURL) + So(len(redirectURL.Query()), ShouldEqual, 2) }) }) }) diff --git a/pkg/middleware/headers.go b/pkg/middleware/headers.go new file mode 100644 index 00000000000..28c623d74b0 --- /dev/null +++ b/pkg/middleware/headers.go @@ -0,0 +1,14 @@ +package middleware + +import ( + m "github.com/grafana/grafana/pkg/models" + macaron "gopkg.in/macaron.v1" +) + +const HeaderNameNoBackendCache = "X-Grafana-NoCache" + +func HandleNoCacheHeader() macaron.Handler { + return func(ctx *m.ReqContext) { + ctx.SkipCache = ctx.Req.Header.Get(HeaderNameNoBackendCache) == "true" + } +} diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index b5b244d5bff..ace72d998eb 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -8,13 +8,19 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/apikeygen" "github.com/grafana/grafana/pkg/log" - l "github.com/grafana/grafana/pkg/login" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) +var ( + ReqGrafanaAdmin = Auth(&AuthOptions{ReqSignedIn: true, ReqGrafanaAdmin: true}) + ReqSignedIn = Auth(&AuthOptions{ReqSignedIn: true}) + ReqEditorRole = RoleAuth(m.ROLE_EDITOR, m.ROLE_ADMIN) + ReqOrgAdmin = RoleAuth(m.ROLE_ADMIN) +) + func GetContextHandler() macaron.Handler { return func(c *macaron.Context) { ctx := &m.ReqContext{ @@ -23,6 +29,7 @@ func GetContextHandler() macaron.Handler { Session: session.GetSession(), IsSignedIn: false, AllowAnonymous: false, + SkipCache: false, Logger: log.New("context"), } @@ -37,12 +44,13 @@ func GetContextHandler() macaron.Handler { // then init session and look for userId in session // then look for api key in session (special case for render calls via api) // then test if anonymous access is enabled - if initContextWithRenderAuth(ctx) || - initContextWithApiKey(ctx) || - initContextWithBasicAuth(ctx, orgId) || - initContextWithAuthProxy(ctx, orgId) || - initContextWithUserSessionCookie(ctx, orgId) || - initContextWithAnonymousUser(ctx) { + switch { + case initContextWithRenderAuth(ctx): + case initContextWithApiKey(ctx): + case initContextWithBasicAuth(ctx, orgId): + case initContextWithAuthProxy(ctx, orgId): + case initContextWithUserSessionCookie(ctx, orgId): + case initContextWithAnonymousUser(ctx): } ctx.Logger = log.New("context", "userId", ctx.UserId, "orgId", ctx.OrgId, "uname", ctx.Login) @@ -50,7 +58,6 @@ func GetContextHandler() macaron.Handler { c.Map(ctx) - // update last seen at // update last seen every 5min if ctx.ShouldUpdateLastSeenAt() { ctx.Logger.Debug("Updating last user_seen_at", "user_id", ctx.UserId) @@ -165,7 +172,7 @@ func initContextWithBasicAuth(ctx *m.ReqContext, orgId int64) bool { user := loginQuery.Result - loginUserQuery := l.LoginUserQuery{Username: username, Password: password, User: user} + loginUserQuery := m.LoginUserQuery{Username: username, Password: password, User: user} if err := bus.Dispatch(&loginUserQuery); err != nil { ctx.JsonApiErr(401, "Invalid username or password", err) return true diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 83efc65d4d4..e9a3c8059f8 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -9,7 +9,6 @@ import ( ms "github.com/go-macaron/session" "github.com/grafana/grafana/pkg/bus" - l "github.com/grafana/grafana/pkg/login" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" @@ -19,6 +18,7 @@ import ( ) func TestMiddlewareContext(t *testing.T) { + setting.ERR_TEMPLATE_NAME = "error-template" Convey("Given the grafana middleware", t, func() { middlewareScenario("middleware should add context to injector", func(sc *scenarioContext) { @@ -72,7 +72,7 @@ func TestMiddlewareContext(t *testing.T) { return nil }) - bus.AddHandler("test", func(loginUserQuery *l.LoginUserQuery) error { + bus.AddHandler("test", func(loginUserQuery *m.LoginUserQuery) error { return nil }) @@ -83,7 +83,7 @@ func TestMiddlewareContext(t *testing.T) { setting.BasicAuthEnabled = true authHeader := util.GetBasicAuthHeader("myUser", "myPass") - sc.fakeReq("GET", "/").withAuthoriziationHeader(authHeader).exec() + sc.fakeReq("GET", "/").withAuthorizationHeader(authHeader).exec() Convey("Should init middleware context with user", func() { So(sc.context.IsSignedIn, ShouldEqual, true) @@ -129,6 +129,28 @@ func TestMiddlewareContext(t *testing.T) { }) }) + middlewareScenario("Valid api key via Basic auth", func(sc *scenarioContext) { + keyhash := util.EncodePassword("v5nAwpMafFP6znaS4urhdWDLS5511M42", "asd") + + bus.AddHandler("test", func(query *m.GetApiKeyByNameQuery) error { + query.Result = &m.ApiKey{OrgId: 12, Role: m.ROLE_EDITOR, Key: keyhash} + return nil + }) + + authHeader := util.GetBasicAuthHeader("api_key", "eyJrIjoidjVuQXdwTWFmRlA2em5hUzR1cmhkV0RMUzU1MTFNNDIiLCJuIjoiYXNkIiwiaWQiOjF9") + sc.fakeReq("GET", "/").withAuthorizationHeader(authHeader).exec() + + Convey("Should return 200", func() { + So(sc.resp.Code, ShouldEqual, 200) + }) + + Convey("Should init middleware context", func() { + So(sc.context.IsSignedIn, ShouldEqual, true) + So(sc.context.OrgId, ShouldEqual, 12) + So(sc.context.OrgRole, ShouldEqual, m.ROLE_EDITOR) + }) + }) + middlewareScenario("UserId in session", func(sc *scenarioContext) { sc.fakeReq("GET", "/").handler(func(c *m.ReqContext) { @@ -177,12 +199,18 @@ func TestMiddlewareContext(t *testing.T) { setting.AuthProxyEnabled = true setting.AuthProxyHeaderName = "X-WEBAUTH-USER" setting.AuthProxyHeaderProperty = "username" + setting.LdapEnabled = false bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { query.Result = &m.SignedInUser{OrgId: 2, UserId: 12} return nil }) + bus.AddHandler("test", func(cmd *m.UpsertUserCommand) error { + cmd.Result = &m.User{Id: 12} + return nil + }) + sc.fakeReq("GET", "/") sc.req.Header.Add("X-WEBAUTH-USER", "torkelo") sc.exec() @@ -199,18 +227,18 @@ func TestMiddlewareContext(t *testing.T) { setting.AuthProxyHeaderName = "X-WEBAUTH-USER" setting.AuthProxyHeaderProperty = "username" setting.AuthProxyAutoSignUp = true + setting.LdapEnabled = false bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { if query.UserId > 0 { query.Result = &m.SignedInUser{OrgId: 4, UserId: 33} return nil - } else { - return m.ErrUserNotFound } + return m.ErrUserNotFound }) - bus.AddHandler("test", func(cmd *m.CreateUserCommand) error { - cmd.Result = m.User{Id: 33} + bus.AddHandler("test", func(cmd *m.UpsertUserCommand) error { + cmd.Result = &m.User{Id: 33} return nil }) @@ -226,11 +254,11 @@ func TestMiddlewareContext(t *testing.T) { }) }) - middlewareScenario("When auth_proxy is enabled and request RemoteAddr is not trusted", func(sc *scenarioContext) { + middlewareScenario("When auth_proxy is enabled and IPv4 request RemoteAddr is not trusted", func(sc *scenarioContext) { setting.AuthProxyEnabled = true setting.AuthProxyHeaderName = "X-WEBAUTH-USER" setting.AuthProxyHeaderProperty = "username" - setting.AuthProxyWhitelist = "192.168.1.1, 192.168.2.1" + setting.AuthProxyWhitelist = "192.168.1.1, 2001::23" sc.fakeReq("GET", "/") sc.req.Header.Add("X-WEBAUTH-USER", "torkelo") @@ -239,6 +267,24 @@ func TestMiddlewareContext(t *testing.T) { Convey("should return 407 status code", func() { So(sc.resp.Code, ShouldEqual, 407) + So(sc.resp.Body.String(), ShouldContainSubstring, "Request for user (torkelo) from 192.168.3.1 is not from the authentication proxy") + }) + }) + + middlewareScenario("When auth_proxy is enabled and IPv6 request RemoteAddr is not trusted", func(sc *scenarioContext) { + setting.AuthProxyEnabled = true + setting.AuthProxyHeaderName = "X-WEBAUTH-USER" + setting.AuthProxyHeaderProperty = "username" + setting.AuthProxyWhitelist = "192.168.1.1, 2001::23" + + sc.fakeReq("GET", "/") + sc.req.Header.Add("X-WEBAUTH-USER", "torkelo") + sc.req.RemoteAddr = "[2001:23]:12345" + sc.exec() + + Convey("should return 407 status code", func() { + So(sc.resp.Code, ShouldEqual, 407) + So(sc.resp.Body.String(), ShouldContainSubstring, "Request for user (torkelo) from 2001:23 is not from the authentication proxy") }) }) @@ -246,16 +292,21 @@ func TestMiddlewareContext(t *testing.T) { setting.AuthProxyEnabled = true setting.AuthProxyHeaderName = "X-WEBAUTH-USER" setting.AuthProxyHeaderProperty = "username" - setting.AuthProxyWhitelist = "192.168.1.1, 192.168.2.1" + setting.AuthProxyWhitelist = "192.168.1.1, 2001::23" bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { query.Result = &m.SignedInUser{OrgId: 4, UserId: 33} return nil }) + bus.AddHandler("test", func(cmd *m.UpsertUserCommand) error { + cmd.Result = &m.User{Id: 33} + return nil + }) + sc.fakeReq("GET", "/") sc.req.Header.Add("X-WEBAUTH-USER", "torkelo") - sc.req.RemoteAddr = "192.168.2.1:12345" + sc.req.RemoteAddr = "[2001::23]:12345" sc.exec() Convey("Should init context with user info", func() { @@ -271,6 +322,11 @@ func TestMiddlewareContext(t *testing.T) { setting.AuthProxyHeaderProperty = "username" setting.AuthProxyWhitelist = "" + bus.AddHandler("test", func(query *m.UpsertUserCommand) error { + query.Result = &m.User{Id: 32} + return nil + }) + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { query.Result = &m.SignedInUser{OrgId: 4, UserId: 32} return nil @@ -301,11 +357,17 @@ func TestMiddlewareContext(t *testing.T) { setting.LdapEnabled = true called := false - syncGrafanaUserWithLdapUser = func(ctx *m.ReqContext, query *m.GetSignedInUserQuery) error { + syncGrafanaUserWithLdapUser = func(query *m.LoginUserQuery) error { called = true + query.User = &m.User{Id: 32} return nil } + bus.AddHandler("test", func(query *m.UpsertUserCommand) error { + query.Result = &m.User{Id: 32} + return nil + }) + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { query.Result = &m.SignedInUser{OrgId: 4, UserId: 32} return nil @@ -338,7 +400,7 @@ func middlewareScenario(desc string, fn scenarioFunc) { sc.m.Use(GetContextHandler()) // mock out gc goroutine session.StartSessionGC = func() {} - sc.m.Use(Sessioner(&ms.Options{})) + sc.m.Use(Sessioner(&ms.Options{}, 0)) sc.m.Use(OrgRedirect()) sc.m.Use(AddDefaultResponseHeaders()) @@ -374,12 +436,7 @@ func (sc *scenarioContext) withValidApiKey() *scenarioContext { return sc } -func (sc *scenarioContext) withInvalidApiKey() *scenarioContext { - sc.apiKey = "nvalidhhhhds" - return sc -} - -func (sc *scenarioContext) withAuthoriziationHeader(authHeader string) *scenarioContext { +func (sc *scenarioContext) withAuthorizationHeader(authHeader string) *scenarioContext { sc.authHeader = authHeader return sc } diff --git a/pkg/middleware/recovery.go b/pkg/middleware/recovery.go index ec289387aa4..eef07c8c24a 100644 --- a/pkg/middleware/recovery.go +++ b/pkg/middleware/recovery.go @@ -35,7 +35,7 @@ var ( slash = []byte("/") ) -// stack returns a nicely formated stack frame, skipping skip frames +// stack returns a nicely formatted stack frame, skipping skip frames func stack(skip int) []byte { buf := new(bytes.Buffer) // the returned data // As we loop, we open files and read them. These variables record the currently @@ -138,7 +138,7 @@ func Recovery() macaron.Handler { c.JSON(500, resp) } else { - c.HTML(500, "error") + c.HTML(500, setting.ERR_TEMPLATE_NAME) } } }() diff --git a/pkg/middleware/recovery_test.go b/pkg/middleware/recovery_test.go index 32545b7caca..c92150f3b7d 100644 --- a/pkg/middleware/recovery_test.go +++ b/pkg/middleware/recovery_test.go @@ -8,16 +8,19 @@ import ( "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/session" + "github.com/grafana/grafana/pkg/setting" . "github.com/smartystreets/goconvey/convey" "gopkg.in/macaron.v1" ) func TestRecoveryMiddleware(t *testing.T) { + setting.ERR_TEMPLATE_NAME = "error-template" + Convey("Given an api route that panics", t, func() { - apiUrl := "/api/whatever" - recoveryScenario("recovery middleware should return json", apiUrl, func(sc *scenarioContext) { + apiURL := "/api/whatever" + recoveryScenario("recovery middleware should return json", apiURL, func(sc *scenarioContext) { sc.handlerFunc = PanicHandler - sc.fakeReq("GET", apiUrl).exec() + sc.fakeReq("GET", apiURL).exec() sc.req.Header.Add("content-type", "application/json") So(sc.resp.Code, ShouldEqual, 500) @@ -27,10 +30,10 @@ func TestRecoveryMiddleware(t *testing.T) { }) Convey("Given a non-api route that panics", t, func() { - apiUrl := "/whatever" - recoveryScenario("recovery middleware should return html", apiUrl, func(sc *scenarioContext) { + apiURL := "/whatever" + recoveryScenario("recovery middleware should return html", apiURL, func(sc *scenarioContext) { sc.handlerFunc = PanicHandler - sc.fakeReq("GET", apiUrl).exec() + sc.fakeReq("GET", apiURL).exec() So(sc.resp.Code, ShouldEqual, 500) So(sc.resp.Header().Get("content-type"), ShouldEqual, "text/html; charset=UTF-8") @@ -50,6 +53,7 @@ func recoveryScenario(desc string, url string, fn scenarioFunc) { sc := &scenarioContext{ url: url, } + viewsPath, _ := filepath.Abs("../../public/views") sc.m = macaron.New() @@ -63,7 +67,7 @@ func recoveryScenario(desc string, url string, fn scenarioFunc) { sc.m.Use(GetContextHandler()) // mock out gc goroutine session.StartSessionGC = func() {} - sc.m.Use(Sessioner(&ms.Options{})) + sc.m.Use(Sessioner(&ms.Options{}, 0)) sc.m.Use(OrgRedirect()) sc.m.Use(AddDefaultResponseHeaders()) diff --git a/pkg/middleware/render_auth.go b/pkg/middleware/render_auth.go index 225645e659e..e30cfe67924 100644 --- a/pkg/middleware/render_auth.go +++ b/pkg/middleware/render_auth.go @@ -2,6 +2,7 @@ package middleware import ( "sync" + "time" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/util" @@ -19,19 +20,18 @@ func initContextWithRenderAuth(ctx *m.ReqContext) bool { renderKeysLock.Lock() defer renderKeysLock.Unlock() - if renderUser, exists := renderKeys[key]; !exists { + renderUser, exists := renderKeys[key] + if !exists { ctx.JsonApiErr(401, "Invalid Render Key", nil) return true - } else { - - ctx.IsSignedIn = true - ctx.SignedInUser = renderUser - ctx.IsRenderCall = true - return true } -} -type renderContextFunc func(key string) (string, error) + ctx.IsSignedIn = true + ctx.SignedInUser = renderUser + ctx.IsRenderCall = true + ctx.LastSeenAt = time.Now() + return true +} func AddRenderAuthKey(orgId int64, userId int64, orgRole m.RoleType) string { renderKeysLock.Lock() diff --git a/pkg/middleware/session.go b/pkg/middleware/session.go index 5654a42cb7d..19cfa368b49 100644 --- a/pkg/middleware/session.go +++ b/pkg/middleware/session.go @@ -8,8 +8,8 @@ import ( "github.com/grafana/grafana/pkg/services/session" ) -func Sessioner(options *ms.Options) macaron.Handler { - session.Init(options) +func Sessioner(options *ms.Options, sessionConnMaxLifetime int64) macaron.Handler { + session.Init(options, sessionConnMaxLifetime) return func(ctx *m.ReqContext) { ctx.Next() diff --git a/pkg/models/alert.go b/pkg/models/alert.go index 88b49350b97..aaf9c50197a 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -34,8 +34,8 @@ const ( ) var ( - ErrCannotChangeStateOnPausedAlert error = fmt.Errorf("Cannot change state on pause alert") - ErrRequiresNewState error = fmt.Errorf("update alert state requires a new state.") + ErrCannotChangeStateOnPausedAlert = fmt.Errorf("Cannot change state on pause alert") + ErrRequiresNewState = fmt.Errorf("update alert state requires a new state.") ) func (s AlertStateType) IsValid() bool { @@ -75,7 +75,7 @@ type Alert struct { EvalData *simplejson.Json NewStateDate time.Time - StateChanges int + StateChanges int64 Created time.Time Updated time.Time @@ -156,17 +156,18 @@ type SetAlertStateCommand struct { Error string EvalData *simplejson.Json - Timestamp time.Time + Result Alert } //Queries type GetAlertsQuery struct { - OrgId int64 - State []string - DashboardId int64 - PanelId int64 - Limit int64 - User *SignedInUser + OrgId int64 + State []string + DashboardIDs []int64 + PanelId int64 + Limit int64 + Query string + User *SignedInUser Result []*AlertListItemDTO } @@ -214,13 +215,14 @@ type AlertStateInfoDTO struct { // "Internal" commands type UpdateDashboardAlertsCommand struct { - UserId int64 OrgId int64 Dashboard *Dashboard + User *SignedInUser } type ValidateDashboardAlertsCommand struct { UserId int64 OrgId int64 Dashboard *Dashboard + User *SignedInUser } diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 87b515f370c..e0fd12937ed 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -1,38 +1,63 @@ package models import ( + "errors" "time" "github.com/grafana/grafana/pkg/components/simplejson" ) +var ( + ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified") + ErrAlertNotificationStateNotFound = errors.New("alert notification state not found") + ErrAlertNotificationStateVersionConflict = errors.New("alert notification state update version conflict") + ErrAlertNotificationStateAlreadyExist = errors.New("alert notification state already exists.") +) + +type AlertNotificationStateType string + +var ( + AlertNotificationStatePending = AlertNotificationStateType("pending") + AlertNotificationStateCompleted = AlertNotificationStateType("completed") + AlertNotificationStateUnknown = AlertNotificationStateType("unknown") +) + type AlertNotification struct { - Id int64 `json:"id"` - OrgId int64 `json:"-"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + OrgId int64 `json:"-"` + Name string `json:"name"` + Type string `json:"type"` + SendReminder bool `json:"sendReminder"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Frequency time.Duration `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type CreateAlertNotificationCommand struct { - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + SendReminder bool `json:"sendReminder"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Frequency string `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` OrgId int64 `json:"-"` Result *AlertNotification } type UpdateAlertNotificationCommand struct { - Id int64 `json:"id" binding:"Required"` - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings" binding:"Required"` + Id int64 `json:"id" binding:"Required"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + SendReminder bool `json:"sendReminder"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Frequency string `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings" binding:"Required"` OrgId int64 `json:"-"` Result *AlertNotification @@ -63,3 +88,35 @@ type GetAllAlertNotificationsQuery struct { Result []*AlertNotification } + +type AlertNotificationState struct { + Id int64 + OrgId int64 + AlertId int64 + NotifierId int64 + State AlertNotificationStateType + Version int64 + UpdatedAt int64 + AlertRuleStateUpdatedVersion int64 +} + +type SetAlertNotificationStateToPendingCommand struct { + Id int64 + AlertRuleStateUpdatedVersion int64 + Version int64 + + ResultVersion int64 +} + +type SetAlertNotificationStateToCompleteCommand struct { + Id int64 + Version int64 +} + +type GetOrCreateNotificationStateQuery struct { + OrgId int64 + AlertId int64 + NotifierId int64 + + Result *AlertNotificationState +} diff --git a/pkg/models/context.go b/pkg/models/context.go index 262f6550954..7cb80a957c3 100644 --- a/pkg/models/context.go +++ b/pkg/models/context.go @@ -20,6 +20,7 @@ type ReqContext struct { IsSignedIn bool IsRenderCall bool AllowAnonymous bool + SkipCache bool Logger log.Logger } @@ -36,7 +37,7 @@ func (ctx *ReqContext) Handle(status int, title string, err error) { ctx.Data["AppSubUrl"] = setting.AppSubUrl ctx.Data["Theme"] = "dark" - ctx.HTML(status, "error") + ctx.HTML(status, setting.ERR_TEMPLATE_NAME) } func (ctx *ReqContext) JsonOK(message string) { diff --git a/pkg/models/dashboard_acl.go b/pkg/models/dashboard_acl.go index 5b91b2a70b4..5fc09bd16b5 100644 --- a/pkg/models/dashboard_acl.go +++ b/pkg/models/dashboard_acl.go @@ -56,7 +56,10 @@ type DashboardAclInfoDTO struct { UserId int64 `json:"userId"` UserLogin string `json:"userLogin"` UserEmail string `json:"userEmail"` + UserAvatarUrl string `json:"userAvatarUrl"` TeamId int64 `json:"teamId"` + TeamEmail string `json:"teamEmail"` + TeamAvatarUrl string `json:"teamAvatarUrl"` Team string `json:"team"` Role *RoleType `json:"role,omitempty"` Permission PermissionType `json:"permission"` @@ -66,6 +69,7 @@ type DashboardAclInfoDTO struct { Slug string `json:"slug"` IsFolder bool `json:"isFolder"` Url string `json:"url"` + Inherited bool `json:"inherited"` } func (dto *DashboardAclInfoDTO) hasSameRoleAs(other *DashboardAclInfoDTO) bool { diff --git a/pkg/models/dashboard_snapshot.go b/pkg/models/dashboard_snapshot.go index ec8b19f3c18..3024ba94122 100644 --- a/pkg/models/dashboard_snapshot.go +++ b/pkg/models/dashboard_snapshot.go @@ -29,7 +29,6 @@ type DashboardSnapshotDTO struct { Id int64 `json:"id"` Name string `json:"name"` Key string `json:"key"` - DeleteKey string `json:"deleteKey"` OrgId int64 `json:"orgId"` UserId int64 `json:"userId"` External bool `json:"external"` diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 4b771038df6..e8aebb1d1f4 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -13,26 +13,26 @@ import ( // Typed errors var ( - ErrDashboardNotFound = errors.New("Dashboard not found") - ErrDashboardFolderNotFound = errors.New("Folder not found") - ErrDashboardSnapshotNotFound = errors.New("Dashboard snapshot not found") - ErrDashboardWithSameUIDExists = errors.New("A dashboard with the same uid already exists") - ErrDashboardWithSameNameInFolderExists = errors.New("A dashboard with the same name in the folder already exists") - ErrDashboardVersionMismatch = errors.New("The dashboard has been changed by someone else") - ErrDashboardTitleEmpty = errors.New("Dashboard title cannot be empty") - ErrDashboardFolderCannotHaveParent = errors.New("A Dashboard Folder cannot be added to another folder") - ErrDashboardContainsInvalidAlertData = errors.New("Invalid alert data. Cannot save dashboard") - ErrDashboardFailedToUpdateAlertData = errors.New("Failed to save alert data") - ErrDashboardsWithSameSlugExists = errors.New("Multiple dashboards with the same slug exists") - ErrDashboardFailedGenerateUniqueUid = errors.New("Failed to generate unique dashboard id") - ErrDashboardTypeMismatch = errors.New("Dashboard cannot be changed to a folder") - ErrDashboardFolderWithSameNameAsDashboard = errors.New("Folder name cannot be the same as one of its dashboards") - ErrDashboardWithSameNameAsFolder = errors.New("Dashboard name cannot be the same as folder") - ErrDashboardFolderNameExists = errors.New("A folder with that name already exists") - ErrDashboardUpdateAccessDenied = errors.New("Access denied to save dashboard") - ErrDashboardInvalidUid = errors.New("uid contains illegal characters") - ErrDashboardUidToLong = errors.New("uid to long. max 40 characters") - RootFolderName = "General" + ErrDashboardNotFound = errors.New("Dashboard not found") + ErrDashboardFolderNotFound = errors.New("Folder not found") + ErrDashboardSnapshotNotFound = errors.New("Dashboard snapshot not found") + ErrDashboardWithSameUIDExists = errors.New("A dashboard with the same uid already exists") + ErrDashboardWithSameNameInFolderExists = errors.New("A dashboard with the same name in the folder already exists") + ErrDashboardVersionMismatch = errors.New("The dashboard has been changed by someone else") + ErrDashboardTitleEmpty = errors.New("Dashboard title cannot be empty") + ErrDashboardFolderCannotHaveParent = errors.New("A Dashboard Folder cannot be added to another folder") + ErrDashboardFailedToUpdateAlertData = errors.New("Failed to save alert data") + ErrDashboardsWithSameSlugExists = errors.New("Multiple dashboards with the same slug exists") + ErrDashboardFailedGenerateUniqueUid = errors.New("Failed to generate unique dashboard id") + ErrDashboardTypeMismatch = errors.New("Dashboard cannot be changed to a folder") + ErrDashboardFolderWithSameNameAsDashboard = errors.New("Folder name cannot be the same as one of its dashboards") + ErrDashboardWithSameNameAsFolder = errors.New("Dashboard name cannot be the same as folder") + ErrDashboardFolderNameExists = errors.New("A folder with that name already exists") + ErrDashboardUpdateAccessDenied = errors.New("Access denied to save dashboard") + ErrDashboardInvalidUid = errors.New("uid contains illegal characters") + ErrDashboardUidToLong = errors.New("uid to long. max 40 characters") + ErrDashboardCannotSaveProvisionedDashboard = errors.New("Cannot save provisioned dashboard") + RootFolderName = "General" ) type UpdatePluginDashboardError struct { @@ -157,7 +157,7 @@ func NewDashboardFromJson(data *simplejson.Json) *Dashboard { return dash } -// GetDashboardModel turns the command into the savable model +// GetDashboardModel turns the command into the saveable model func (cmd *SaveDashboardCommand) GetDashboardModel() *Dashboard { dash := NewDashboardFromJson(cmd.Dashboard) userId := cmd.UserId @@ -209,14 +209,14 @@ func GetDashboardFolderUrl(isFolder bool, uid string, slug string) string { return GetDashboardUrl(uid, slug) } -// Return the html url for a dashboard +// GetDashboardUrl return the html url for a dashboard func GetDashboardUrl(uid string, slug string) string { return fmt.Sprintf("%s/d/%s/%s", setting.AppSubUrl, uid, slug) } -// Return the full url for a dashboard +// GetFullDashboardUrl return the full url for a dashboard func GetFullDashboardUrl(uid string, slug string) string { - return fmt.Sprintf("%s%s", setting.AppUrl, GetDashboardUrl(uid, slug)) + return fmt.Sprintf("%sd/%s/%s", setting.AppUrl, uid, slug) } // GetFolderUrl return the html url for a folder @@ -224,6 +224,10 @@ func GetFolderUrl(folderUid string, slug string) string { return fmt.Sprintf("%s/dashboards/f/%s/%s", setting.AppSubUrl, folderUid, slug) } +type ValidateDashboardBeforeSaveResult struct { + IsParentFolderChanged bool +} + // // COMMANDS // @@ -249,6 +253,7 @@ type DashboardProvisioning struct { DashboardId int64 Name string ExternalId string + CheckSum string Updated int64 } @@ -268,6 +273,7 @@ type ValidateDashboardBeforeSaveCommand struct { OrgId int64 Dashboard *Dashboard Overwrite bool + Result *ValidateDashboardBeforeSaveResult } // @@ -317,6 +323,12 @@ type GetDashboardSlugByIdQuery struct { Result string } +type IsDashboardProvisionedQuery struct { + DashboardId int64 + + Result bool +} + type GetProvisionedDashboardDataQuery struct { Name string diff --git a/pkg/models/dashboards_test.go b/pkg/models/dashboards_test.go index ad865b575bb..69bc8ab7bd9 100644 --- a/pkg/models/dashboards_test.go +++ b/pkg/models/dashboards_test.go @@ -4,11 +4,24 @@ import ( "testing" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/setting" . "github.com/smartystreets/goconvey/convey" ) func TestDashboardModel(t *testing.T) { + Convey("Generate full dashboard url", t, func() { + setting.AppUrl = "http://grafana.local/" + fullUrl := GetFullDashboardUrl("uid", "my-dashboard") + So(fullUrl, ShouldEqual, "http://grafana.local/d/uid/my-dashboard") + }) + + Convey("Generate relative dashboard url", t, func() { + setting.AppUrl = "" + fullUrl := GetDashboardUrl("uid", "my-dashboard") + So(fullUrl, ShouldEqual, "/d/uid/my-dashboard") + }) + Convey("When generating slug", t, func() { dashboard := NewDashboard("Grafana Play Home") dashboard.UpdateSlug() diff --git a/pkg/models/datasource.go b/pkg/models/datasource.go index f2236ad8477..89439420d7a 100644 --- a/pkg/models/datasource.go +++ b/pkg/models/datasource.go @@ -22,6 +22,7 @@ const ( DS_MSSQL = "mssql" DS_ACCESS_DIRECT = "direct" DS_ACCESS_PROXY = "proxy" + DS_STACKDRIVER = "stackdriver" ) var ( @@ -29,6 +30,7 @@ var ( ErrDataSourceNameExists = errors.New("Data source with same name already exists") ErrDataSourceUpdatingOldVersion = errors.New("Trying to update old version of datasource") ErrDatasourceIsReadOnly = errors.New("Data source is readonly. Can only be updated from configuration.") + ErrDataSourceAccessDenied = errors.New("Data source access denied") ) type DsAccess string @@ -58,24 +60,24 @@ type DataSource struct { Updated time.Time } -var knownDatasourcePlugins map[string]bool = map[string]bool{ - DS_ES: true, - DS_GRAPHITE: true, - DS_INFLUXDB: true, - DS_INFLUXDB_08: true, - DS_KAIROSDB: true, - DS_CLOUDWATCH: true, - DS_PROMETHEUS: true, - DS_OPENTSDB: true, - DS_POSTGRES: true, - DS_MYSQL: true, - DS_MSSQL: true, - "opennms": true, - "abhisant-druid-datasource": true, - "dalmatinerdb-datasource": true, - "gnocci": true, - "zabbix": true, - "alexanderzobnin-zabbix-datasource": true, +var knownDatasourcePlugins = map[string]bool{ + DS_ES: true, + DS_GRAPHITE: true, + DS_INFLUXDB: true, + DS_INFLUXDB_08: true, + DS_KAIROSDB: true, + DS_CLOUDWATCH: true, + DS_PROMETHEUS: true, + DS_OPENTSDB: true, + DS_POSTGRES: true, + DS_MYSQL: true, + DS_MSSQL: true, + DS_STACKDRIVER: true, + "opennms": true, + "abhisant-druid-datasource": true, + "dalmatinerdb-datasource": true, + "gnocci": true, + "zabbix": true, "newrelic-app": true, "grafana-datadog-datasource": true, "grafana-simple-json": true, @@ -88,6 +90,7 @@ var knownDatasourcePlugins map[string]bool = map[string]bool{ "ayoungprogrammer-finance-datasource": true, "monasca-datasource": true, "vertamedia-clickhouse-datasource": true, + "alexanderzobnin-zabbix-datasource": true, } func IsKnownDataSourcePlugin(dsType string) bool { @@ -165,6 +168,7 @@ type DeleteDataSourceByNameCommand struct { type GetDataSourcesQuery struct { OrgId int64 + User *SignedInUser Result []*DataSource } @@ -185,6 +189,26 @@ type GetDataSourceByNameQuery struct { } // --------------------- -// EVENTS -type DataSourceCreatedEvent struct { +// Permissions +// --------------------- + +type DsPermissionType int + +const ( + DsPermissionNoAccess DsPermissionType = iota + DsPermissionQuery +) + +func (p DsPermissionType) String() string { + names := map[int]string{ + int(DsPermissionQuery): "Query", + int(DsPermissionNoAccess): "No Access", + } + return names[int(p)] +} + +type DatasourcesPermissionFilterQuery struct { + User *SignedInUser + Datasources []*DataSource + Result []*DataSource } diff --git a/pkg/models/datasource_cache.go b/pkg/models/datasource_cache.go index b4a4e7f8a4d..66ba66e4d39 100644 --- a/pkg/models/datasource_cache.go +++ b/pkg/models/datasource_cache.go @@ -33,7 +33,7 @@ func (ds *DataSource) GetHttpClient() (*http.Client, error) { } return &http.Client{ - Timeout: time.Duration(30 * time.Second), + Timeout: 30 * time.Second, Transport: transport, }, nil } diff --git a/pkg/models/folders.go b/pkg/models/folders.go index c61620a11fc..f4dd7e5b776 100644 --- a/pkg/models/folders.go +++ b/pkg/models/folders.go @@ -32,7 +32,7 @@ type Folder struct { HasAcl bool } -// GetDashboardModel turns the command into the savable model +// GetDashboardModel turns the command into the saveable model func (cmd *CreateFolderCommand) GetDashboardModel(orgId int64, userId int64) *Dashboard { dashFolder := NewDashboardFolder(strings.TrimSpace(cmd.Title)) dashFolder.OrgId = orgId @@ -89,3 +89,12 @@ type UpdateFolderCommand struct { Result *Folder } + +// +// QUERIES +// + +type HasEditPermissionInFoldersQuery struct { + SignedInUser *SignedInUser + Result bool +} diff --git a/pkg/models/models.go b/pkg/models/models.go index c2560021ee1..ba894ae591f 100644 --- a/pkg/models/models.go +++ b/pkg/models/models.go @@ -8,4 +8,5 @@ const ( TWITTER GENERIC GRAFANA_COM + GITLAB ) diff --git a/pkg/models/notifications.go b/pkg/models/notifications.go index 089d7c4360d..4b25ecb4dc7 100644 --- a/pkg/models/notifications.go +++ b/pkg/models/notifications.go @@ -19,12 +19,13 @@ type SendEmailCommandSync struct { } type SendWebhookSync struct { - Url string - User string - Password string - Body string - HttpMethod string - HttpHeader map[string]string + Url string + User string + Password string + Body string + HttpMethod string + HttpHeader map[string]string + ContentType string } type SendResetPasswordEmailCommand struct { diff --git a/pkg/models/org_user.go b/pkg/models/org_user.go index ca32cc50060..b6ecd924e9a 100644 --- a/pkg/models/org_user.go +++ b/pkg/models/org_user.go @@ -48,9 +48,9 @@ func (r *RoleType) UnmarshalJSON(data []byte) error { *r = RoleType(str) - if (*r).IsValid() == false { + if !(*r).IsValid() { if (*r) != "" { - return errors.New(fmt.Sprintf("JSON validation error: invalid role value: %s", *r)) + return fmt.Errorf("JSON validation error: invalid role value: %s", *r) } *r = ROLE_VIEWER @@ -72,8 +72,10 @@ type OrgUser struct { // COMMANDS type RemoveOrgUserCommand struct { - UserId int64 - OrgId int64 + UserId int64 + OrgId int64 + ShouldDeleteOrphanedUser bool + UserWasDeleted bool } type AddOrgUserCommand struct { diff --git a/pkg/models/playlist.go b/pkg/models/playlist.go index 5c49bb9256c..c52da202293 100644 --- a/pkg/models/playlist.go +++ b/pkg/models/playlist.go @@ -63,7 +63,7 @@ type PlaylistDashboards []*PlaylistDashboard type UpdatePlaylistCommand struct { OrgId int64 `json:"-"` - Id int64 `json:"id" binding:"Required"` + Id int64 `json:"id"` Name string `json:"name" binding:"Required"` Interval string `json:"interval"` Items []PlaylistItemDTO `json:"items"` diff --git a/pkg/models/preferences.go b/pkg/models/preferences.go index 4c77bc96d4d..c73e0be4949 100644 --- a/pkg/models/preferences.go +++ b/pkg/models/preferences.go @@ -14,6 +14,7 @@ type Preferences struct { Id int64 OrgId int64 UserId int64 + TeamId int64 Version int HomeDashboardId int64 Timezone string @@ -29,14 +30,13 @@ type GetPreferencesQuery struct { Id int64 OrgId int64 UserId int64 + TeamId int64 Result *Preferences } type GetPreferencesWithDefaultsQuery struct { - Id int64 - OrgId int64 - UserId int64 + User *SignedInUser Result *Preferences } @@ -46,6 +46,7 @@ type GetPreferencesWithDefaultsQuery struct { type SavePreferencesCommand struct { UserId int64 OrgId int64 + TeamId int64 HomeDashboardId int64 `json:"homeDashboardId"` Timezone string `json:"timezone"` diff --git a/pkg/models/stats.go b/pkg/models/stats.go index e132d88c030..d3e145dedf4 100644 --- a/pkg/models/stats.go +++ b/pkg/models/stats.go @@ -1,14 +1,20 @@ package models type SystemStats struct { - Dashboards int64 - Datasources int64 - Users int64 - ActiveUsers int64 - Orgs int64 - Playlists int64 - Alerts int64 - Stars int64 + Dashboards int64 + Datasources int64 + Users int64 + ActiveUsers int64 + Orgs int64 + Playlists int64 + Alerts int64 + Stars int64 + Snapshots int64 + Teams int64 + DashboardPermissions int64 + FolderPermissions int64 + Folders int64 + ProvisionedDashboards int64 } type DataSourceStats struct { @@ -24,6 +30,25 @@ type GetDataSourceStatsQuery struct { Result []*DataSourceStats } +type DataSourceAccessStats struct { + Type string + Access string + Count int64 +} + +type GetDataSourceAccessStatsQuery struct { + Result []*DataSourceAccessStats +} + +type NotifierUsageStats struct { + Type string + Count int64 +} + +type GetAlertNotifierUsageStatsQuery struct { + Result []*NotifierUsageStats +} + type AdminStats struct { Users int `json:"users"` Orgs int `json:"orgs"` @@ -40,3 +65,11 @@ type AdminStats struct { type GetAdminStatsQuery struct { Result *AdminStats } + +type SystemUserCountStats struct { + Count int64 +} + +type GetSystemUserCountStatsQuery struct { + Result *SystemUserCountStats +} diff --git a/pkg/models/team.go b/pkg/models/team.go index 9c679a13394..61285db3a5f 100644 --- a/pkg/models/team.go +++ b/pkg/models/team.go @@ -49,13 +49,13 @@ type DeleteTeamCommand struct { type GetTeamByIdQuery struct { OrgId int64 Id int64 - Result *Team + Result *TeamDTO } type GetTeamsByUserQuery struct { OrgId int64 - UserId int64 `json:"userId"` - Result []*Team `json:"teams"` + UserId int64 `json:"userId"` + Result []*TeamDTO `json:"teams"` } type SearchTeamsQuery struct { @@ -68,7 +68,7 @@ type SearchTeamsQuery struct { Result SearchTeamQueryResult } -type SearchTeamDto struct { +type TeamDTO struct { Id int64 `json:"id"` OrgId int64 `json:"orgId"` Name string `json:"name"` @@ -78,8 +78,8 @@ type SearchTeamDto struct { } type SearchTeamQueryResult struct { - TotalCount int64 `json:"totalCount"` - Teams []*SearchTeamDto `json:"teams"` - Page int `json:"page"` - PerPage int `json:"perPage"` + TotalCount int64 `json:"totalCount"` + Teams []*TeamDTO `json:"teams"` + Page int `json:"page"` + PerPage int `json:"perPage"` } diff --git a/pkg/models/team_member.go b/pkg/models/team_member.go index 19cf657292d..dd64787f465 100644 --- a/pkg/models/team_member.go +++ b/pkg/models/team_member.go @@ -12,10 +12,11 @@ var ( // TeamMember model type TeamMember struct { - Id int64 - OrgId int64 - TeamId int64 - UserId int64 + Id int64 + OrgId int64 + TeamId int64 + UserId int64 + External bool Created time.Time Updated time.Time @@ -25,9 +26,10 @@ type TeamMember struct { // COMMANDS type AddTeamMemberCommand struct { - UserId int64 `json:"userId" binding:"Required"` - OrgId int64 `json:"-"` - TeamId int64 `json:"-"` + UserId int64 `json:"userId" binding:"Required"` + OrgId int64 `json:"-"` + TeamId int64 `json:"-"` + External bool `json:"-"` } type RemoveTeamMemberCommand struct { @@ -40,19 +42,23 @@ type RemoveTeamMemberCommand struct { // QUERIES type GetTeamMembersQuery struct { - OrgId int64 - TeamId int64 - Result []*TeamMemberDTO + OrgId int64 + TeamId int64 + UserId int64 + External bool + Result []*TeamMemberDTO } // ---------------------- // Projections and DTOs type TeamMemberDTO struct { - OrgId int64 `json:"orgId"` - TeamId int64 `json:"teamId"` - UserId int64 `json:"userId"` - Email string `json:"email"` - Login string `json:"login"` - AvatarUrl string `json:"avatarUrl"` + OrgId int64 `json:"orgId"` + TeamId int64 `json:"teamId"` + UserId int64 `json:"userId"` + External bool `json:"-"` + Email string `json:"email"` + Login string `json:"login"` + AvatarUrl string `json:"avatarUrl"` + Labels []string `json:"labels"` } diff --git a/pkg/models/user.go b/pkg/models/user.go index d5b912e0a9c..e3c7b556d35 100644 --- a/pkg/models/user.go +++ b/pkg/models/user.go @@ -165,6 +165,7 @@ type SignedInUser struct { IsAnonymous bool HelpFlags1 HelpFlags1 LastSeenAt time.Time + Teams []int64 } func (u *SignedInUser) ShouldUpdateLastSeenAt() bool { diff --git a/pkg/models/user_auth.go b/pkg/models/user_auth.go new file mode 100644 index 00000000000..28189005737 --- /dev/null +++ b/pkg/models/user_auth.go @@ -0,0 +1,79 @@ +package models + +import ( + "time" +) + +type UserAuth struct { + Id int64 + UserId int64 + AuthModule string + AuthId string + Created time.Time +} + +type ExternalUserInfo struct { + AuthModule string + AuthId string + UserId int64 + Email string + Login string + Name string + Groups []string + OrgRoles map[int64]RoleType + IsGrafanaAdmin *bool // This is a pointer to know if we should sync this or not (nil = ignore sync) +} + +// --------------------- +// COMMANDS + +type UpsertUserCommand struct { + ReqContext *ReqContext + ExternalUser *ExternalUserInfo + SignupAllowed bool + + Result *User +} + +type SetAuthInfoCommand struct { + AuthModule string + AuthId string + UserId int64 +} + +type DeleteAuthInfoCommand struct { + UserAuth *UserAuth +} + +// ---------------------- +// QUERIES + +type LoginUserQuery struct { + ReqContext *ReqContext + Username string + Password string + User *User + IpAddress string +} + +type GetUserByAuthInfoQuery struct { + AuthModule string + AuthId string + UserId int64 + Email string + Login string + + Result *User +} + +type GetAuthInfoQuery struct { + AuthModule string + AuthId string + + Result *UserAuth +} + +type SyncTeamsCommand struct { + ExternalUser *ExternalUserInfo + User *User +} diff --git a/pkg/plugins/app_plugin.go b/pkg/plugins/app_plugin.go index b070ba592f0..922b2444b7b 100644 --- a/pkg/plugins/app_plugin.go +++ b/pkg/plugins/app_plugin.go @@ -23,12 +23,13 @@ type AppPlugin struct { } type AppPluginRoute struct { - Path string `json:"path"` - Method string `json:"method"` - ReqRole models.RoleType `json:"reqRole"` - Url string `json:"url"` - Headers []AppPluginRouteHeader `json:"headers"` - TokenAuth *JwtTokenAuth `json:"tokenAuth"` + Path string `json:"path"` + Method string `json:"method"` + ReqRole models.RoleType `json:"reqRole"` + Url string `json:"url"` + Headers []AppPluginRouteHeader `json:"headers"` + TokenAuth *JwtTokenAuth `json:"tokenAuth"` + JwtTokenAuth *JwtTokenAuth `json:"jwtTokenAuth"` } type AppPluginRouteHeader struct { @@ -36,8 +37,11 @@ type AppPluginRouteHeader struct { Content string `json:"content"` } +// JwtTokenAuth struct is both for normal Token Auth and JWT Token Auth with +// an uploaded JWT file. type JwtTokenAuth struct { Url string `json:"url"` + Scopes []string `json:"scopes"` Params map[string]string `json:"params"` } diff --git a/pkg/plugins/backend_utils.go b/pkg/plugins/backend_utils.go new file mode 100644 index 00000000000..d3ee32f0545 --- /dev/null +++ b/pkg/plugins/backend_utils.go @@ -0,0 +1,19 @@ +package plugins + +import ( + "fmt" + "runtime" + "strings" +) + +func ComposePluginStartCommmand(executable string) string { + os := strings.ToLower(runtime.GOOS) + arch := runtime.GOARCH + extension := "" + + if os == "windows" { + extension = ".exe" + } + + return fmt.Sprintf("%s_%s_%s%s", executable, os, strings.ToLower(arch), extension) +} diff --git a/pkg/plugins/dashboard_importer.go b/pkg/plugins/dashboard_importer.go index fb4d63a1fe4..9b319358780 100644 --- a/pkg/plugins/dashboard_importer.go +++ b/pkg/plugins/dashboard_importer.go @@ -16,6 +16,7 @@ type ImportDashboardCommand struct { Path string Inputs []ImportDashboardInput Overwrite bool + FolderId int64 OrgId int64 User *m.SignedInUser @@ -70,7 +71,7 @@ func ImportDashboard(cmd *ImportDashboardCommand) error { UserId: cmd.User.UserId, Overwrite: cmd.Overwrite, PluginId: cmd.PluginId, - FolderId: dashboard.FolderId, + FolderId: cmd.FolderId, } dto := &dashboards.SaveDashboardDTO{ @@ -91,6 +92,7 @@ func ImportDashboard(cmd *ImportDashboardCommand) error { Title: savedDash.Title, Path: cmd.Path, Revision: savedDash.Data.Get("revision").MustInt64(1), + FolderId: savedDash.FolderId, ImportedUri: "db/" + savedDash.Slug, ImportedUrl: savedDash.GetUrl(), ImportedRevision: dashboard.Data.Get("revision").MustInt64(1), @@ -148,11 +150,11 @@ func (this *DashTemplateEvaluator) evalValue(source *simplejson.Json) interface{ switch v := sourceValue.(type) { case string: interpolated := this.varRegex.ReplaceAllStringFunc(v, func(match string) string { - if replacement, exists := this.variables[match]; exists { + replacement, exists := this.variables[match] + if exists { return replacement - } else { - return match } + return match }) return interpolated case bool: diff --git a/pkg/plugins/dashboard_importer_test.go b/pkg/plugins/dashboard_importer_test.go index 549b3bb4cf9..ca8dfcd515c 100644 --- a/pkg/plugins/dashboard_importer_test.go +++ b/pkg/plugins/dashboard_importer_test.go @@ -1,7 +1,6 @@ package plugins import ( - "context" "io/ioutil" "testing" @@ -36,7 +35,7 @@ func TestDashboardImport(t *testing.T) { So(cmd.Result, ShouldNotBeNil) resultStr, _ := mock.SavedDashboards[0].Dashboard.Data.EncodePretty() - expectedBytes, _ := ioutil.ReadFile("../../tests/test-app/dashboards/connections_result.json") + expectedBytes, _ := ioutil.ReadFile("testdata/test-app/dashboards/connections_result.json") expectedJson, _ := simplejson.NewJson(expectedBytes) expectedStr, _ := expectedJson.EncodePretty() @@ -88,13 +87,14 @@ func TestDashboardImport(t *testing.T) { func pluginScenario(desc string, t *testing.T, fn func()) { Convey("Given a plugin", t, func() { - setting.Cfg = ini.Empty() - sec, _ := setting.Cfg.NewSection("plugin.test-app") - sec.NewKey("path", "../../tests/test-app") - err := initPlugins(context.Background()) + setting.Raw = ini.Empty() + sec, _ := setting.Raw.NewSection("plugin.test-app") + sec.NewKey("path", "testdata/test-app") + + pm := &PluginManager{} + err := pm.Init() So(err, ShouldBeNil) - Convey(desc, fn) }) } diff --git a/pkg/plugins/dashboards.go b/pkg/plugins/dashboards.go index d15bcdd6db5..500d97e38ca 100644 --- a/pkg/plugins/dashboards.go +++ b/pkg/plugins/dashboards.go @@ -17,6 +17,7 @@ type PluginDashboardInfoDTO struct { ImportedUrl string `json:"importedUrl"` Slug string `json:"slug"` DashboardId int64 `json:"dashboardId"` + FolderId int64 `json:"folderId"` ImportedRevision int64 `json:"importedRevision"` Revision int64 `json:"revision"` Description string `json:"description"` diff --git a/pkg/plugins/dashboards_test.go b/pkg/plugins/dashboards_test.go index 8573d452409..6fc6ace0e00 100644 --- a/pkg/plugins/dashboards_test.go +++ b/pkg/plugins/dashboards_test.go @@ -1,7 +1,6 @@ package plugins import ( - "context" "testing" "github.com/grafana/grafana/pkg/bus" @@ -15,10 +14,12 @@ import ( func TestPluginDashboards(t *testing.T) { Convey("When asking plugin dashboard info", t, func() { - setting.Cfg = ini.Empty() - sec, _ := setting.Cfg.NewSection("plugin.test-app") - sec.NewKey("path", "../../tests/test-app") - err := initPlugins(context.Background()) + setting.Raw = ini.Empty() + sec, _ := setting.Raw.NewSection("plugin.test-app") + sec.NewKey("path", "testdata/test-app") + + pm := &PluginManager{} + err := pm.Init() So(err, ShouldBeNil) diff --git a/pkg/plugins/dashboards_updater.go b/pkg/plugins/dashboards_updater.go index 835e8873810..616d4541bec 100644 --- a/pkg/plugins/dashboards_updater.go +++ b/pkg/plugins/dashboards_updater.go @@ -1,8 +1,6 @@ package plugins import ( - "time" - "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" ) @@ -11,10 +9,8 @@ func init() { bus.AddEventListener(handlePluginStateChanged) } -func updateAppDashboards() { - time.Sleep(time.Second * 5) - - plog.Debug("Looking for App Dashboard Updates") +func (pm *PluginManager) updateAppDashboards() { + pm.log.Debug("Looking for App Dashboard Updates") query := m.GetPluginSettingsQuery{OrgId: 0} @@ -38,24 +34,21 @@ func updateAppDashboards() { } func autoUpdateAppDashboard(pluginDashInfo *PluginDashboardInfoDTO, orgId int64) error { - if dash, err := loadPluginDashboard(pluginDashInfo.PluginId, pluginDashInfo.Path); err != nil { + dash, err := loadPluginDashboard(pluginDashInfo.PluginId, pluginDashInfo.Path) + if err != nil { return err - } else { - plog.Info("Auto updating App dashboard", "dashboard", dash.Title, "newRev", pluginDashInfo.Revision, "oldRev", pluginDashInfo.ImportedRevision) - updateCmd := ImportDashboardCommand{ - OrgId: orgId, - PluginId: pluginDashInfo.PluginId, - Overwrite: true, - Dashboard: dash.Data, - User: &m.SignedInUser{UserId: 0, OrgRole: m.ROLE_ADMIN}, - Path: pluginDashInfo.Path, - } - - if err := bus.Dispatch(&updateCmd); err != nil { - return err - } } - return nil + plog.Info("Auto updating App dashboard", "dashboard", dash.Title, "newRev", pluginDashInfo.Revision, "oldRev", pluginDashInfo.ImportedRevision) + updateCmd := ImportDashboardCommand{ + OrgId: orgId, + PluginId: pluginDashInfo.PluginId, + Overwrite: true, + Dashboard: dash.Data, + User: &m.SignedInUser{UserId: 0, OrgRole: m.ROLE_ADMIN}, + Path: pluginDashInfo.Path, + } + + return bus.Dispatch(&updateCmd) } func syncPluginDashboards(pluginDef *PluginBase, orgId int64) { @@ -122,15 +115,14 @@ func handlePluginStateChanged(event *m.PluginStateChangedEvent) error { if err := bus.Dispatch(&query); err != nil { return err - } else { - for _, dash := range query.Result { - deleteCmd := m.DeleteDashboardCommand{OrgId: dash.OrgId, Id: dash.Id} + } + for _, dash := range query.Result { + deleteCmd := m.DeleteDashboardCommand{OrgId: dash.OrgId, Id: dash.Id} - plog.Info("Deleting plugin dashboard", "pluginId", event.PluginId, "dashboard", dash.Slug) + plog.Info("Deleting plugin dashboard", "pluginId", event.PluginId, "dashboard", dash.Slug) - if err := bus.Dispatch(&deleteCmd); err != nil { - return err - } + if err := bus.Dispatch(&deleteCmd); err != nil { + return err } } } diff --git a/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go index 170e187b282..0af727e14df 100644 --- a/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go +++ b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper.go @@ -5,11 +5,12 @@ import ( "errors" "fmt" + "github.com/grafana/grafana-plugin-model/go/datasource" "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" - "github.com/grafana/grafana_plugin_model/go/datasource" ) func NewDatasourcePluginWrapper(log log.Logger, plugin datasource.DatasourcePlugin) *DatasourcePluginWrapper { @@ -79,6 +80,14 @@ func (tw *DatasourcePluginWrapper) Query(ctx context.Context, ds *models.DataSou qr.ErrorString = r.Error } + if r.MetaJson != "" { + metaJson, err := simplejson.NewJson([]byte(r.MetaJson)) + if err != nil { + tw.logger.Error("Error parsing JSON Meta field: " + err.Error()) + } + qr.Meta = metaJson + } + for _, s := range r.GetSeries() { points := tsdb.TimeSeriesPoints{} diff --git a/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_test.go b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_test.go index 834e8238e3a..e312913fc56 100644 --- a/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_test.go +++ b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_test.go @@ -3,9 +3,9 @@ package wrapper import ( "testing" + "github.com/grafana/grafana-plugin-model/go/datasource" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/tsdb" - "github.com/grafana/grafana_plugin_model/go/datasource" ) func TestMapTables(t *testing.T) { @@ -74,7 +74,7 @@ func TestMappingRowValue(t *testing.T) { boolRowValue, _ := dpw.mapRowValue(&datasource.RowValue{Kind: datasource.RowValue_TYPE_BOOL, BoolValue: true}) haveBool, ok := boolRowValue.(bool) - if !ok || haveBool != true { + if !ok || !haveBool { t.Fatalf("Expected true, was %v", haveBool) } diff --git a/pkg/plugins/datasource_plugin.go b/pkg/plugins/datasource_plugin.go index 37ce175efe4..ff44805e35f 100644 --- a/pkg/plugins/datasource_plugin.go +++ b/pkg/plugins/datasource_plugin.go @@ -3,28 +3,28 @@ package plugins import ( "context" "encoding/json" - "fmt" "os" "os/exec" "path" "path/filepath" - "runtime" - "strings" "time" + "github.com/grafana/grafana-plugin-model/go/datasource" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins/datasource/wrapper" "github.com/grafana/grafana/pkg/tsdb" - "github.com/grafana/grafana_plugin_model/go/datasource" plugin "github.com/hashicorp/go-plugin" ) +// DataSourcePlugin contains all metadata about a datasource plugin type DataSourcePlugin struct { FrontendPluginBase Annotations bool `json:"annotations"` Metrics bool `json:"metrics"` Alerting bool `json:"alerting"` + Explore bool `json:"explore"` + Logs bool `json:"logs"` QueryOptions map[string]bool `json:"queryOptions,omitempty"` BuiltIn bool `json:"builtIn,omitempty"` Mixed bool `json:"mixed,omitempty"` @@ -66,17 +66,7 @@ var handshakeConfig = plugin.HandshakeConfig{ MagicCookieValue: "datasource", } -func composeBinaryName(executable, os, arch string) string { - var extension string - os = strings.ToLower(os) - if os == "windows" { - extension = ".exe" - } - - return fmt.Sprintf("%s_%s_%s%s", executable, os, strings.ToLower(arch), extension) -} - -func (p *DataSourcePlugin) initBackendPlugin(ctx context.Context, log log.Logger) error { +func (p *DataSourcePlugin) startBackendPlugin(ctx context.Context, log log.Logger) error { p.log = log.New("plugin-id", p.Id) err := p.spawnSubProcess() @@ -88,7 +78,7 @@ func (p *DataSourcePlugin) initBackendPlugin(ctx context.Context, log log.Logger } func (p *DataSourcePlugin) spawnSubProcess() error { - cmd := composeBinaryName(p.Executable, runtime.GOOS, runtime.GOARCH) + cmd := ComposePluginStartCommmand(p.Executable) fullpath := path.Join(p.PluginDir, cmd) p.client = plugin.NewClient(&plugin.ClientConfig{ diff --git a/pkg/plugins/datasource_plugin_test.go b/pkg/plugins/datasource_plugin_test.go deleted file mode 100644 index 147f0310f5c..00000000000 --- a/pkg/plugins/datasource_plugin_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package plugins - -import ( - "testing" -) - -func TestComposeBinaryName(t *testing.T) { - tests := []struct { - name string - os string - arch string - - expectedPath string - }{ - { - name: "simple-json", - os: "linux", - arch: "amd64", - expectedPath: `simple-json_linux_amd64`, - }, - { - name: "simple-json", - os: "windows", - arch: "amd64", - expectedPath: `simple-json_windows_amd64.exe`, - }, - } - - for _, v := range tests { - have := composeBinaryName(v.name, v.os, v.arch) - if have != v.expectedPath { - t.Errorf("expected %s got %s", v.expectedPath, have) - } - } -} diff --git a/pkg/plugins/models.go b/pkg/plugins/models.go index 541b37c8a8a..5ac436205c1 100644 --- a/pkg/plugins/models.go +++ b/pkg/plugins/models.go @@ -17,6 +17,13 @@ var ( PluginTypeDashboard = "dashboard" ) +type PluginState string + +var ( + PluginStateAlpha PluginState = "alpha" + PluginStateBeta PluginState = "beta" +) + type PluginNotFoundError struct { PluginId string } @@ -39,7 +46,7 @@ type PluginBase struct { Module string `json:"module"` BaseUrl string `json:"baseUrl"` HideFromList bool `json:"hideFromList,omitempty"` - State string `json:"state,omitempty"` + State PluginState `json:"state,omitempty"` IncludedInAppId string `json:"-"` PluginDir string `json:"-"` @@ -69,7 +76,7 @@ func (pb *PluginBase) registerPlugin(pluginDir string) error { for _, include := range pb.Includes { if include.Role == "" { - include.Role = m.RoleType(m.ROLE_VIEWER) + include.Role = m.ROLE_VIEWER } } diff --git a/pkg/plugins/plugins.go b/pkg/plugins/plugins.go index 417f565dd0c..4f15441bb2f 100644 --- a/pkg/plugins/plugins.go +++ b/pkg/plugins/plugins.go @@ -11,8 +11,10 @@ import ( "path/filepath" "reflect" "strings" + "time" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -24,6 +26,7 @@ var ( Apps map[string]*AppPlugin Plugins map[string]*PluginBase PluginTypes map[string]interface{} + Renderer *RendererPlugin GrafanaLatestVersion string GrafanaHasUpdate bool @@ -39,30 +42,12 @@ type PluginManager struct { log log.Logger } -func NewPluginManager(ctx context.Context) (*PluginManager, error) { - err := initPlugins(ctx) - - if err != nil { - return nil, err - } - - return &PluginManager{ - log: log.New("plugins"), - }, nil +func init() { + registry.RegisterService(&PluginManager{}) } -func (p *PluginManager) Run(ctx context.Context) error { - <-ctx.Done() - - for _, p := range DataSources { - p.Kill() - } - - p.log.Info("Stopped Plugins", "error", ctx.Err()) - return ctx.Err() -} - -func initPlugins(ctx context.Context) error { +func (pm *PluginManager) Init() error { + pm.log = log.New("plugins") plog = log.New("plugins") DataSources = map[string]*DataSourcePlugin{} @@ -74,9 +59,10 @@ func initPlugins(ctx context.Context) error { "panel": PanelPlugin{}, "datasource": DataSourcePlugin{}, "app": AppPlugin{}, + "renderer": RendererPlugin{}, } - plog.Info("Starting plugin search") + pm.log.Info("Starting plugin search") scan(path.Join(setting.StaticRootPath, "app/plugins")) // check if plugins dir exists @@ -99,13 +85,6 @@ func initPlugins(ctx context.Context) error { } for _, ds := range DataSources { - if ds.Backend { - err := ds.initBackendPlugin(ctx, plog) - if err != nil { - plog.Error("Failed to init plugin.", "error", err, "plugin", ds.Id) - } - } - ds.initFrontendPlugin() } @@ -113,14 +92,48 @@ func initPlugins(ctx context.Context) error { app.initApp() } - go StartPluginUpdateChecker() - go updateAppDashboards() + return nil +} + +func (pm *PluginManager) startBackendPlugins(ctx context.Context) error { + for _, ds := range DataSources { + if ds.Backend { + if err := ds.startBackendPlugin(ctx, plog); err != nil { + pm.log.Error("Failed to init plugin.", "error", err, "plugin", ds.Id) + } + } + } return nil } +func (pm *PluginManager) Run(ctx context.Context) error { + pm.startBackendPlugins(ctx) + pm.updateAppDashboards() + pm.checkForUpdates() + + ticker := time.NewTicker(time.Minute * 10) + run := true + + for run { + select { + case <-ticker.C: + pm.checkForUpdates() + case <-ctx.Done(): + run = false + } + } + + // kil backend plugins + for _, p := range DataSources { + p.Kill() + } + + return ctx.Err() +} + func checkPluginPaths() error { - for _, section := range setting.Cfg.Sections() { + for _, section := range setting.Raw.Sections() { if strings.HasPrefix(section.Name(), "plugin.") { path := section.Key("path").String() if path != "" { @@ -193,11 +206,11 @@ func (scanner *PluginScanner) loadPluginJson(pluginJsonFilePath string) error { } var loader PluginLoader - if pluginGoType, exists := PluginTypes[pluginCommon.Type]; !exists { + pluginGoType, exists := PluginTypes[pluginCommon.Type] + if !exists { return errors.New("Unknown plugin type " + pluginCommon.Type) - } else { - loader = reflect.New(reflect.TypeOf(pluginGoType)).Interface().(PluginLoader) } + loader = reflect.New(reflect.TypeOf(pluginGoType)).Interface().(PluginLoader) reader.Seek(0, 0) return loader.Load(jsonParser, currentDir) @@ -218,9 +231,9 @@ func GetPluginMarkdown(pluginId string, name string) ([]byte, error) { return make([]byte, 0), nil } - if data, err := ioutil.ReadFile(path); err != nil { + data, err := ioutil.ReadFile(path) + if err != nil { return nil, err - } else { - return data, nil } + return data, nil } diff --git a/pkg/plugins/plugins_test.go b/pkg/plugins/plugins_test.go index 4d3ccb4502b..d16e6abb4c7 100644 --- a/pkg/plugins/plugins_test.go +++ b/pkg/plugins/plugins_test.go @@ -1,7 +1,6 @@ package plugins import ( - "context" "path/filepath" "testing" @@ -12,10 +11,12 @@ import ( func TestPluginScans(t *testing.T) { - Convey("When scaning for plugins", t, func() { + Convey("When scanning for plugins", t, func() { setting.StaticRootPath, _ = filepath.Abs("../../public/") - setting.Cfg = ini.Empty() - err := initPlugins(context.Background()) + setting.Raw = ini.Empty() + + pm := &PluginManager{} + err := pm.Init() So(err, ShouldBeNil) So(len(DataSources), ShouldBeGreaterThan, 1) @@ -27,10 +28,12 @@ func TestPluginScans(t *testing.T) { }) Convey("When reading app plugin definition", t, func() { - setting.Cfg = ini.Empty() - sec, _ := setting.Cfg.NewSection("plugin.nginx-app") - sec.NewKey("path", "../../tests/test-app") - err := initPlugins(context.Background()) + setting.Raw = ini.Empty() + sec, _ := setting.Raw.NewSection("plugin.nginx-app") + sec.NewKey("path", "testdata/test-app") + + pm := &PluginManager{} + err := pm.Init() So(err, ShouldBeNil) So(len(Apps), ShouldBeGreaterThan, 0) diff --git a/pkg/plugins/queries.go b/pkg/plugins/queries.go index 5ae1825a88f..5bd412d2cc9 100644 --- a/pkg/plugins/queries.go +++ b/pkg/plugins/queries.go @@ -37,7 +37,7 @@ func GetPluginSettings(orgId int64) (map[string]*m.PluginSettingInfoDTO, error) // if it's included in app check app settings if pluginDef.IncludedInAppId != "" { - // app componets are by default disabled + // app components are by default disabled opt.Enabled = false if appSettings, ok := pluginMap[pluginDef.IncludedInAppId]; ok { diff --git a/pkg/plugins/renderer_plugin.go b/pkg/plugins/renderer_plugin.go new file mode 100644 index 00000000000..286c670eb41 --- /dev/null +++ b/pkg/plugins/renderer_plugin.go @@ -0,0 +1,22 @@ +package plugins + +import "encoding/json" + +type RendererPlugin struct { + PluginBase + + Executable string `json:"executable,omitempty"` +} + +func (r *RendererPlugin) Load(decoder *json.Decoder, pluginDir string) error { + if err := decoder.Decode(&r); err != nil { + return err + } + + if err := r.registerPlugin(pluginDir); err != nil { + return err + } + + Renderer = r + return nil +} diff --git a/tests/datasource-test/module.js b/pkg/plugins/testdata/datasource-test/module.js similarity index 100% rename from tests/datasource-test/module.js rename to pkg/plugins/testdata/datasource-test/module.js diff --git a/tests/datasource-test/plugin.json b/pkg/plugins/testdata/datasource-test/plugin.json similarity index 100% rename from tests/datasource-test/plugin.json rename to pkg/plugins/testdata/datasource-test/plugin.json diff --git a/tests/test-app/dashboards/connections.json b/pkg/plugins/testdata/test-app/dashboards/connections.json similarity index 100% rename from tests/test-app/dashboards/connections.json rename to pkg/plugins/testdata/test-app/dashboards/connections.json diff --git a/tests/test-app/dashboards/connections_result.json b/pkg/plugins/testdata/test-app/dashboards/connections_result.json similarity index 100% rename from tests/test-app/dashboards/connections_result.json rename to pkg/plugins/testdata/test-app/dashboards/connections_result.json diff --git a/tests/test-app/dashboards/memory.json b/pkg/plugins/testdata/test-app/dashboards/memory.json similarity index 100% rename from tests/test-app/dashboards/memory.json rename to pkg/plugins/testdata/test-app/dashboards/memory.json diff --git a/tests/test-app/plugin.json b/pkg/plugins/testdata/test-app/plugin.json similarity index 100% rename from tests/test-app/plugin.json rename to pkg/plugins/testdata/test-app/plugin.json diff --git a/pkg/plugins/update_checker.go b/pkg/plugins/update_checker.go index 68ccdeaf840..e61f4cf1df7 100644 --- a/pkg/plugins/update_checker.go +++ b/pkg/plugins/update_checker.go @@ -13,7 +13,7 @@ import ( ) var ( - httpClient http.Client = http.Client{Timeout: time.Duration(10 * time.Second)} + httpClient = http.Client{Timeout: 10 * time.Second} ) type GrafanaNetPlugin struct { @@ -26,23 +26,6 @@ type GithubLatest struct { Testing string `json:"testing"` } -func StartPluginUpdateChecker() { - if !setting.CheckForUpdates { - return - } - - // do one check directly - go checkForUpdates() - - ticker := time.NewTicker(time.Minute * 10) - for { - select { - case <-ticker.C: - checkForUpdates() - } - } -} - func getAllExternalPluginSlugs() string { var result []string for _, plug := range Plugins { @@ -56,8 +39,12 @@ func getAllExternalPluginSlugs() string { return strings.Join(result, ",") } -func checkForUpdates() { - log.Trace("Checking for updates") +func (pm *PluginManager) checkForUpdates() { + if !setting.CheckForUpdates { + return + } + + pm.log.Debug("Checking for updates") pluginSlugs := getAllExternalPluginSlugs() resp, err := httpClient.Get("https://grafana.com/api/plugins/versioncheck?slugIn=" + pluginSlugs + "&grafanaVersion=" + setting.BuildVersion) diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go new file mode 100644 index 00000000000..487a6db7927 --- /dev/null +++ b/pkg/registry/registry.go @@ -0,0 +1,119 @@ +package registry + +import ( + "context" + "reflect" + "sort" + + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +) + +type Descriptor struct { + Name string + Instance Service + InitPriority Priority +} + +var services []*Descriptor + +func RegisterService(instance Service) { + services = append(services, &Descriptor{ + Name: reflect.TypeOf(instance).Elem().Name(), + Instance: instance, + InitPriority: Low, + }) +} + +func Register(descriptor *Descriptor) { + services = append(services, descriptor) +} + +func GetServices() []*Descriptor { + slice := getServicesWithOverrides() + + sort.Slice(slice, func(i, j int) bool { + return slice[i].InitPriority > slice[j].InitPriority + }) + + return slice +} + +type OverrideServiceFunc func(descriptor Descriptor) (*Descriptor, bool) + +var overrides []OverrideServiceFunc + +func RegisterOverride(fn OverrideServiceFunc) { + overrides = append(overrides, fn) +} + +func getServicesWithOverrides() []*Descriptor { + slice := []*Descriptor{} + for _, s := range services { + var descriptor *Descriptor + for _, fn := range overrides { + if newDescriptor, override := fn(*s); override { + descriptor = newDescriptor + break + } + } + + if descriptor != nil { + slice = append(slice, descriptor) + } else { + slice = append(slice, s) + } + } + + return slice +} + +// Service interface is the lowest common shape that services +// are expected to forfill to be started within Grafana. +type Service interface { + + // Init is called by Grafana main process which gives the service + // the possibility do some initial work before its started. Things + // like adding routes, bus handlers should be done in the Init function + Init() error +} + +// CanBeDisabled allows the services to decide if it should +// be started or not by itself. This is useful for services +// that might not always be started, ex alerting. +// This will be called after `Init()`. +type CanBeDisabled interface { + + // IsDisabled should return a bool saying if it can be started or not. + IsDisabled() bool +} + +// BackgroundService should be implemented for services that have +// long running tasks in the background. +type BackgroundService interface { + // Run starts the background process of the service after `Init` have been called + // on all services. The `context.Context` passed into the function should be used + // to subscribe to ctx.Done() so the service can be notified when Grafana shuts down. + Run(ctx context.Context) error +} + +// DatabaseMigrator allows the caller to add migrations to +// the migrator passed as argument +type DatabaseMigrator interface { + + // AddMigrations allows the service to add migrations to + // the database migrator. + AddMigration(mg *migrator.Migrator) +} + +// IsDisabled takes an service and return true if its disabled +func IsDisabled(srv Service) bool { + canBeDisabled, ok := srv.(CanBeDisabled) + return ok && canBeDisabled.IsDisabled() +} + +type Priority int + +const ( + High Priority = 100 + Low Priority = 0 +) diff --git a/pkg/services/alerting/commands.go b/pkg/services/alerting/commands.go index 2c145614751..dd2ff5658d6 100644 --- a/pkg/services/alerting/commands.go +++ b/pkg/services/alerting/commands.go @@ -11,33 +11,26 @@ func init() { } func validateDashboardAlerts(cmd *m.ValidateDashboardAlertsCommand) error { - extractor := NewDashAlertExtractor(cmd.Dashboard, cmd.OrgId) + extractor := NewDashAlertExtractor(cmd.Dashboard, cmd.OrgId, cmd.User) - if _, err := extractor.GetAlerts(); err != nil { - return err - } - - return nil + return extractor.ValidateAlerts() } func updateDashboardAlerts(cmd *m.UpdateDashboardAlertsCommand) error { saveAlerts := m.SaveAlertsCommand{ OrgId: cmd.OrgId, - UserId: cmd.UserId, + UserId: cmd.User.UserId, DashboardId: cmd.Dashboard.Id, } - extractor := NewDashAlertExtractor(cmd.Dashboard, cmd.OrgId) + extractor := NewDashAlertExtractor(cmd.Dashboard, cmd.OrgId, cmd.User) - if alerts, err := extractor.GetAlerts(); err != nil { - return err - } else { - saveAlerts.Alerts = alerts - } - - if err := bus.Dispatch(&saveAlerts); err != nil { + alerts, err := extractor.GetAlerts() + if err != nil { return err } - return nil + saveAlerts.Alerts = alerts + + return bus.Dispatch(&saveAlerts) } diff --git a/pkg/services/alerting/conditions/evaluator.go b/pkg/services/alerting/conditions/evaluator.go index 1b8fb952f65..eef593d39e2 100644 --- a/pkg/services/alerting/conditions/evaluator.go +++ b/pkg/services/alerting/conditions/evaluator.go @@ -2,6 +2,7 @@ package conditions import ( "encoding/json" + "fmt" "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" @@ -9,8 +10,8 @@ import ( ) var ( - defaultTypes []string = []string{"gt", "lt"} - rangedTypes []string = []string{"within_range", "outside_range"} + defaultTypes = []string{"gt", "lt"} + rangedTypes = []string{"within_range", "outside_range"} ) type AlertEvaluator interface { @@ -20,7 +21,7 @@ type AlertEvaluator interface { type NoValueEvaluator struct{} func (e *NoValueEvaluator) Eval(reducedValue null.Float) bool { - return reducedValue.Valid == false + return !reducedValue.Valid } type ThresholdEvaluator struct { @@ -31,12 +32,12 @@ type ThresholdEvaluator struct { func newThresholdEvaluator(typ string, model *simplejson.Json) (*ThresholdEvaluator, error) { params := model.Get("params").MustArray() if len(params) == 0 { - return nil, alerting.ValidationError{Reason: "Evaluator missing threshold parameter"} + return nil, fmt.Errorf("Evaluator missing threshold parameter") } firstParam, ok := params[0].(json.Number) if !ok { - return nil, alerting.ValidationError{Reason: "Evaluator has invalid parameter"} + return nil, fmt.Errorf("Evaluator has invalid parameter") } defaultEval := &ThresholdEvaluator{Type: typ} @@ -45,7 +46,7 @@ func newThresholdEvaluator(typ string, model *simplejson.Json) (*ThresholdEvalua } func (e *ThresholdEvaluator) Eval(reducedValue null.Float) bool { - if reducedValue.Valid == false { + if !reducedValue.Valid { return false } @@ -88,7 +89,7 @@ func newRangedEvaluator(typ string, model *simplejson.Json) (*RangedEvaluator, e } func (e *RangedEvaluator) Eval(reducedValue null.Float) bool { - if reducedValue.Valid == false { + if !reducedValue.Valid { return false } @@ -107,7 +108,7 @@ func (e *RangedEvaluator) Eval(reducedValue null.Float) bool { func NewAlertEvaluator(model *simplejson.Json) (AlertEvaluator, error) { typ := model.Get("type").MustString() if typ == "" { - return nil, alerting.ValidationError{Reason: "Evaluator missing type property"} + return nil, fmt.Errorf("Evaluator missing type property") } if inSlice(typ, defaultTypes) { @@ -122,7 +123,7 @@ func NewAlertEvaluator(model *simplejson.Json) (AlertEvaluator, error) { return &NoValueEvaluator{}, nil } - return nil, alerting.ValidationError{Reason: "Evaluator invalid evaluator type: " + typ} + return nil, fmt.Errorf("Evaluator invalid evaluator type: %s", typ) } func inSlice(a string, list []string) bool { diff --git a/pkg/services/alerting/conditions/query.go b/pkg/services/alerting/conditions/query.go index d499c5e8532..7d1a276c42e 100644 --- a/pkg/services/alerting/conditions/query.go +++ b/pkg/services/alerting/conditions/query.go @@ -53,7 +53,7 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio reducedValue := c.Reducer.Reduce(series) evalMatch := c.Evaluator.Eval(reducedValue) - if reducedValue.Valid == false { + if !reducedValue.Valid { emptySerieCount++ } diff --git a/pkg/services/alerting/conditions/reducer.go b/pkg/services/alerting/conditions/reducer.go index 0a61c13fa12..1e8ae792746 100644 --- a/pkg/services/alerting/conditions/reducer.go +++ b/pkg/services/alerting/conditions/reducer.go @@ -108,9 +108,9 @@ func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) null.Float { break } } - // get other points + // get the oldest point points = points[0:i] - for i := len(points) - 1; i >= 0; i-- { + for i := 0; i < len(points); i++ { if points[i][0].Valid { allNull = false value = first - points[i][0].Float64 @@ -131,9 +131,9 @@ func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) null.Float { break } } - // get other points + // get the oldest point points = points[0:i] - for i := len(points) - 1; i >= 0; i-- { + for i := 0; i < len(points); i++ { if points[i][0].Valid { allNull = false val := (first - points[i][0].Float64) / points[i][0].Float64 * 100 diff --git a/pkg/services/alerting/conditions/reducer_test.go b/pkg/services/alerting/conditions/reducer_test.go index 866b574f59f..7f11fc498bd 100644 --- a/pkg/services/alerting/conditions/reducer_test.go +++ b/pkg/services/alerting/conditions/reducer_test.go @@ -52,6 +52,24 @@ func TestSimpleReducer(t *testing.T) { So(result, ShouldEqual, float64(1)) }) + Convey("median should ignore null values", func() { + reducer := NewSimpleReducer("median") + series := &tsdb.TimeSeries{ + Name: "test time serie", + } + + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), 1)) + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), 2)) + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), 3)) + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(float64(1)), 4)) + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(float64(2)), 5)) + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(float64(3)), 6)) + + result := reducer.Reduce(series) + So(result.Valid, ShouldEqual, true) + So(result.Float64, ShouldEqual, float64(2)) + }) + Convey("avg", func() { result := testReducer("avg", 1, 2, 3) So(result, ShouldEqual, float64(2)) @@ -110,16 +128,35 @@ func TestSimpleReducer(t *testing.T) { So(reducer.Reduce(series).Float64, ShouldEqual, float64(3)) }) - Convey("diff", func() { + Convey("diff one point", func() { + result := testReducer("diff", 30) + So(result, ShouldEqual, float64(0)) + }) + + Convey("diff two points", func() { result := testReducer("diff", 30, 40) So(result, ShouldEqual, float64(10)) }) - Convey("percent_diff", func() { + Convey("diff three points", func() { + result := testReducer("diff", 30, 40, 40) + So(result, ShouldEqual, float64(10)) + }) + + Convey("percent_diff one point", func() { + result := testReducer("percent_diff", 40) + So(result, ShouldEqual, float64(0)) + }) + + Convey("percent_diff two points", func() { result := testReducer("percent_diff", 30, 40) So(result, ShouldEqual, float64(33.33333333333333)) }) + Convey("percent_diff three points", func() { + result := testReducer("percent_diff", 30, 40, 40) + So(result, ShouldEqual, float64(33.33333333333333)) + }) }) } diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 4448a5cb978..0f8e24bcef5 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -11,12 +11,17 @@ import ( "github.com/benbjohnson/clock" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/rendering" + "github.com/grafana/grafana/pkg/setting" "golang.org/x/sync/errgroup" ) -type Engine struct { - execQueue chan *Job - clock clock.Clock +type AlertingService struct { + RenderService rendering.Service `inject:""` + + execQueue chan *Job + //clock clock.Clock ticker *Ticker scheduler Scheduler evalHandler EvalHandler @@ -25,35 +30,41 @@ type Engine struct { resultHandler ResultHandler } -func NewEngine() *Engine { - e := &Engine{ - ticker: NewTicker(time.Now(), time.Second*0, clock.New()), - execQueue: make(chan *Job, 1000), - scheduler: NewScheduler(), - evalHandler: NewEvalHandler(), - ruleReader: NewRuleReader(), - log: log.New("alerting.engine"), - resultHandler: NewResultHandler(), - } +func init() { + registry.RegisterService(&AlertingService{}) +} +func NewEngine() *AlertingService { + e := &AlertingService{} + e.Init() return e } -func (e *Engine) Run(ctx context.Context) error { - e.log.Info("Initializing Alerting") +func (e *AlertingService) IsDisabled() bool { + return !setting.AlertingEnabled || !setting.ExecuteAlerts +} +func (e *AlertingService) Init() error { + e.ticker = NewTicker(time.Now(), time.Second*0, clock.New()) + e.execQueue = make(chan *Job, 1000) + e.scheduler = NewScheduler() + e.evalHandler = NewEvalHandler() + e.ruleReader = NewRuleReader() + e.log = log.New("alerting.engine") + e.resultHandler = NewResultHandler(e.RenderService) + return nil +} + +func (e *AlertingService) Run(ctx context.Context) error { alertGroup, ctx := errgroup.WithContext(ctx) - alertGroup.Go(func() error { return e.alertingTicker(ctx) }) alertGroup.Go(func() error { return e.runJobDispatcher(ctx) }) err := alertGroup.Wait() - - e.log.Info("Stopped Alerting", "reason", err) return err } -func (e *Engine) alertingTicker(grafanaCtx context.Context) error { +func (e *AlertingService) alertingTicker(grafanaCtx context.Context) error { defer func() { if err := recover(); err != nil { e.log.Error("Scheduler Panic: stopping alertingTicker", "error", err, "stack", log.Stack(1)) @@ -78,7 +89,7 @@ func (e *Engine) alertingTicker(grafanaCtx context.Context) error { } } -func (e *Engine) runJobDispatcher(grafanaCtx context.Context) error { +func (e *AlertingService) runJobDispatcher(grafanaCtx context.Context) error { dispatcherGroup, alertCtx := errgroup.WithContext(grafanaCtx) for { @@ -86,17 +97,63 @@ func (e *Engine) runJobDispatcher(grafanaCtx context.Context) error { case <-grafanaCtx.Done(): return dispatcherGroup.Wait() case job := <-e.execQueue: - dispatcherGroup.Go(func() error { return e.processJob(alertCtx, job) }) + dispatcherGroup.Go(func() error { return e.processJobWithRetry(alertCtx, job) }) } } } var ( - unfinishedWorkTimeout time.Duration = time.Second * 5 - alertTimeout time.Duration = time.Second * 30 + unfinishedWorkTimeout = time.Second * 5 + // TODO: Make alertTimeout and alertMaxAttempts configurable in the config file. + alertTimeout = time.Second * 30 + alertMaxAttempts = 3 ) -func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error { +func (e *AlertingService) processJobWithRetry(grafanaCtx context.Context, job *Job) error { + defer func() { + if err := recover(); err != nil { + e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1)) + } + }() + + cancelChan := make(chan context.CancelFunc, alertMaxAttempts) + attemptChan := make(chan int, 1) + + // Initialize with first attemptID=1 + attemptChan <- 1 + job.Running = true + + for { + select { + case <-grafanaCtx.Done(): + // In case grafana server context is cancel, let a chance to job processing + // to finish gracefully - by waiting a timeout duration - before forcing its end. + unfinishedWorkTimer := time.NewTimer(unfinishedWorkTimeout) + select { + case <-unfinishedWorkTimer.C: + return e.endJob(grafanaCtx.Err(), cancelChan, job) + case <-attemptChan: + return e.endJob(nil, cancelChan, job) + } + case attemptID, more := <-attemptChan: + if !more { + return e.endJob(nil, cancelChan, job) + } + go e.processJob(attemptID, attemptChan, cancelChan, job) + } + } +} + +func (e *AlertingService) endJob(err error, cancelChan chan context.CancelFunc, job *Job) error { + job.Running = false + close(cancelChan) + for cancelFn := range cancelChan { + cancelFn() + } + return err +} + +func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancelChan chan context.CancelFunc, job *Job) { defer func() { if err := recover(); err != nil { e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1)) @@ -104,14 +161,13 @@ func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error { }() alertCtx, cancelFn := context.WithTimeout(context.Background(), alertTimeout) + cancelChan <- cancelFn span := opentracing.StartSpan("alert execution") alertCtx = opentracing.ContextWithSpan(alertCtx, span) - job.Running = true evalContext := NewEvalContext(alertCtx, job.Rule) evalContext.Ctx = alertCtx - done := make(chan struct{}) go func() { defer func() { if err := recover(); err != nil { @@ -122,43 +178,36 @@ func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error { tlog.String("message", "failed to execute alert rule. panic was recovered."), ) span.Finish() - close(done) + close(attemptChan) } }() e.evalHandler.Eval(evalContext) - e.resultHandler.Handle(evalContext) span.SetTag("alertId", evalContext.Rule.Id) span.SetTag("dashboardId", evalContext.Rule.DashboardId) span.SetTag("firing", evalContext.Firing) span.SetTag("nodatapoints", evalContext.NoDataFound) + span.SetTag("attemptID", attemptID) + if evalContext.Error != nil { ext.Error.Set(span, true) span.LogFields( tlog.Error(evalContext.Error), - tlog.String("message", "alerting execution failed"), + tlog.String("message", "alerting execution attempt failed"), ) + if attemptID < alertMaxAttempts { + span.Finish() + e.log.Debug("Job Execution attempt triggered retry", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) + attemptChan <- (attemptID + 1) + return + } } + evalContext.Rule.State = evalContext.GetNewState() + e.resultHandler.Handle(evalContext) span.Finish() - close(done) + e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) + close(attemptChan) }() - - var err error = nil - select { - case <-grafanaCtx.Done(): - select { - case <-time.After(unfinishedWorkTimeout): - cancelFn() - err = grafanaCtx.Err() - case <-done: - } - case <-done: - } - - e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing) - job.Running = false - cancelFn() - return err } diff --git a/pkg/services/alerting/engine_test.go b/pkg/services/alerting/engine_test.go new file mode 100644 index 00000000000..63108bbb9aa --- /dev/null +++ b/pkg/services/alerting/engine_test.go @@ -0,0 +1,118 @@ +package alerting + +import ( + "context" + "errors" + "math" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +type FakeEvalHandler struct { + SuccessCallID int // 0 means never success + CallNb int +} + +func NewFakeEvalHandler(successCallID int) *FakeEvalHandler { + return &FakeEvalHandler{ + SuccessCallID: successCallID, + CallNb: 0, + } +} + +func (handler *FakeEvalHandler) Eval(evalContext *EvalContext) { + handler.CallNb++ + if handler.CallNb != handler.SuccessCallID { + evalContext.Error = errors.New("Fake evaluation failure") + } +} + +type FakeResultHandler struct{} + +func (handler *FakeResultHandler) Handle(evalContext *EvalContext) error { + return nil +} + +func TestEngineProcessJob(t *testing.T) { + Convey("Alerting engine job processing", t, func() { + engine := NewEngine() + engine.resultHandler = &FakeResultHandler{} + job := &Job{Running: true, Rule: &Rule{}} + + Convey("Should trigger retry if needed", func() { + + Convey("error + not last attempt -> retry", func() { + engine.evalHandler = NewFakeEvalHandler(0) + + for i := 1; i < alertMaxAttempts; i++ { + attemptChan := make(chan int, 1) + cancelChan := make(chan context.CancelFunc, alertMaxAttempts) + + engine.processJob(i, attemptChan, cancelChan, job) + nextAttemptID, more := <-attemptChan + + So(nextAttemptID, ShouldEqual, i+1) + So(more, ShouldEqual, true) + So(<-cancelChan, ShouldNotBeNil) + } + }) + + Convey("error + last attempt -> no retry", func() { + engine.evalHandler = NewFakeEvalHandler(0) + attemptChan := make(chan int, 1) + cancelChan := make(chan context.CancelFunc, alertMaxAttempts) + + engine.processJob(alertMaxAttempts, attemptChan, cancelChan, job) + nextAttemptID, more := <-attemptChan + + So(nextAttemptID, ShouldEqual, 0) + So(more, ShouldEqual, false) + So(<-cancelChan, ShouldNotBeNil) + }) + + Convey("no error -> no retry", func() { + engine.evalHandler = NewFakeEvalHandler(1) + attemptChan := make(chan int, 1) + cancelChan := make(chan context.CancelFunc, alertMaxAttempts) + + engine.processJob(1, attemptChan, cancelChan, job) + nextAttemptID, more := <-attemptChan + + So(nextAttemptID, ShouldEqual, 0) + So(more, ShouldEqual, false) + So(<-cancelChan, ShouldNotBeNil) + }) + }) + + Convey("Should trigger as many retries as needed", func() { + + Convey("never success -> max retries number", func() { + expectedAttempts := alertMaxAttempts + evalHandler := NewFakeEvalHandler(0) + engine.evalHandler = evalHandler + + engine.processJobWithRetry(context.TODO(), job) + So(evalHandler.CallNb, ShouldEqual, expectedAttempts) + }) + + Convey("always success -> never retry", func() { + expectedAttempts := 1 + evalHandler := NewFakeEvalHandler(1) + engine.evalHandler = evalHandler + + engine.processJobWithRetry(context.TODO(), job) + So(evalHandler.CallNb, ShouldEqual, expectedAttempts) + }) + + Convey("some errors before success -> some retries", func() { + expectedAttempts := int(math.Ceil(float64(alertMaxAttempts) / 2)) + evalHandler := NewFakeEvalHandler(expectedAttempts) + engine.evalHandler = evalHandler + + engine.processJobWithRetry(context.TODO(), job) + So(evalHandler.CallNb, ShouldEqual, expectedAttempts) + }) + }) + }) +} diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index d598203d675..d0441d379b7 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -106,9 +106,40 @@ func (c *EvalContext) GetRuleUrl() (string, error) { return setting.AppUrl, nil } - if ref, err := c.GetDashboardUID(); err != nil { + ref, err := c.GetDashboardUID() + if err != nil { return "", err - } else { - return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil } + return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil +} + +func (c *EvalContext) GetNewState() m.AlertStateType { + if c.Error != nil { + c.log.Error("Alert Rule Result Error", + "ruleId", c.Rule.Id, + "name", c.Rule.Name, + "error", c.Error, + "changing state to", c.Rule.ExecutionErrorState.ToAlertState()) + + if c.Rule.ExecutionErrorState == m.ExecutionErrorKeepState { + return c.PrevAlertState + } + return c.Rule.ExecutionErrorState.ToAlertState() + + } else if c.Firing { + return m.AlertStateAlerting + + } else if c.NoDataFound { + c.log.Info("Alert Rule returned no data", + "ruleId", c.Rule.Id, + "name", c.Rule.Name, + "changing state to", c.Rule.NoDataState.ToAlertState()) + + if c.Rule.NoDataState == m.NoDataKeepState { + return c.PrevAlertState + } + return c.Rule.NoDataState.ToAlertState() + } + + return m.AlertStateOK } diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go index 019ca1ed01f..750fa959683 100644 --- a/pkg/services/alerting/eval_context_test.go +++ b/pkg/services/alerting/eval_context_test.go @@ -2,31 +2,100 @@ package alerting import ( "context" + "fmt" "testing" "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" ) +func TestStateIsUpdatedWhenNeeded(t *testing.T) { + ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) + + t.Run("ok -> alerting", func(t *testing.T) { + ctx.PrevAlertState = models.AlertStateOK + ctx.Rule.State = models.AlertStateAlerting + + if !ctx.ShouldUpdateAlertState() { + t.Fatalf("expected should updated to be true") + } + }) + + t.Run("ok -> ok", func(t *testing.T) { + ctx.PrevAlertState = models.AlertStateOK + ctx.Rule.State = models.AlertStateOK + + if ctx.ShouldUpdateAlertState() { + t.Fatalf("expected should updated to be false") + } + }) +} + func TestAlertingEvalContext(t *testing.T) { - Convey("Eval context", t, func() { + Convey("Should compute and replace properly new rule state", t, func() { ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) + dummieError := fmt.Errorf("dummie error") - Convey("Should update alert state", func() { + Convey("ok -> alerting", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Firing = true - Convey("ok -> alerting", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.State = models.AlertStateAlerting + ctx.Rule.State = ctx.GetNewState() + So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) + }) - So(ctx.ShouldUpdateAlertState(), ShouldBeTrue) - }) + Convey("ok -> error(alerting)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Error = dummieError + ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting - Convey("ok -> ok", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.State = models.AlertStateOK + ctx.Rule.State = ctx.GetNewState() + So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) + }) - So(ctx.ShouldUpdateAlertState(), ShouldBeFalse) - }) + Convey("ok -> error(keep_last)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Error = dummieError + ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState + + ctx.Rule.State = ctx.GetNewState() + So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) + }) + + Convey("pending -> error(keep_last)", func() { + ctx.PrevAlertState = models.AlertStatePending + ctx.Error = dummieError + ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState + + ctx.Rule.State = ctx.GetNewState() + So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) + }) + + Convey("ok -> no_data(alerting)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Rule.NoDataState = models.NoDataSetAlerting + ctx.NoDataFound = true + + ctx.Rule.State = ctx.GetNewState() + So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) + }) + + Convey("ok -> no_data(keep_last)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Rule.NoDataState = models.NoDataKeepState + ctx.NoDataFound = true + + ctx.Rule.State = ctx.GetNewState() + So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) + }) + + Convey("pending -> no_data(keep_last)", func() { + ctx.PrevAlertState = models.AlertStatePending + ctx.Rule.NoDataState = models.NoDataKeepState + ctx.NoDataFound = true + + ctx.Rule.State = ctx.GetNewState() + So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) }) }) } diff --git a/pkg/services/alerting/eval_handler.go b/pkg/services/alerting/eval_handler.go index 457e02000fa..aa24efa77cd 100644 --- a/pkg/services/alerting/eval_handler.go +++ b/pkg/services/alerting/eval_handler.go @@ -7,7 +7,6 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" - "github.com/grafana/grafana/pkg/models" ) type DefaultEvalHandler struct { @@ -66,40 +65,7 @@ func (e *DefaultEvalHandler) Eval(context *EvalContext) { context.Firing = firing context.NoDataFound = noDataFound context.EndTime = time.Now() - context.Rule.State = e.getNewState(context) elapsedTime := context.EndTime.Sub(context.StartTime).Nanoseconds() / int64(time.Millisecond) metrics.M_Alerting_Execution_Time.Observe(float64(elapsedTime)) } - -// This should be move into evalContext once its been refactored. (Carl Bergquist) -func (handler *DefaultEvalHandler) getNewState(evalContext *EvalContext) models.AlertStateType { - if evalContext.Error != nil { - handler.log.Error("Alert Rule Result Error", - "ruleId", evalContext.Rule.Id, - "name", evalContext.Rule.Name, - "error", evalContext.Error, - "changing state to", evalContext.Rule.ExecutionErrorState.ToAlertState()) - - if evalContext.Rule.ExecutionErrorState == models.ExecutionErrorKeepState { - return evalContext.PrevAlertState - } else { - return evalContext.Rule.ExecutionErrorState.ToAlertState() - } - } else if evalContext.Firing { - return models.AlertStateAlerting - } else if evalContext.NoDataFound { - handler.log.Info("Alert Rule returned no data", - "ruleId", evalContext.Rule.Id, - "name", evalContext.Rule.Name, - "changing state to", evalContext.Rule.NoDataState.ToAlertState()) - - if evalContext.Rule.NoDataState == models.NoDataKeepState { - return evalContext.PrevAlertState - } else { - return evalContext.Rule.NoDataState.ToAlertState() - } - } - - return models.AlertStateOK -} diff --git a/pkg/services/alerting/eval_handler_test.go b/pkg/services/alerting/eval_handler_test.go index c942e24818f..a7c1f1e67fa 100644 --- a/pkg/services/alerting/eval_handler_test.go +++ b/pkg/services/alerting/eval_handler_test.go @@ -2,10 +2,8 @@ package alerting import ( "context" - "fmt" "testing" - "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" ) @@ -203,73 +201,5 @@ func TestAlertingEvaluationHandler(t *testing.T) { handler.Eval(context) So(context.NoDataFound, ShouldBeTrue) }) - - Convey("EvalHandler can replace alert state based for errors and no_data", func() { - ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) - dummieError := fmt.Errorf("dummie error") - Convey("Should update alert state", func() { - - Convey("ok -> alerting", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Firing = true - - So(handler.getNewState(ctx), ShouldEqual, models.AlertStateAlerting) - }) - - Convey("ok -> error(alerting)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting - - ctx.Rule.State = handler.getNewState(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) - }) - - Convey("ok -> error(keep_last)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState - - ctx.Rule.State = handler.getNewState(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) - }) - - Convey("pending -> error(keep_last)", func() { - ctx.PrevAlertState = models.AlertStatePending - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState - - ctx.Rule.State = handler.getNewState(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) - }) - - Convey("ok -> no_data(alerting)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.NoDataState = models.NoDataSetAlerting - ctx.NoDataFound = true - - ctx.Rule.State = handler.getNewState(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) - }) - - Convey("ok -> no_data(keep_last)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.NoDataState = models.NoDataKeepState - ctx.NoDataFound = true - - ctx.Rule.State = handler.getNewState(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) - }) - - Convey("pending -> no_data(keep_last)", func() { - ctx.PrevAlertState = models.AlertStatePending - ctx.Rule.NoDataState = models.NoDataKeepState - ctx.NoDataFound = true - - ctx.Rule.State = handler.getNewState(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) - }) - }) - }) }) } diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go index 2ae26c1a382..0abacc91313 100644 --- a/pkg/services/alerting/extractor.go +++ b/pkg/services/alerting/extractor.go @@ -11,81 +11,85 @@ import ( m "github.com/grafana/grafana/pkg/models" ) +// DashAlertExtractor extracts alerts from the dashboard json type DashAlertExtractor struct { + User *m.SignedInUser Dash *m.Dashboard - OrgId int64 + OrgID int64 log log.Logger } -func NewDashAlertExtractor(dash *m.Dashboard, orgId int64) *DashAlertExtractor { +// NewDashAlertExtractor returns a new DashAlertExtractor +func NewDashAlertExtractor(dash *m.Dashboard, orgID int64, user *m.SignedInUser) *DashAlertExtractor { return &DashAlertExtractor{ + User: user, Dash: dash, - OrgId: orgId, + OrgID: orgID, log: log.New("alerting.extractor"), } } -func (e *DashAlertExtractor) lookupDatasourceId(dsName string) (*m.DataSource, error) { +func (e *DashAlertExtractor) lookupDatasourceID(dsName string) (*m.DataSource, error) { if dsName == "" { - query := &m.GetDataSourcesQuery{OrgId: e.OrgId} + query := &m.GetDataSourcesQuery{OrgId: e.OrgID} if err := bus.Dispatch(query); err != nil { return nil, err - } else { - for _, ds := range query.Result { - if ds.IsDefault { - return ds, nil - } + } + + for _, ds := range query.Result { + if ds.IsDefault { + return ds, nil } } } else { - query := &m.GetDataSourceByNameQuery{Name: dsName, OrgId: e.OrgId} + query := &m.GetDataSourceByNameQuery{Name: dsName, OrgId: e.OrgID} if err := bus.Dispatch(query); err != nil { return nil, err - } else { - return query.Result, nil } + + return query.Result, nil } return nil, errors.New("Could not find datasource id for " + dsName) } -func findPanelQueryByRefId(panel *simplejson.Json, refId string) *simplejson.Json { +func findPanelQueryByRefID(panel *simplejson.Json, refID string) *simplejson.Json { for _, targetsObj := range panel.Get("targets").MustArray() { target := simplejson.NewFromAny(targetsObj) - if target.Get("refId").MustString() == refId { + if target.Get("refId").MustString() == refID { return target } } return nil } -func copyJson(in *simplejson.Json) (*simplejson.Json, error) { - rawJson, err := in.MarshalJSON() +func copyJSON(in *simplejson.Json) (*simplejson.Json, error) { + rawJSON, err := in.MarshalJSON() if err != nil { return nil, err } - return simplejson.NewJson(rawJson) + return simplejson.NewJson(rawJSON) } -func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json) ([]*m.Alert, error) { +func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, validateAlertFunc func(*m.Alert) bool) ([]*m.Alert, error) { alerts := make([]*m.Alert, 0) for _, panelObj := range jsonWithPanels.Get("panels").MustArray() { panel := simplejson.NewFromAny(panelObj) - collapsedJson, collapsed := panel.CheckGet("collapsed") + collapsedJSON, collapsed := panel.CheckGet("collapsed") // check if the panel is collapsed - if collapsed && collapsedJson.MustBool() { + if collapsed && collapsedJSON.MustBool() { // extract alerts from sub panels for collapsed panels - als, err := e.GetAlertFromPanels(panel) + alertSlice, err := e.getAlertFromPanels(panel, validateAlertFunc) if err != nil { return nil, err } - alerts = append(alerts, als...) + alerts = append(alerts, alertSlice...) continue } @@ -95,14 +99,14 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json) continue } - panelId, err := panel.Get("id").Int64() + panelID, err := panel.Get("id").Int64() if err != nil { - return nil, fmt.Errorf("panel id is required. err %v", err) + return nil, ValidationError{Reason: "A numeric panel id property is missing"} } // backward compatibility check, can be removed later enabled, hasEnabled := jsonAlert.CheckGet("enabled") - if hasEnabled && enabled.MustBool() == false { + if hasEnabled && !enabled.MustBool() { continue } @@ -113,8 +117,8 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json) alert := &m.Alert{ DashboardId: e.Dash.Id, - OrgId: e.OrgId, - PanelId: panelId, + OrgId: e.OrgID, + PanelId: panelID, Id: jsonAlert.Get("id").MustInt64(), Name: jsonAlert.Get("name").MustString(), Handler: jsonAlert.Get("handler").MustInt64(), @@ -126,11 +130,11 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json) jsonCondition := simplejson.NewFromAny(condition) jsonQuery := jsonCondition.Get("query") - queryRefId := jsonQuery.Get("params").MustArray()[0].(string) - panelQuery := findPanelQueryByRefId(panel, queryRefId) + queryRefID := jsonQuery.Get("params").MustArray()[0].(string) + panelQuery := findPanelQueryByRefID(panel, queryRefID) if panelQuery == nil { - reason := fmt.Sprintf("Alert on PanelId: %v refers to query(%s) that cannot be found", alert.PanelId, queryRefId) + reason := fmt.Sprintf("Alert on PanelId: %v refers to query(%s) that cannot be found", alert.PanelId, queryRefID) return nil, ValidationError{Reason: reason} } @@ -141,12 +145,29 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json) dsName = panel.Get("datasource").MustString() } - if datasource, err := e.lookupDatasourceId(dsName); err != nil { - return nil, err - } else { - jsonQuery.SetPath([]string{"datasourceId"}, datasource.Id) + datasource, err := e.lookupDatasourceID(dsName) + if err != nil { + e.log.Debug("Error looking up datasource", "error", err) + return nil, ValidationError{Reason: fmt.Sprintf("Data source used by alert rule not found, alertName=%v, datasource=%s", alert.Name, dsName)} } + dsFilterQuery := m.DatasourcesPermissionFilterQuery{ + User: e.User, + Datasources: []*m.DataSource{datasource}, + } + + if err := bus.Dispatch(&dsFilterQuery); err != nil { + if err != bus.ErrHandlerNotFound { + return nil, err + } + } else { + if len(dsFilterQuery.Result) == 0 { + return nil, m.ErrDataSourceAccessDenied + } + } + + jsonQuery.SetPath([]string{"datasourceId"}, datasource.Id) + if interval, err := panel.Get("interval").String(); err == nil { panelQuery.Set("interval", interval) } @@ -162,21 +183,27 @@ func (e *DashAlertExtractor) GetAlertFromPanels(jsonWithPanels *simplejson.Json) return nil, err } - if alert.ValidToSave() { - alerts = append(alerts, alert) - } else { - e.log.Debug("Invalid Alert Data. Dashboard, Org or Panel ID is not correct", "alertName", alert.Name, "panelId", alert.PanelId) - return nil, m.ErrDashboardContainsInvalidAlertData + if !validateAlertFunc(alert) { + return nil, ValidationError{Reason: fmt.Sprintf("Panel id is not correct, alertName=%v, panelId=%v", alert.Name, alert.PanelId)} } + + alerts = append(alerts, alert) } return alerts, nil } -func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) { - e.log.Debug("GetAlerts") +func validateAlertRule(alert *m.Alert) bool { + return alert.ValidToSave() +} - dashboardJson, err := copyJson(e.Dash.Data) +// GetAlerts extracts alerts from the dashboard json and does full validation on the alert json data +func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) { + return e.extractAlerts(validateAlertRule) +} + +func (e *DashAlertExtractor) extractAlerts(validateFunc func(alert *m.Alert) bool) ([]*m.Alert, error) { + dashboardJSON, err := copyJSON(e.Dash.Data) if err != nil { return nil, err } @@ -185,11 +212,11 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) { // We extract alerts from rows to be backwards compatible // with the old dashboard json model. - rows := dashboardJson.Get("rows").MustArray() + rows := dashboardJSON.Get("rows").MustArray() if len(rows) > 0 { for _, rowObj := range rows { row := simplejson.NewFromAny(rowObj) - a, err := e.GetAlertFromPanels(row) + a, err := e.getAlertFromPanels(row, validateFunc) if err != nil { return nil, err } @@ -197,7 +224,7 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) { alerts = append(alerts, a...) } } else { - a, err := e.GetAlertFromPanels(dashboardJson) + a, err := e.getAlertFromPanels(dashboardJSON, validateFunc) if err != nil { return nil, err } @@ -208,3 +235,10 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) { e.log.Debug("Extracted alerts from dashboard", "alertCount", len(alerts)) return alerts, nil } + +// ValidateAlerts validates alerts in the dashboard json but does not require a valid dashboard id +// in the first validation pass +func (e *DashAlertExtractor) ValidateAlerts() error { + _, err := e.extractAlerts(func(alert *m.Alert) bool { return alert.OrgId != 0 && alert.PanelId != 0 }) + return err +} diff --git a/pkg/services/alerting/extractor_test.go b/pkg/services/alerting/extractor_test.go index 3bda6c771fb..0890b9e1bd1 100644 --- a/pkg/services/alerting/extractor_test.go +++ b/pkg/services/alerting/extractor_test.go @@ -50,7 +50,7 @@ func TestAlertRuleExtraction(t *testing.T) { So(err, ShouldBeNil) Convey("Extractor should not modify the original json", func() { - dashJson, err := simplejson.NewJson([]byte(json)) + dashJson, err := simplejson.NewJson(json) So(err, ShouldBeNil) dash := m.NewDashboardFromJson(dashJson) @@ -69,7 +69,7 @@ func TestAlertRuleExtraction(t *testing.T) { So(getTarget(dashJson), ShouldEqual, "") }) - extractor := NewDashAlertExtractor(dash, 1) + extractor := NewDashAlertExtractor(dash, 1, nil) _, _ = extractor.GetAlerts() Convey("Dashboard json should not be updated after extracting rules", func() { @@ -79,11 +79,11 @@ func TestAlertRuleExtraction(t *testing.T) { Convey("Parsing and validating dashboard containing graphite alerts", func() { - dashJson, err := simplejson.NewJson([]byte(json)) + dashJson, err := simplejson.NewJson(json) So(err, ShouldBeNil) dash := m.NewDashboardFromJson(dashJson) - extractor := NewDashAlertExtractor(dash, 1) + extractor := NewDashAlertExtractor(dash, 1, nil) alerts, err := extractor.GetAlerts() @@ -143,10 +143,10 @@ func TestAlertRuleExtraction(t *testing.T) { panelWithoutId, err := ioutil.ReadFile("./test-data/panels-missing-id.json") So(err, ShouldBeNil) - dashJson, err := simplejson.NewJson([]byte(panelWithoutId)) + dashJson, err := simplejson.NewJson(panelWithoutId) So(err, ShouldBeNil) dash := m.NewDashboardFromJson(dashJson) - extractor := NewDashAlertExtractor(dash, 1) + extractor := NewDashAlertExtractor(dash, 1, nil) _, err = extractor.GetAlerts() @@ -159,10 +159,10 @@ func TestAlertRuleExtraction(t *testing.T) { panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json") So(err, ShouldBeNil) - dashJson, err := simplejson.NewJson([]byte(panelWithIdZero)) + dashJson, err := simplejson.NewJson(panelWithIdZero) So(err, ShouldBeNil) dash := m.NewDashboardFromJson(dashJson) - extractor := NewDashAlertExtractor(dash, 1) + extractor := NewDashAlertExtractor(dash, 1, nil) _, err = extractor.GetAlerts() @@ -178,7 +178,7 @@ func TestAlertRuleExtraction(t *testing.T) { dashJson, err := simplejson.NewJson(json) So(err, ShouldBeNil) dash := m.NewDashboardFromJson(dashJson) - extractor := NewDashAlertExtractor(dash, 1) + extractor := NewDashAlertExtractor(dash, 1, nil) alerts, err := extractor.GetAlerts() @@ -198,7 +198,7 @@ func TestAlertRuleExtraction(t *testing.T) { dashJson, err := simplejson.NewJson(json) So(err, ShouldBeNil) dash := m.NewDashboardFromJson(dashJson) - extractor := NewDashAlertExtractor(dash, 1) + extractor := NewDashAlertExtractor(dash, 1, nil) alerts, err := extractor.GetAlerts() @@ -228,7 +228,7 @@ func TestAlertRuleExtraction(t *testing.T) { So(err, ShouldBeNil) dash := m.NewDashboardFromJson(dashJson) - extractor := NewDashAlertExtractor(dash, 1) + extractor := NewDashAlertExtractor(dash, 1, nil) alerts, err := extractor.GetAlerts() @@ -240,5 +240,26 @@ func TestAlertRuleExtraction(t *testing.T) { So(len(alerts), ShouldEqual, 4) }) }) + + Convey("Parse and validate dashboard without id and containing an alert", func() { + json, err := ioutil.ReadFile("./test-data/dash-without-id.json") + So(err, ShouldBeNil) + + dashJSON, err := simplejson.NewJson(json) + So(err, ShouldBeNil) + dash := m.NewDashboardFromJson(dashJSON) + extractor := NewDashAlertExtractor(dash, 1, nil) + + err = extractor.ValidateAlerts() + + Convey("Should validate without error", func() { + So(err, ShouldBeNil) + }) + + Convey("Should fail on save", func() { + _, err := extractor.GetAlerts() + So(err.Error(), ShouldEqual, "Alert validation error: Panel id is not correct, alertName=Influxdb, panelId=1") + }) + }) }) } diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 18f969ba1b9..040d0991861 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -1,6 +1,11 @@ package alerting -import "time" +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/models" +) type EvalHandler interface { Eval(evalContext *EvalContext) @@ -15,17 +20,27 @@ type Notifier interface { Notify(evalContext *EvalContext) error GetType() string NeedsImage() bool - ShouldNotify(evalContext *EvalContext) bool + + // ShouldNotify checks this evaluation should send an alert notification + ShouldNotify(ctx context.Context, evalContext *EvalContext, notificationState *models.AlertNotificationState) bool GetNotifierId() int64 GetIsDefault() bool + GetSendReminder() bool + GetDisableResolveMessage() bool + GetFrequency() time.Duration } -type NotifierSlice []Notifier +type notifierState struct { + notifier Notifier + state *models.AlertNotificationState +} -func (notifiers NotifierSlice) ShouldUploadImage() bool { - for _, notifier := range notifiers { - if notifier.NeedsImage() { +type notifierStateSlice []*notifierState + +func (notifiers notifierStateSlice) ShouldUploadImage() bool { + for _, ns := range notifiers { + if ns.notifier.NeedsImage() { return true } } diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index af9ba52a52a..9ce50eadd6b 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -4,13 +4,12 @@ import ( "errors" "fmt" - "golang.org/x/sync/errgroup" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/imguploader" - "github.com/grafana/grafana/pkg/components/renderer" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" + "github.com/grafana/grafana/pkg/services/rendering" + "github.com/grafana/grafana/pkg/setting" m "github.com/grafana/grafana/pkg/models" ) @@ -27,50 +26,95 @@ type NotificationService interface { SendIfNeeded(context *EvalContext) error } -func NewNotificationService() NotificationService { - return newNotificationService() -} - -type notificationService struct { - log log.Logger -} - -func newNotificationService() *notificationService { +func NewNotificationService(renderService rendering.Service) NotificationService { return ¬ificationService{ - log: log.New("alerting.notifier"), + log: log.New("alerting.notifier"), + renderService: renderService, } } +type notificationService struct { + log log.Logger + renderService rendering.Service +} + func (n *notificationService) SendIfNeeded(context *EvalContext) error { - notifiers, err := n.getNeededNotifiers(context.Rule.OrgId, context.Rule.Notifications, context) + notifierStates, err := n.getNeededNotifiers(context.Rule.OrgId, context.Rule.Notifications, context) if err != nil { return err } - if len(notifiers) == 0 { + if len(notifierStates) == 0 { return nil } - if notifiers.ShouldUploadImage() { + if notifierStates.ShouldUploadImage() { if err = n.uploadImage(context); err != nil { n.log.Error("Failed to upload alert panel image.", "error", err) } } - return n.sendNotifications(context, notifiers) + return n.sendNotifications(context, notifierStates) } -func (n *notificationService) sendNotifications(context *EvalContext, notifiers []Notifier) error { - g, _ := errgroup.WithContext(context.Ctx) +func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, notifierState *notifierState) error { + notifier := notifierState.notifier - for _, notifier := range notifiers { - not := notifier //avoid updating scope variable in go routine - n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) - metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() - g.Go(func() error { return not.Notify(context) }) + n.log.Debug("Sending notification", "type", notifier.GetType(), "id", notifier.GetNotifierId(), "isDefault", notifier.GetIsDefault()) + metrics.M_Alerting_Notification_Sent.WithLabelValues(notifier.GetType()).Inc() + + err := notifier.Notify(evalContext) + + if err != nil { + n.log.Error("failed to send notification", "id", notifier.GetNotifierId(), "error", err) } - return g.Wait() + if evalContext.IsTestRun { + return nil + } + + cmd := &m.SetAlertNotificationStateToCompleteCommand{ + Id: notifierState.state.Id, + Version: notifierState.state.Version, + } + + return bus.DispatchCtx(evalContext.Ctx, cmd) +} + +func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *notifierState) error { + if !evalContext.IsTestRun { + setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ + Id: notifierState.state.Id, + Version: notifierState.state.Version, + AlertRuleStateUpdatedVersion: evalContext.Rule.StateChanges, + } + + err := bus.DispatchCtx(evalContext.Ctx, setPendingCmd) + if err == m.ErrAlertNotificationStateVersionConflict { + return nil + } + + if err != nil { + return err + } + + // We need to update state version to be able to log + // unexpected version conflicts when marking notifications as ok + notifierState.state.Version = setPendingCmd.ResultVersion + } + + return n.sendAndMarkAsComplete(evalContext, notifierState) +} + +func (n *notificationService) sendNotifications(evalContext *EvalContext, notifierStates notifierStateSlice) error { + for _, notifierState := range notifierStates { + err := n.sendNotification(evalContext, notifierState) + if err != nil { + n.log.Error("failed to send notification", "id", notifierState.notifier.GetNotifierId(), "error", err) + } + } + + return nil } func (n *notificationService) uploadImage(context *EvalContext) (err error) { @@ -79,50 +123,72 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) { return err } - renderOpts := &renderer.RenderOpts{ - Width: "800", - Height: "400", - Timeout: "30", - OrgId: context.Rule.OrgId, - IsAlertContext: true, + renderOpts := rendering.Opts{ + Width: 1000, + Height: 500, + Timeout: alertTimeout / 2, + OrgId: context.Rule.OrgId, + OrgRole: m.ROLE_ADMIN, + ConcurrentLimit: setting.AlertingRenderLimit, } - if ref, err := context.GetDashboardUID(); err != nil { + ref, err := context.GetDashboardUID() + if err != nil { return err - } else { - renderOpts.Path = fmt.Sprintf("d-solo/%s/%s?panelId=%d", ref.Uid, ref.Slug, context.Rule.PanelId) } - if imagePath, err := renderer.RenderToPng(renderOpts); err != nil { + renderOpts.Path = fmt.Sprintf("d-solo/%s/%s?panelId=%d", ref.Uid, ref.Slug, context.Rule.PanelId) + + result, err := n.renderService.Render(context.Ctx, renderOpts) + if err != nil { return err - } else { - context.ImageOnDiskPath = imagePath } + context.ImageOnDiskPath = result.FilePath context.ImagePublicUrl, err = uploader.Upload(context.Ctx, context.ImageOnDiskPath) if err != nil { return err } - n.log.Info("uploaded", "url", context.ImagePublicUrl) + if context.ImagePublicUrl != "" { + n.log.Info("uploaded screenshot of alert to external image store", "url", context.ImagePublicUrl) + } + return nil } -func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, context *EvalContext) (NotifierSlice, error) { +func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (notifierStateSlice, error) { query := &m.GetAlertNotificationsToSendQuery{OrgId: orgId, Ids: notificationIds} if err := bus.Dispatch(query); err != nil { return nil, err } - var result []Notifier + var result notifierStateSlice for _, notification := range query.Result { - if not, err := n.createNotifierFor(notification); err != nil { - return nil, err - } else { - if not.ShouldNotify(context) { - result = append(result, not) - } + not, err := n.createNotifierFor(notification) + if err != nil { + n.log.Error("Could not create notifier", "notifier", notification.Id, "error", err) + continue + } + + query := &m.GetOrCreateNotificationStateQuery{ + NotifierId: notification.Id, + AlertId: evalContext.Rule.Id, + OrgId: evalContext.Rule.OrgId, + } + + err = bus.DispatchCtx(evalContext.Ctx, query) + if err != nil { + n.log.Error("Could not get notification state.", "notifier", notification.Id, "error", err) + continue + } + + if not.ShouldNotify(evalContext.Ctx, evalContext, query.Result) { + result = append(result, ¬ifierState{ + notifier: not, + state: query.Result, + }) } } @@ -140,7 +206,7 @@ func (n *notificationService) createNotifierFor(model *m.AlertNotification) (Not type NotifierFactory func(notification *m.AlertNotification) (Notifier, error) -var notifierFactories map[string]*NotifierPlugin = make(map[string]*NotifierPlugin) +var notifierFactories = make(map[string]*NotifierPlugin) func RegisterNotifier(plugin *NotifierPlugin) { notifierFactories[plugin.Type] = plugin diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index d449167de13..2caa4d5ab58 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -1,6 +1,7 @@ package notifiers import ( + "context" "time" "github.com/grafana/grafana/pkg/bus" @@ -33,7 +34,7 @@ func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, err } return &AlertmanagerNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, log: log.New("alerting.notifier.prometheus-alertmanager"), }, nil @@ -45,7 +46,7 @@ type AlertmanagerNotifier struct { log log.Logger } -func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool { +func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext, notificationState *m.AlertNotificationState) bool { this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState) // Do not notify when we become OK for the first time. diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 7a3cc71c4db..d141d6cd257 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -1,45 +1,95 @@ package notifiers import ( - "github.com/grafana/grafana/pkg/components/simplejson" - m "github.com/grafana/grafana/pkg/models" + "context" + "time" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" ) +const ( + triggMetrString = "Triggered metrics:\n\n" +) + type NotifierBase struct { - Name string - Type string - Id int64 - IsDeault bool - UploadImage bool + Name string + Type string + Id int64 + IsDeault bool + UploadImage bool + SendReminder bool + DisableResolveMessage bool + Frequency time.Duration + + log log.Logger } -func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model *simplejson.Json) NotifierBase { - uploadImage := model.Get("uploadImage").MustBool(false) +func NewNotifierBase(model *models.AlertNotification) NotifierBase { + uploadImage := true + value, exist := model.Settings.CheckGet("uploadImage") + if exist { + uploadImage = value.MustBool() + } return NotifierBase{ - Id: id, - Name: name, - IsDeault: isDefault, - Type: notifierType, - UploadImage: uploadImage, + Id: model.Id, + Name: model.Name, + IsDeault: model.IsDefault, + Type: model.Type, + UploadImage: uploadImage, + SendReminder: model.SendReminder, + DisableResolveMessage: model.DisableResolveMessage, + Frequency: model.Frequency, + log: log.New("alerting.notifier." + model.Name), } } -func defaultShouldNotify(context *alerting.EvalContext) bool { +// ShouldNotify checks this evaluation should send an alert notification +func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalContext, notiferState *models.AlertNotificationState) bool { // Only notify on state change. - if context.PrevAlertState == context.Rule.State { + if context.PrevAlertState == context.Rule.State && !n.SendReminder { return false } - // Do not notify when we become OK for the first time. - if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) { - return false - } - return true -} -func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool { - return defaultShouldNotify(context) + if context.PrevAlertState == context.Rule.State && n.SendReminder { + // Do not notify if interval has not elapsed + lastNotify := time.Unix(notiferState.UpdatedAt, 0) + if notiferState.UpdatedAt != 0 && lastNotify.Add(n.Frequency).After(time.Now()) { + return false + } + + // Do not notify if alert state is OK or pending even on repeated notify + if context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending { + return false + } + } + + // Do not notify when we become OK for the first time. + if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK { + return false + } + + // Do not notify when we OK -> Pending + if context.PrevAlertState == models.AlertStateOK && context.Rule.State == models.AlertStatePending { + return false + } + + // Do not notifu if state pending and it have been updated last minute + if notiferState.State == models.AlertNotificationStatePending { + lastUpdated := time.Unix(notiferState.UpdatedAt, 0) + if lastUpdated.Add(1 * time.Minute).After(time.Now()) { + return false + } + } + + // Do not notify when state is OK if DisableResolveMessage is set to true + if context.Rule.State == models.AlertStateOK && n.DisableResolveMessage { + return false + } + + return true } func (n *NotifierBase) GetType() string { @@ -57,3 +107,15 @@ func (n *NotifierBase) GetNotifierId() int64 { func (n *NotifierBase) GetIsDefault() bool { return n.IsDeault } + +func (n *NotifierBase) GetSendReminder() bool { + return n.SendReminder +} + +func (n *NotifierBase) GetDisableResolveMessage() bool { + return n.DisableResolveMessage +} + +func (n *NotifierBase) GetFrequency() time.Duration { + return n.Frequency +} diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 4225e203a3d..5062828cb4f 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -3,30 +3,186 @@ package notifiers import ( "context" "testing" + "time" + "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" . "github.com/smartystreets/goconvey/convey" ) -func TestBaseNotifier(t *testing.T) { - Convey("Base notifier tests", t, func() { - Convey("should notify", func() { - Convey("pending -> ok", func() { - context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ - State: m.AlertStatePending, - }) - context.Rule.State = m.AlertStateOK - So(defaultShouldNotify(context), ShouldBeFalse) - }) +func TestShouldSendAlertNotification(t *testing.T) { + tnow := time.Now() - Convey("ok -> alerting", func() { - context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ - State: m.AlertStateOK, - }) - context.Rule.State = m.AlertStateAlerting - So(defaultShouldNotify(context), ShouldBeTrue) - }) + tcs := []struct { + name string + prevState m.AlertStateType + newState m.AlertStateType + sendReminder bool + frequency time.Duration + state *m.AlertNotificationState + + expect bool + }{ + { + name: "pending -> ok should not trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStatePending, + sendReminder: false, + state: &m.AlertNotificationState{}, + + expect: false, + }, + { + name: "ok -> alerting should trigger an notification", + newState: m.AlertStateAlerting, + prevState: m.AlertStateOK, + sendReminder: false, + state: &m.AlertNotificationState{}, + + expect: true, + }, + { + name: "ok -> pending should not trigger an notification", + newState: m.AlertStatePending, + prevState: m.AlertStateOK, + sendReminder: false, + state: &m.AlertNotificationState{}, + + expect: false, + }, + { + name: "ok -> ok should not trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStateOK, + sendReminder: false, + state: &m.AlertNotificationState{}, + + expect: false, + }, + { + name: "ok -> ok with reminder should not trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStateOK, + sendReminder: true, + state: &m.AlertNotificationState{}, + + expect: false, + }, + { + name: "alerting -> ok should trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStateAlerting, + sendReminder: false, + state: &m.AlertNotificationState{}, + + expect: true, + }, + { + name: "alerting -> ok should trigger an notification when reminders enabled", + newState: m.AlertStateOK, + prevState: m.AlertStateAlerting, + frequency: time.Minute * 10, + sendReminder: true, + state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()}, + + expect: true, + }, + { + name: "alerting -> alerting with reminder and no state should trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateAlerting, + frequency: time.Minute * 10, + sendReminder: true, + state: &m.AlertNotificationState{}, + + expect: true, + }, + { + name: "alerting -> alerting with reminder and last notification sent 1 minute ago should not trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateAlerting, + frequency: time.Minute * 10, + sendReminder: true, + state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()}, + + expect: false, + }, + { + name: "alerting -> alerting with reminder and last notifciation sent 11 minutes ago should trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateAlerting, + frequency: time.Minute * 10, + sendReminder: true, + state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-11 * time.Minute).Unix()}, + + expect: true, + }, + { + name: "OK -> alerting with notifciation state pending and updated 30 seconds ago should not trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateOK, + state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-30 * time.Second).Unix()}, + + expect: false, + }, + { + name: "OK -> alerting with notifciation state pending and updated 2 minutes ago should trigger", + newState: m.AlertStateAlerting, + prevState: m.AlertStateOK, + state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()}, + + expect: true, + }, + } + + for _, tc := range tcs { + evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ + State: tc.prevState, + }) + + evalContext.Rule.State = tc.newState + nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency} + + if nb.ShouldNotify(evalContext.Ctx, evalContext, tc.state) != tc.expect { + t.Errorf("failed test %s.\n expected \n%+v \nto return: %v", tc.name, tc, tc.expect) + } + } +} + +func TestBaseNotifier(t *testing.T) { + Convey("default constructor for notifiers", t, func() { + bJson := simplejson.New() + + model := &m.AlertNotification{ + Id: 1, + Name: "name", + Type: "email", + Settings: bJson, + } + + Convey("can parse false value", func() { + bJson.Set("uploadImage", false) + + base := NewNotifierBase(model) + So(base.UploadImage, ShouldBeFalse) + }) + + Convey("can parse true value", func() { + bJson.Set("uploadImage", true) + + base := NewNotifierBase(model) + So(base.UploadImage, ShouldBeTrue) + }) + + Convey("default value should be true for backwards compatibility", func() { + base := NewNotifierBase(model) + So(base.UploadImage, ShouldBeTrue) + }) + + Convey("default value should be false for backwards compatibility", func() { + base := NewNotifierBase(model) + So(base.DisableResolveMessage, ShouldBeFalse) }) }) } diff --git a/pkg/services/alerting/notifiers/dingding.go b/pkg/services/alerting/notifiers/dingding.go index e32b9d34f91..1ef085c82f1 100644 --- a/pkg/services/alerting/notifiers/dingding.go +++ b/pkg/services/alerting/notifiers/dingding.go @@ -32,7 +32,7 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &DingDingNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, log: log.New("alerting.notifier.dingding"), }, nil @@ -57,6 +57,9 @@ func (this *DingDingNotifier) Notify(evalContext *alerting.EvalContext) error { message := evalContext.Rule.Message picUrl := evalContext.ImagePublicUrl title := evalContext.GetNotificationTitle() + if message == "" { + message = title + } bodyJSON, err := simplejson.NewJson([]byte(`{ "msgtype": "link", @@ -72,7 +75,10 @@ func (this *DingDingNotifier) Notify(evalContext *alerting.EvalContext) error { this.log.Error("Failed to create Json data", "error", err, "dingding", this.Name) } - body, _ := bodyJSON.MarshalJSON() + body, err := bodyJSON.MarshalJSON() + if err != nil { + return err + } cmd := &m.SendWebhookSync{ Url: this.Url, diff --git a/pkg/services/alerting/notifiers/discord.go b/pkg/services/alerting/notifiers/discord.go new file mode 100644 index 00000000000..57d9d438fa2 --- /dev/null +++ b/pkg/services/alerting/notifiers/discord.go @@ -0,0 +1,173 @@ +package notifiers + +import ( + "bytes" + "io" + "mime/multipart" + "os" + "strconv" + "strings" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" + "github.com/grafana/grafana/pkg/setting" +) + +func init() { + alerting.RegisterNotifier(&alerting.NotifierPlugin{ + Type: "discord", + Name: "Discord", + Description: "Sends notifications to Discord", + Factory: NewDiscordNotifier, + OptionsTemplate: ` +

Discord settings

+
+ Webhook URL + +
+ `, + }) +} + +func NewDiscordNotifier(model *m.AlertNotification) (alerting.Notifier, error) { + url := model.Settings.Get("url").MustString() + if url == "" { + return nil, alerting.ValidationError{Reason: "Could not find webhook url property in settings"} + } + + return &DiscordNotifier{ + NotifierBase: NewNotifierBase(model), + WebhookURL: url, + log: log.New("alerting.notifier.discord"), + }, nil +} + +type DiscordNotifier struct { + NotifierBase + WebhookURL string + log log.Logger +} + +func (this *DiscordNotifier) Notify(evalContext *alerting.EvalContext) error { + this.log.Info("Sending alert notification to", "webhook_url", this.WebhookURL) + + ruleUrl, err := evalContext.GetRuleUrl() + if err != nil { + this.log.Error("Failed get rule link", "error", err) + return err + } + + bodyJSON := simplejson.New() + bodyJSON.Set("username", "Grafana") + + fields := make([]map[string]interface{}, 0) + + for _, evt := range evalContext.EvalMatches { + + fields = append(fields, map[string]interface{}{ + "name": evt.Metric, + "value": evt.Value.FullString(), + "inline": true, + }) + } + + footer := map[string]interface{}{ + "text": "Grafana v" + setting.BuildVersion, + "icon_url": "https://grafana.com/assets/img/fav32.png", + } + + color, _ := strconv.ParseInt(strings.TrimLeft(evalContext.GetStateModel().Color, "#"), 16, 0) + + embed := simplejson.New() + embed.Set("title", evalContext.GetNotificationTitle()) + //Discord takes integer for color + embed.Set("color", color) + embed.Set("url", ruleUrl) + embed.Set("description", evalContext.Rule.Message) + embed.Set("type", "rich") + embed.Set("fields", fields) + embed.Set("footer", footer) + + var image map[string]interface{} + var embeddedImage = false + + if evalContext.ImagePublicUrl != "" { + image = map[string]interface{}{ + "url": evalContext.ImagePublicUrl, + } + embed.Set("image", image) + } else { + image = map[string]interface{}{ + "url": "attachment://graph.png", + } + embed.Set("image", image) + embeddedImage = true + } + + bodyJSON.Set("embeds", []interface{}{embed}) + + json, _ := bodyJSON.MarshalJSON() + + content_type := "application/json" + + var body []byte + + if embeddedImage { + + var b bytes.Buffer + + w := multipart.NewWriter(&b) + + f, err := os.Open(evalContext.ImageOnDiskPath) + + if err != nil { + this.log.Error("Can't open graph file", err) + return err + } + + defer f.Close() + + fw, err := w.CreateFormField("payload_json") + if err != nil { + return err + } + + if _, err = fw.Write([]byte(string(json))); err != nil { + return err + } + + fw, err = w.CreateFormFile("file", "graph.png") + if err != nil { + return err + } + + if _, err = io.Copy(fw, f); err != nil { + return err + } + + w.Close() + + body = b.Bytes() + content_type = w.FormDataContentType() + + } else { + body = json + } + + cmd := &m.SendWebhookSync{ + Url: this.WebhookURL, + Body: string(body), + HttpMethod: "POST", + ContentType: content_type, + } + + if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { + this.log.Error("Failed to send notification to Discord", "error", err) + return err + } + + return nil +} diff --git a/pkg/services/alerting/notifiers/discord_test.go b/pkg/services/alerting/notifiers/discord_test.go new file mode 100644 index 00000000000..fe925aab362 --- /dev/null +++ b/pkg/services/alerting/notifiers/discord_test.go @@ -0,0 +1,52 @@ +package notifiers + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestDiscordNotifier(t *testing.T) { + Convey("Telegram notifier tests", t, func() { + + Convey("Parsing alert notification from settings", func() { + Convey("empty settings should return error", func() { + json := `{ }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "discord_testing", + Type: "discord", + Settings: settingsJSON, + } + + _, err := NewDiscordNotifier(model) + So(err, ShouldNotBeNil) + }) + + Convey("settings should trigger incident", func() { + json := ` + { + "url": "https://web.hook/" + }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "discord_testing", + Type: "discord", + Settings: settingsJSON, + } + + not, err := NewDiscordNotifier(model) + discordNotifier := not.(*DiscordNotifier) + + So(err, ShouldBeNil) + So(discordNotifier.Name, ShouldEqual, "discord_testing") + So(discordNotifier.Type, ShouldEqual, "discord") + So(discordNotifier.WebhookURL, ShouldEqual, "https://web.hook/") + }) + }) + }) +} diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go index 562ffbe1269..17b88f7d97f 100644 --- a/pkg/services/alerting/notifiers/email.go +++ b/pkg/services/alerting/notifiers/email.go @@ -52,7 +52,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) { }) return &EmailNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Addresses: addresses, log: log.New("alerting.notifier.email"), }, nil diff --git a/pkg/services/alerting/notifiers/hipchat.go b/pkg/services/alerting/notifiers/hipchat.go index f1f63d42a04..388cec79597 100644 --- a/pkg/services/alerting/notifiers/hipchat.go +++ b/pkg/services/alerting/notifiers/hipchat.go @@ -59,7 +59,7 @@ func NewHipChatNotifier(model *models.AlertNotification) (alerting.Notifier, err roomId := model.Settings.Get("roomid").MustString() return &HipChatNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, ApiKey: apikey, RoomId: roomId, @@ -111,7 +111,7 @@ func (this *HipChatNotifier) Notify(evalContext *alerting.EvalContext) error { } message := "" - if evalContext.Rule.State != models.AlertStateOK { //dont add message when going back to alert state ok. + if evalContext.Rule.State != models.AlertStateOK { //don't add message when going back to alert state ok. message += " " + evalContext.Rule.Message } @@ -125,7 +125,7 @@ func (this *HipChatNotifier) Notify(evalContext *alerting.EvalContext) error { case models.AlertStateOK: color = "green" case models.AlertStateNoData: - color = "grey" + color = "gray" case models.AlertStateAlerting: color = "red" } diff --git a/pkg/services/alerting/notifiers/kafka.go b/pkg/services/alerting/notifiers/kafka.go index 92f6489106b..a8a424c87a7 100644 --- a/pkg/services/alerting/notifiers/kafka.go +++ b/pkg/services/alerting/notifiers/kafka.go @@ -43,7 +43,7 @@ func NewKafkaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &KafkaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Endpoint: endpoint, Topic: topic, log: log.New("alerting.notifier.kafka"), @@ -61,7 +61,7 @@ func (this *KafkaNotifier) Notify(evalContext *alerting.EvalContext) error { state := evalContext.Rule.State - customData := "Triggered metrics:\n\n" + customData := triggMetrString for _, evt := range evalContext.EvalMatches { customData = customData + fmt.Sprintf("%s: %v\n", evt.Metric, evt.Value) } diff --git a/pkg/services/alerting/notifiers/line.go b/pkg/services/alerting/notifiers/line.go index 4fbaa2d543e..9e3888b8f95 100644 --- a/pkg/services/alerting/notifiers/line.go +++ b/pkg/services/alerting/notifiers/line.go @@ -39,7 +39,7 @@ func NewLINENotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &LineNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Token: token, log: log.New("alerting.notifier.line"), }, nil @@ -90,7 +90,7 @@ func (this *LineNotifier) createAlert(evalContext *alerting.EvalContext) error { } if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { - this.log.Error("Failed to send notification to LINE", "error", err, "body", string(body)) + this.log.Error("Failed to send notification to LINE", "error", err, "body", body) return err } diff --git a/pkg/services/alerting/notifiers/opsgenie.go b/pkg/services/alerting/notifiers/opsgenie.go index 5d8b15160c4..629968b5102 100644 --- a/pkg/services/alerting/notifiers/opsgenie.go +++ b/pkg/services/alerting/notifiers/opsgenie.go @@ -41,7 +41,7 @@ func init() { } var ( - opsgenieAlertURL string = "https://api.opsgenie.com/v2/alerts" + opsgenieAlertURL = "https://api.opsgenie.com/v2/alerts" ) func NewOpsGenieNotifier(model *m.AlertNotification) (alerting.Notifier, error) { @@ -56,7 +56,7 @@ func NewOpsGenieNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &OpsGenieNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), ApiKey: apiKey, ApiUrl: apiUrl, AutoClose: autoClose, @@ -95,7 +95,7 @@ func (this *OpsGenieNotifier) createAlert(evalContext *alerting.EvalContext) err return err } - customData := "Triggered metrics:\n\n" + customData := triggMetrString for _, evt := range evalContext.EvalMatches { customData = customData + fmt.Sprintf("%s: %v\n", evt.Metric, evt.Value) } diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go index 58484051432..9f6ce3c2dc8 100644 --- a/pkg/services/alerting/notifiers/pagerduty.go +++ b/pkg/services/alerting/notifiers/pagerduty.go @@ -40,7 +40,7 @@ func init() { } var ( - pagerdutyEventApiUrl string = "https://events.pagerduty.com/v2/enqueue" + pagerdutyEventApiUrl = "https://events.pagerduty.com/v2/enqueue" ) func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error) { @@ -51,7 +51,7 @@ func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &PagerdutyNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Key: key, AutoResolve: autoResolve, log: log.New("alerting.notifier.pagerduty"), @@ -76,7 +76,7 @@ func (this *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error { if evalContext.Rule.State == m.AlertStateOK { eventType = "resolve" } - customData := "Triggered metrics:\n\n" + customData := triggMetrString for _, evt := range evalContext.EvalMatches { customData = customData + fmt.Sprintf("%s: %v\n", evt.Metric, evt.Value) } diff --git a/pkg/services/alerting/notifiers/pushover.go b/pkg/services/alerting/notifiers/pushover.go index cbe9e16801a..55dc02c5f4a 100644 --- a/pkg/services/alerting/notifiers/pushover.go +++ b/pkg/services/alerting/notifiers/pushover.go @@ -99,7 +99,7 @@ func NewPushoverNotifier(model *m.AlertNotification) (alerting.Notifier, error) return nil, alerting.ValidationError{Reason: "API token not given"} } return &PushoverNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), UserKey: userKey, ApiToken: apiToken, Priority: priority, diff --git a/pkg/services/alerting/notifiers/sensu.go b/pkg/services/alerting/notifiers/sensu.go index 9f77801d458..21d5d3d9d9e 100644 --- a/pkg/services/alerting/notifiers/sensu.go +++ b/pkg/services/alerting/notifiers/sensu.go @@ -51,7 +51,7 @@ func NewSensuNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &SensuNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, User: model.Settings.Get("username").MustString(), Source: model.Settings.Get("source").MustString(), diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go index e051a71740a..ca5f47a322f 100644 --- a/pkg/services/alerting/notifiers/slack.go +++ b/pkg/services/alerting/notifiers/slack.go @@ -39,6 +39,39 @@ func init() { Override default channel or user, use #channel-name or @username +
+ Username + + + + Set the username for the bot's message + +
+
+ Icon emoji + + + + Provide an emoji to use as the icon for the bot's message. Overrides the icon URL + +
+
+ Icon URL + + + + Provide a URL to an image to use as the icon for the bot's message + +
Mention - Provide a bot token to use the Slack file.upload API (starts with "xoxb") + Provide a bot token to use the Slack file.upload API (starts with "xoxb"). Specify #channel-name or @username in Recipient for this to work
`, @@ -73,14 +106,20 @@ func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } recipient := model.Settings.Get("recipient").MustString() + username := model.Settings.Get("username").MustString() + iconEmoji := model.Settings.Get("icon_emoji").MustString() + iconUrl := model.Settings.Get("icon_url").MustString() mention := model.Settings.Get("mention").MustString() token := model.Settings.Get("token").MustString() uploadImage := model.Settings.Get("uploadImage").MustBool(true) return &SlackNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, Recipient: recipient, + Username: username, + IconEmoji: iconEmoji, + IconUrl: iconUrl, Mention: mention, Token: token, Upload: uploadImage, @@ -92,6 +131,9 @@ type SlackNotifier struct { NotifierBase Url string Recipient string + Username string + IconEmoji string + IconUrl string Mention string Token string Upload bool @@ -129,7 +171,7 @@ func (this *SlackNotifier) Notify(evalContext *alerting.EvalContext) error { } message := this.Mention - if evalContext.Rule.State != m.AlertStateOK { //dont add message when going back to alert state ok. + if evalContext.Rule.State != m.AlertStateOK { //don't add message when going back to alert state ok. message += " " + evalContext.Rule.Message } image_url := "" @@ -160,6 +202,15 @@ func (this *SlackNotifier) Notify(evalContext *alerting.EvalContext) error { if this.Recipient != "" { body["channel"] = this.Recipient } + if this.Username != "" { + body["username"] = this.Username + } + if this.IconEmoji != "" { + body["icon_emoji"] = this.IconEmoji + } + if this.IconUrl != "" { + body["icon_url"] = this.IconUrl + } data, _ := json.Marshal(&body) cmd := &m.SendWebhookSync{Url: this.Url, Body: string(data)} if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { diff --git a/pkg/services/alerting/notifiers/slack_test.go b/pkg/services/alerting/notifiers/slack_test.go index dd82973bc95..17362bc850b 100644 --- a/pkg/services/alerting/notifiers/slack_test.go +++ b/pkg/services/alerting/notifiers/slack_test.go @@ -47,15 +47,21 @@ func TestSlackNotifier(t *testing.T) { So(slackNotifier.Type, ShouldEqual, "slack") So(slackNotifier.Url, ShouldEqual, "http://google.com") So(slackNotifier.Recipient, ShouldEqual, "") + So(slackNotifier.Username, ShouldEqual, "") + So(slackNotifier.IconEmoji, ShouldEqual, "") + So(slackNotifier.IconUrl, ShouldEqual, "") So(slackNotifier.Mention, ShouldEqual, "") So(slackNotifier.Token, ShouldEqual, "") }) - Convey("from settings with Recipient, Mention, and Token", func() { + Convey("from settings with Recipient, Username, IconEmoji, IconUrl, Mention, and Token", func() { json := ` { "url": "http://google.com", "recipient": "#ds-opentsdb", + "username": "Grafana Alerts", + "icon_emoji": ":smile:", + "icon_url": "https://grafana.com/img/fav32.png", "mention": "@carl", "token": "xoxb-XXXXXXXX-XXXXXXXX-XXXXXXXXXX" }` @@ -75,6 +81,9 @@ func TestSlackNotifier(t *testing.T) { So(slackNotifier.Type, ShouldEqual, "slack") So(slackNotifier.Url, ShouldEqual, "http://google.com") So(slackNotifier.Recipient, ShouldEqual, "#ds-opentsdb") + So(slackNotifier.Username, ShouldEqual, "Grafana Alerts") + So(slackNotifier.IconEmoji, ShouldEqual, ":smile:") + So(slackNotifier.IconUrl, ShouldEqual, "https://grafana.com/img/fav32.png") So(slackNotifier.Mention, ShouldEqual, "@carl") So(slackNotifier.Token, ShouldEqual, "xoxb-XXXXXXXX-XXXXXXXX-XXXXXXXXXX") }) diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go index 9a9e93dbc47..2dad11285b4 100644 --- a/pkg/services/alerting/notifiers/teams.go +++ b/pkg/services/alerting/notifiers/teams.go @@ -13,7 +13,7 @@ func init() { alerting.RegisterNotifier(&alerting.NotifierPlugin{ Type: "teams", Name: "Microsoft Teams", - Description: "Sends notifications using Incomming Webhook connector to Microsoft Teams", + Description: "Sends notifications using Incoming Webhook connector to Microsoft Teams", Factory: NewTeamsNotifier, OptionsTemplate: `

Teams settings

@@ -33,7 +33,7 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &TeamsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, log: log.New("alerting.notifier.teams"), }, nil @@ -41,10 +41,8 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) { type TeamsNotifier struct { NotifierBase - Url string - Recipient string - Mention string - log log.Logger + Url string + log log.Logger } func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error { @@ -75,17 +73,17 @@ func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error { }) } - message := this.Mention - if evalContext.Rule.State != m.AlertStateOK { //dont add message when going back to alert state ok. - message += " " + evalContext.Rule.Message - } else { - message += " " // summary must not be empty + message := "" + if evalContext.Rule.State != m.AlertStateOK { //don't add message when going back to alert state ok. + message = evalContext.Rule.Message } body := map[string]interface{}{ - "@type": "MessageCard", - "@context": "http://schema.org/extensions", - "summary": message, + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + // summary MUST not be empty or the webhook request fails + // summary SHOULD contain some meaningful information, since it is used for mobile notifications + "summary": evalContext.GetNotificationTitle(), "title": evalContext.GetNotificationTitle(), "themeColor": evalContext.GetStateModel().Color, "sections": []map[string]interface{}{ @@ -98,14 +96,26 @@ func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error { }, }, "text": message, - "potentialAction": []map[string]interface{}{ + }, + }, + "potentialAction": []map[string]interface{}{ + { + "@context": "http://schema.org", + "@type": "OpenUri", + "name": "View Rule", + "targets": []map[string]interface{}{ { - "@context": "http://schema.org", - "@type": "ViewAction", - "name": "View Rule", - "target": []string{ - ruleUrl, - }, + "os": "default", "uri": ruleUrl, + }, + }, + }, + { + "@context": "http://schema.org", + "@type": "OpenUri", + "name": "View Graph", + "targets": []map[string]interface{}{ + { + "os": "default", "uri": evalContext.ImagePublicUrl, }, }, }, diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go index 5cbdad60906..4a4a989d873 100644 --- a/pkg/services/alerting/notifiers/telegram.go +++ b/pkg/services/alerting/notifiers/telegram.go @@ -3,21 +3,22 @@ package notifiers import ( "bytes" "fmt" + "io" + "mime/multipart" + "os" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" - "io" - "mime/multipart" - "os" ) const ( - captionLengthLimit = 200 + captionLengthLimit = 1024 ) var ( - telegramApiUrl string = "https://api.telegram.org/bot%s/%s" + telegramApiUrl = "https://api.telegram.org/bot%s/%s" ) func init() { @@ -77,7 +78,7 @@ func NewTelegramNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &TelegramNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), BotToken: botToken, ChatID: chatId, UploadImage: uploadImage, @@ -90,9 +91,8 @@ func (this *TelegramNotifier) buildMessage(evalContext *alerting.EvalContext, se cmd, err := this.buildMessageInlineImage(evalContext) if err == nil { return cmd - } else { - this.log.Error("Could not generate Telegram message with inline image.", "err", err) } + this.log.Error("Could not generate Telegram message with inline image.", "err", err) } return this.buildMessageLinkedImage(evalContext) @@ -127,12 +127,21 @@ func (this *TelegramNotifier) buildMessageInlineImage(evalContext *alerting.Eval var err error imageFile, err = os.Open(evalContext.ImageOnDiskPath) - defer imageFile.Close() + defer func() { + err := imageFile.Close() + if err != nil { + log.Error2("Could not close Telegram inline image.", "err", err) + } + }() + if err != nil { return nil, err } ruleUrl, err := evalContext.GetRuleUrl() + if err != nil { + return nil, err + } metrics := generateMetricsMessage(evalContext) message := generateImageCaption(evalContext, ruleUrl, metrics) @@ -213,13 +222,13 @@ func appendIfPossible(message string, extra string, sizeLimit int) string { if len(extra)+len(message) <= sizeLimit { return message + extra } - log.Debug("Line too long for image caption.", "value", extra) + log.Debug("Line too long for image caption. value: %s", extra) return message } func (this *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error { var cmd *m.SendWebhookSync - if evalContext.ImagePublicUrl == "" && this.UploadImage == true { + if evalContext.ImagePublicUrl == "" && this.UploadImage { cmd = this.buildMessage(evalContext, true) } else { cmd = this.buildMessage(evalContext, false) diff --git a/pkg/services/alerting/notifiers/telegram_test.go b/pkg/services/alerting/notifiers/telegram_test.go index 05be787dced..9906a2ffd95 100644 --- a/pkg/services/alerting/notifiers/telegram_test.go +++ b/pkg/services/alerting/notifiers/telegram_test.go @@ -1,6 +1,7 @@ package notifiers import ( + "context" "testing" "github.com/grafana/grafana/pkg/components/simplejson" @@ -52,14 +53,15 @@ func TestTelegramNotifier(t *testing.T) { }) Convey("generateCaption should generate a message with all pertinent details", func() { - evalContext := alerting.NewEvalContext(nil, &alerting.Rule{ - Name: "This is an alarm", - Message: "Some kind of message.", - State: m.AlertStateOK, - }) + evalContext := alerting.NewEvalContext(context.Background(), + &alerting.Rule{ + Name: "This is an alarm", + Message: "Some kind of message.", + State: m.AlertStateOK, + }) caption := generateImageCaption(evalContext, "http://grafa.url/abcdef", "") - So(len(caption), ShouldBeLessThanOrEqualTo, 200) + So(len(caption), ShouldBeLessThanOrEqualTo, 1024) So(caption, ShouldContainSubstring, "Some kind of message.") So(caption, ShouldContainSubstring, "[OK] This is an alarm") So(caption, ShouldContainSubstring, "http://grafa.url/abcdef") @@ -68,16 +70,17 @@ func TestTelegramNotifier(t *testing.T) { Convey("When generating a message", func() { Convey("URL should be skipped if it's too long", func() { - evalContext := alerting.NewEvalContext(nil, &alerting.Rule{ - Name: "This is an alarm", - Message: "Some kind of message.", - State: m.AlertStateOK, - }) + evalContext := alerting.NewEvalContext(context.Background(), + &alerting.Rule{ + Name: "This is an alarm", + Message: "Some kind of message.", + State: m.AlertStateOK, + }) caption := generateImageCaption(evalContext, - "http://grafa.url/abcdefaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "http://grafa.url/abcdefaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "foo bar") - So(len(caption), ShouldBeLessThanOrEqualTo, 200) + So(len(caption), ShouldBeLessThanOrEqualTo, 1024) So(caption, ShouldContainSubstring, "Some kind of message.") So(caption, ShouldContainSubstring, "[OK] This is an alarm") So(caption, ShouldContainSubstring, "foo bar") @@ -85,32 +88,34 @@ func TestTelegramNotifier(t *testing.T) { }) Convey("Message should be trimmed if it's too long", func() { - evalContext := alerting.NewEvalContext(nil, &alerting.Rule{ - Name: "This is an alarm", - Message: "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I promise I will. Yes siree that's it.", - State: m.AlertStateOK, - }) + evalContext := alerting.NewEvalContext(context.Background(), + &alerting.Rule{ + Name: "This is an alarm", + Message: "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I promise I will. Yes siree that's it. But suddenly Telegram increased the length so now we need some lorem ipsum to fix this test. Here we go: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus consectetur molestie cursus. Donec suscipit egestas nisi. Proin ut efficitur ex. Mauris mi augue, volutpat a nisi vel, euismod dictum arcu. Sed quis tempor eros, sed malesuada dolor. Ut orci augue, viverra sit amet blandit quis, faucibus sit amet ex. Duis condimentum efficitur lectus, id dignissim quam tempor id. Morbi sollicitudin rhoncus diam, id tincidunt lectus scelerisque vitae. Etiam imperdiet semper sem, vel eleifend ligula mollis eget. Etiam ultrices fringilla lacus, sit amet pharetra ex blandit quis. Suspendisse in egestas neque, et posuere lectus. Vestibulum eu ex dui. Sed molestie nulla a lobortis scelerisque. Nulla ipsum ex, iaculis vitae vehicula sit amet, fermentum eu eros.", + State: m.AlertStateOK, + }) caption := generateImageCaption(evalContext, "http://grafa.url/foo", "") - So(len(caption), ShouldBeLessThanOrEqualTo, 200) + So(len(caption), ShouldBeLessThanOrEqualTo, 1024) So(caption, ShouldContainSubstring, "[OK] This is an alarm") So(caption, ShouldNotContainSubstring, "http") - So(caption, ShouldContainSubstring, "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I promise ") + So(caption, ShouldContainSubstring, "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I promise I will. Yes siree that's it. But suddenly Telegram increased the length so now we need some lorem ipsum to fix this test. Here we go: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus consectetur molestie cursus. Donec suscipit egestas nisi. Proin ut efficitur ex. Mauris mi augue, volutpat a nisi vel, euismod dictum arcu. Sed quis tempor eros, sed malesuada dolor. Ut orci augue, viverra sit amet blandit quis, faucibus sit amet ex. Duis condimentum efficitur lectus, id dignissim quam tempor id. Morbi sollicitudin rhoncus diam, id tincidunt lectus scelerisque vitae. Etiam imperdiet semper sem, vel eleifend ligula mollis eget. Etiam ultrices fringilla lacus, sit amet pharetra ex blandit quis. Suspendisse in egestas neque, et posuere lectus. Vestibulum eu ex dui. Sed molestie nulla a lobortis sceleri") }) - Convey("Metrics should be skipped if they dont fit", func() { - evalContext := alerting.NewEvalContext(nil, &alerting.Rule{ - Name: "This is an alarm", - Message: "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I ", - State: m.AlertStateOK, - }) + Convey("Metrics should be skipped if they don't fit", func() { + evalContext := alerting.NewEvalContext(context.Background(), + &alerting.Rule{ + Name: "This is an alarm", + Message: "Some kind of message that is too long for appending to our pretty little message, this line is actually exactly 197 chars long and I will get there in the end I promise I will. Yes siree that's it. But suddenly Telegram increased the length so now we need some lorem ipsum to fix this test. Here we go: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus consectetur molestie cursus. Donec suscipit egestas nisi. Proin ut efficitur ex. Mauris mi augue, volutpat a nisi vel, euismod dictum arcu. Sed quis tempor eros, sed malesuada dolor. Ut orci augue, viverra sit amet blandit quis, faucibus sit amet ex. Duis condimentum efficitur lectus, id dignissim quam tempor id. Morbi sollicitudin rhoncus diam, id tincidunt lectus scelerisque vitae. Etiam imperdiet semper sem, vel eleifend ligula mollis eget. Etiam ultrices fringilla lacus, sit amet pharetra ex blandit quis. Suspendisse in egestas neque, et posuere lectus. Vestibulum eu ex dui. Sed molestie nulla a lobortis sceleri", + State: m.AlertStateOK, + }) caption := generateImageCaption(evalContext, "http://grafa.url/foo", "foo bar long song") - So(len(caption), ShouldBeLessThanOrEqualTo, 200) + So(len(caption), ShouldBeLessThanOrEqualTo, 1024) So(caption, ShouldContainSubstring, "[OK] This is an alarm") So(caption, ShouldNotContainSubstring, "http") So(caption, ShouldNotContainSubstring, "foo bar") diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go index e4ffffc9108..28a62fade17 100644 --- a/pkg/services/alerting/notifiers/threema.go +++ b/pkg/services/alerting/notifiers/threema.go @@ -106,7 +106,7 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &ThreemaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), GatewayID: gatewayID, RecipientID: recipientID, APISecret: apiSecret, diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index 4b4db553cde..3093aec9957 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -51,7 +51,7 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e } return &VictoropsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), URL: url, AutoResolve: autoResolve, log: log.New("alerting.notifier.victorops"), @@ -83,27 +83,6 @@ func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { return nil } - fields := make([]map[string]interface{}, 0) - fieldLimitCount := 4 - for index, evt := range evalContext.EvalMatches { - fields = append(fields, map[string]interface{}{ - "title": evt.Metric, - "value": evt.Value, - "short": true, - }) - if index > fieldLimitCount { - break - } - } - - if evalContext.Error != nil { - fields = append(fields, map[string]interface{}{ - "title": "Error message", - "value": evalContext.Error.Error(), - "short": false, - }) - } - messageType := evalContext.Rule.State if evalContext.Rule.State == models.AlertStateAlerting { // translate 'Alerting' to 'CRITICAL' (Victorops analog) messageType = AlertStateCritical diff --git a/pkg/services/alerting/notifiers/webhook.go b/pkg/services/alerting/notifiers/webhook.go index 4c97ed2b75e..4045e496af9 100644 --- a/pkg/services/alerting/notifiers/webhook.go +++ b/pkg/services/alerting/notifiers/webhook.go @@ -47,7 +47,7 @@ func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &WebhookNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, User: model.Settings.Get("username").MustString(), Password: model.Settings.Get("password").MustString(), diff --git a/pkg/services/alerting/reader.go b/pkg/services/alerting/reader.go index 45f0c65d4fb..2cdbc57b41d 100644 --- a/pkg/services/alerting/reader.go +++ b/pkg/services/alerting/reader.go @@ -16,7 +16,7 @@ type RuleReader interface { type DefaultRuleReader struct { sync.RWMutex - serverID string + //serverID string serverPosition int clusterSize int log log.Logger @@ -34,11 +34,8 @@ func NewRuleReader() *DefaultRuleReader { func (arr *DefaultRuleReader) initReader() { heartbeat := time.NewTicker(time.Second * 10) - for { - select { - case <-heartbeat.C: - arr.heartbeat() - } + for range heartbeat.C { + arr.heartbeat() } } diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 8f9deb758a6..420ffeb9a55 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/metrics" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/annotations" + "github.com/grafana/grafana/pkg/services/rendering" ) type ResultHandler interface { @@ -20,10 +21,10 @@ type DefaultResultHandler struct { log log.Logger } -func NewResultHandler() *DefaultResultHandler { +func NewResultHandler(renderService rendering.Service) *DefaultResultHandler { return &DefaultResultHandler{ log: log.New("alerting.resultHandler"), - notifier: NewNotificationService(), + notifier: NewNotificationService(renderService), } } @@ -56,7 +57,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { if err := bus.Dispatch(cmd); err != nil { if err == m.ErrCannotChangeStateOnPausedAlert { - handler.log.Error("Cannot change state on alert thats pause", "error", err) + handler.log.Error("Cannot change state on alert that's paused", "error", err) return err } @@ -66,6 +67,12 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } handler.log.Error("Failed to save state", "error", err) + } else { + + // StateChanges is used for de duping alert notifications + // when two servers are raising. This makes sure that the server + // with the last state change always sends a notification. + evalContext.Rule.StateChanges = cmd.Result.StateChanges } // save annotation @@ -77,7 +84,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { Text: "", NewState: string(evalContext.Rule.State), PrevState: string(evalContext.PrevAlertState), - Epoch: time.Now().Unix(), + Epoch: time.Now().UnixNano() / int64(time.Millisecond), Data: annotationData, } @@ -88,6 +95,5 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } handler.notifier.SendIfNeeded(evalContext) - return nil } diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index bdf53798e34..999611f15c4 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -23,6 +23,8 @@ type Rule struct { State m.AlertStateType Conditions []Condition Notifications []int64 + + StateChanges int64 } type ValidationError struct { @@ -34,13 +36,13 @@ type ValidationError struct { } func (e ValidationError) Error() string { - extraInfo := "" + extraInfo := e.Reason if e.Alertid != 0 { extraInfo = fmt.Sprintf("%s AlertId: %v", extraInfo, e.Alertid) } if e.PanelId != 0 { - extraInfo = fmt.Sprintf("%s PanelId: %v ", extraInfo, e.PanelId) + extraInfo = fmt.Sprintf("%s PanelId: %v", extraInfo, e.PanelId) } if e.DashboardId != 0 { @@ -48,15 +50,15 @@ func (e ValidationError) Error() string { } if e.Err != nil { - return fmt.Sprintf("%s %s%s", e.Err.Error(), e.Reason, extraInfo) + return fmt.Sprintf("Alert validation error: %s%s", e.Err.Error(), extraInfo) } - return fmt.Sprintf("Failed to extract alert.Reason: %s %s", e.Reason, extraInfo) + return fmt.Sprintf("Alert validation error: %s", extraInfo) } var ( - ValueFormatRegex = regexp.MustCompile("^\\d+") - UnitFormatRegex = regexp.MustCompile("\\w{1}$") + ValueFormatRegex = regexp.MustCompile(`^\d+`) + UnitFormatRegex = regexp.MustCompile(`\w{1}$`) ) var unitMultiplier = map[string]int{ @@ -100,32 +102,33 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { model.State = ruleDef.State model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) + model.StateChanges = ruleDef.StateChanges for _, v := range ruleDef.Settings.Get("notifications").MustArray() { jsonModel := simplejson.NewFromAny(v) - if id, err := jsonModel.Get("id").Int64(); err != nil { + id, err := jsonModel.Get("id").Int64() + if err != nil { return nil, ValidationError{Reason: "Invalid notification schema", DashboardId: model.DashboardId, Alertid: model.Id, PanelId: model.PanelId} - } else { - model.Notifications = append(model.Notifications, id) } + model.Notifications = append(model.Notifications, id) } for index, condition := range ruleDef.Settings.Get("conditions").MustArray() { conditionModel := simplejson.NewFromAny(condition) conditionType := conditionModel.Get("type").MustString() - if factory, exist := conditionFactories[conditionType]; !exist { + factory, exist := conditionFactories[conditionType] + if !exist { return nil, ValidationError{Reason: "Unknown alert condition: " + conditionType, DashboardId: model.DashboardId, Alertid: model.Id, PanelId: model.PanelId} - } else { - if queryCondition, err := factory(conditionModel, index); err != nil { - return nil, ValidationError{Err: err, DashboardId: model.DashboardId, Alertid: model.Id, PanelId: model.PanelId} - } else { - model.Conditions = append(model.Conditions, queryCondition) - } } + queryCondition, err := factory(conditionModel, index) + if err != nil { + return nil, ValidationError{Err: err, DashboardId: model.DashboardId, Alertid: model.Id, PanelId: model.PanelId} + } + model.Conditions = append(model.Conditions, queryCondition) } if len(model.Conditions) == 0 { - return nil, fmt.Errorf("Alert is missing conditions") + return nil, ValidationError{Reason: "Alert is missing conditions"} } return model, nil @@ -133,7 +136,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { type ConditionFactory func(model *simplejson.Json, index int) (Condition, error) -var conditionFactories map[string]ConditionFactory = make(map[string]ConditionFactory) +var conditionFactories = make(map[string]ConditionFactory) func RegisterCondition(typeName string, factory ConditionFactory) { conditionFactories[typeName] = factory diff --git a/pkg/services/alerting/scheduler.go b/pkg/services/alerting/scheduler.go index 151f802ec15..b7555ae8d89 100644 --- a/pkg/services/alerting/scheduler.go +++ b/pkg/services/alerting/scheduler.go @@ -15,7 +15,7 @@ type SchedulerImpl struct { func NewScheduler() Scheduler { return &SchedulerImpl{ - jobs: make(map[int64]*Job, 0), + jobs: make(map[int64]*Job), log: log.New("alerting.scheduler"), } } @@ -23,7 +23,7 @@ func NewScheduler() Scheduler { func (s *SchedulerImpl) Update(rules []*Rule) { s.log.Debug("Scheduling update", "ruleCount", len(rules)) - jobs := make(map[int64]*Job, 0) + jobs := make(map[int64]*Job) for i, rule := range rules { var job *Job @@ -58,7 +58,7 @@ func (s *SchedulerImpl) Tick(tickTime time.Time, execQueue chan *Job) { if job.OffsetWait && now%job.Offset == 0 { job.OffsetWait = false - s.enque(job, execQueue) + s.enqueue(job, execQueue) continue } @@ -66,13 +66,13 @@ func (s *SchedulerImpl) Tick(tickTime time.Time, execQueue chan *Job) { if job.Offset > 0 { job.OffsetWait = true } else { - s.enque(job, execQueue) + s.enqueue(job, execQueue) } } } } -func (s *SchedulerImpl) enque(job *Job, execQueue chan *Job) { +func (s *SchedulerImpl) enqueue(job *Job, execQueue chan *Job) { s.log.Debug("Scheduler: Putting job on to exec queue", "name", job.Rule.Name, "id", job.Rule.Id) execQueue <- job } diff --git a/pkg/services/alerting/test-data/dash-without-id.json b/pkg/services/alerting/test-data/dash-without-id.json new file mode 100644 index 00000000000..e0a212695d8 --- /dev/null +++ b/pkg/services/alerting/test-data/dash-without-id.json @@ -0,0 +1,281 @@ +{ + "title": "Influxdb", + "tags": [ + "apa" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "450px", + "panels": [ + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 10 + ], + "type": "gt" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "frequency": "3s", + "handler": 1, + "name": "Influxdb", + "noDataState": "no_data", + "notifications": [ + { + "id": 6 + } + ] + }, + "alerting": {}, + "aliasColors": { + "logins.count.count": "#890F02" + }, + "bars": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 1, + "interval": ">10s", + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "datacenter" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "logins.count", + "policy": "default", + "query": "SELECT 8 * count(\"value\") FROM \"logins.count\" WHERE $timeFilter GROUP BY time($interval), \"datacenter\" fill(none)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + }, + { + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": true, + "measurement": "cpu", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ], + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "tooltip": { + "msResolution": false, + "ordering": "alphabetical", + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "editable": true, + "error": false, + "id": 2, + "isNew": true, + "limit": 10, + "links": [], + "show": "current", + "span": 2, + "stateFilter": [ + "alerting" + ], + "title": "Alert status", + "type": "alertlist" + } + ], + "title": "Row" + } + ], + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "schemaVersion": 13, + "version": 120, + "links": [], + "gnetId": null + } diff --git a/pkg/services/alerting/test-data/influxdb-alert.json b/pkg/services/alerting/test-data/influxdb-alert.json index 79ca355c5a1..fd6feb31a47 100644 --- a/pkg/services/alerting/test-data/influxdb-alert.json +++ b/pkg/services/alerting/test-data/influxdb-alert.json @@ -279,4 +279,4 @@ "version": 120, "links": [], "gnetId": null - } \ No newline at end of file + } diff --git a/pkg/services/alerting/test_notification.go b/pkg/services/alerting/test_notification.go index 7dc9a150d92..8aa1b80aa22 100644 --- a/pkg/services/alerting/test_notification.go +++ b/pkg/services/alerting/test_notification.go @@ -24,7 +24,7 @@ func init() { } func handleNotificationTestCommand(cmd *NotificationTestCommand) error { - notifier := newNotificationService() + notifier := NewNotificationService(nil).(*notificationService) model := &m.AlertNotification{ Name: cmd.Name, @@ -39,7 +39,7 @@ func handleNotificationTestCommand(cmd *NotificationTestCommand) error { return err } - return notifier.sendNotifications(createTestEvalContext(cmd), []Notifier{notifiers}) + return notifier.sendNotifications(createTestEvalContext(cmd), notifierStateSlice{{notifier: notifiers}}) } func createTestEvalContext(cmd *NotificationTestCommand) *EvalContext { diff --git a/pkg/services/alerting/test_rule.go b/pkg/services/alerting/test_rule.go index e3aa95e0ede..360ee065de0 100644 --- a/pkg/services/alerting/test_rule.go +++ b/pkg/services/alerting/test_rule.go @@ -13,6 +13,7 @@ type AlertTestCommand struct { Dashboard *simplejson.Json PanelId int64 OrgId int64 + User *m.SignedInUser Result *EvalContext } @@ -25,7 +26,7 @@ func handleAlertTestCommand(cmd *AlertTestCommand) error { dash := m.NewDashboardFromJson(cmd.Dashboard) - extractor := NewDashAlertExtractor(dash, cmd.OrgId) + extractor := NewDashAlertExtractor(dash, cmd.OrgId, cmd.User) alerts, err := extractor.GetAlerts() if err != nil { return err @@ -53,6 +54,7 @@ func testAlertRule(rule *Rule) *EvalContext { context.IsTestRun = true handler.Eval(context) + context.Rule.State = context.GetNewState() return context } diff --git a/pkg/services/alerting/ticker.go b/pkg/services/alerting/ticker.go index 5ce19b1b232..8cee2653ee9 100644 --- a/pkg/services/alerting/ticker.go +++ b/pkg/services/alerting/ticker.go @@ -37,10 +37,6 @@ func NewTicker(last time.Time, initialOffset time.Duration, c clock.Clock) *Tick return t } -func (t *Ticker) updateOffset(offset time.Duration) { - t.newOffset <- offset -} - func (t *Ticker) run() { for { next := t.last.Add(time.Duration(1) * time.Second) diff --git a/pkg/services/annotations/annotations.go b/pkg/services/annotations/annotations.go index a6cd7a33318..60a92aa897a 100644 --- a/pkg/services/annotations/annotations.go +++ b/pkg/services/annotations/annotations.go @@ -13,6 +13,7 @@ type ItemQuery struct { OrgId int64 `json:"orgId"` From int64 `json:"from"` To int64 `json:"to"` + UserId int64 `json:"userId"` AlertId int64 `json:"alertId"` DashboardId int64 `json:"dashboardId"` PanelId int64 `json:"panelId"` @@ -20,6 +21,7 @@ type ItemQuery struct { RegionId int64 `json:"regionId"` Tags []string `json:"tags"` Type string `json:"type"` + MatchAny bool `json:"matchAny"` Limit int64 `json:"limit"` } @@ -34,11 +36,12 @@ type PostParams struct { } type DeleteParams struct { - Id int64 `json:"id"` - AlertId int64 `json:"alertId"` - DashboardId int64 `json:"dashboardId"` - PanelId int64 `json:"panelId"` - RegionId int64 `json:"regionId"` + OrgId int64 + Id int64 + AlertId int64 + DashboardId int64 + PanelId int64 + RegionId int64 } var repositoryInstance Repository @@ -63,6 +66,8 @@ type Item struct { PrevState string `json:"prevState"` NewState string `json:"newState"` Epoch int64 `json:"epoch"` + Created int64 `json:"created"` + Updated int64 `json:"updated"` Tags []string `json:"tags"` Data *simplejson.Json `json:"data"` @@ -80,6 +85,8 @@ type ItemDTO struct { UserId int64 `json:"userId"` NewState string `json:"newState"` PrevState string `json:"prevState"` + Created int64 `json:"created"` + Updated int64 `json:"updated"` Time int64 `json:"time"` Text string `json:"text"` RegionId int64 `json:"regionId"` diff --git a/pkg/services/cache/cache.go b/pkg/services/cache/cache.go new file mode 100644 index 00000000000..93b2cf76e26 --- /dev/null +++ b/pkg/services/cache/cache.go @@ -0,0 +1,17 @@ +package cache + +import ( + "time" + + gocache "github.com/patrickmn/go-cache" +) + +type CacheService struct { + *gocache.Cache +} + +func New(defaultExpiration, cleanupInterval time.Duration) *CacheService { + return &CacheService{ + Cache: gocache.New(defaultExpiration, cleanupInterval), + } +} diff --git a/pkg/services/cleanup/cleanup.go b/pkg/services/cleanup/cleanup.go index 5e9efeea3b0..c15ae8ef36c 100644 --- a/pkg/services/cleanup/cleanup.go +++ b/pkg/services/cleanup/cleanup.go @@ -7,101 +7,103 @@ import ( "path" "time" - "golang.org/x/sync/errgroup" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/setting" ) type CleanUpService struct { log log.Logger + Cfg *setting.Cfg `inject:""` } -func NewCleanUpService() *CleanUpService { - return &CleanUpService{ - log: log.New("cleanup"), - } +func init() { + registry.RegisterService(&CleanUpService{}) } -func (service *CleanUpService) Run(ctx context.Context) error { - service.log.Info("Initializing CleanUpService") - - g, _ := errgroup.WithContext(ctx) - g.Go(func() error { return service.start(ctx) }) - - err := g.Wait() - service.log.Info("Stopped CleanUpService", "reason", err) - return err +func (srv *CleanUpService) Init() error { + srv.log = log.New("cleanup") + return nil } -func (service *CleanUpService) start(ctx context.Context) error { - service.cleanUpTmpFiles() +func (srv *CleanUpService) Run(ctx context.Context) error { + srv.cleanUpTmpFiles() ticker := time.NewTicker(time.Minute * 10) for { select { case <-ticker.C: - service.cleanUpTmpFiles() - service.deleteExpiredSnapshots() - service.deleteExpiredDashboardVersions() - service.deleteOldLoginAttempts() + srv.cleanUpTmpFiles() + srv.deleteExpiredSnapshots() + srv.deleteExpiredDashboardVersions() + srv.deleteOldLoginAttempts() case <-ctx.Done(): return ctx.Err() } } } -func (service *CleanUpService) cleanUpTmpFiles() { - if _, err := os.Stat(setting.ImagesDir); os.IsNotExist(err) { +func (srv *CleanUpService) cleanUpTmpFiles() { + if _, err := os.Stat(srv.Cfg.ImagesDir); os.IsNotExist(err) { return } - files, err := ioutil.ReadDir(setting.ImagesDir) + files, err := ioutil.ReadDir(srv.Cfg.ImagesDir) if err != nil { - service.log.Error("Problem reading image dir", "error", err) + srv.log.Error("Problem reading image dir", "error", err) return } var toDelete []os.FileInfo + var now = time.Now() + for _, file := range files { - if file.ModTime().AddDate(0, 0, 1).Before(time.Now()) { + if srv.shouldCleanupTempFile(file.ModTime(), now) { toDelete = append(toDelete, file) } } for _, file := range toDelete { - fullPath := path.Join(setting.ImagesDir, file.Name()) + fullPath := path.Join(srv.Cfg.ImagesDir, file.Name()) err := os.Remove(fullPath) if err != nil { - service.log.Error("Failed to delete temp file", "file", file.Name(), "error", err) + srv.log.Error("Failed to delete temp file", "file", file.Name(), "error", err) } } - service.log.Debug("Found old rendered image to delete", "deleted", len(toDelete), "keept", len(files)) + srv.log.Debug("Found old rendered image to delete", "deleted", len(toDelete), "kept", len(files)) } -func (service *CleanUpService) deleteExpiredSnapshots() { +func (srv *CleanUpService) shouldCleanupTempFile(filemtime time.Time, now time.Time) bool { + if srv.Cfg.TempDataLifetime == 0 { + return false + } + + return filemtime.Add(srv.Cfg.TempDataLifetime).Before(now) +} + +func (srv *CleanUpService) deleteExpiredSnapshots() { cmd := m.DeleteExpiredSnapshotsCommand{} if err := bus.Dispatch(&cmd); err != nil { - service.log.Error("Failed to delete expired snapshots", "error", err.Error()) + srv.log.Error("Failed to delete expired snapshots", "error", err.Error()) } else { - service.log.Debug("Deleted expired snapshots", "rows affected", cmd.DeletedRows) + srv.log.Debug("Deleted expired snapshots", "rows affected", cmd.DeletedRows) } } -func (service *CleanUpService) deleteExpiredDashboardVersions() { +func (srv *CleanUpService) deleteExpiredDashboardVersions() { cmd := m.DeleteExpiredVersionsCommand{} if err := bus.Dispatch(&cmd); err != nil { - service.log.Error("Failed to delete expired dashboard versions", "error", err.Error()) + srv.log.Error("Failed to delete expired dashboard versions", "error", err.Error()) } else { - service.log.Debug("Deleted old/expired dashboard versions", "rows affected", cmd.DeletedRows) + srv.log.Debug("Deleted old/expired dashboard versions", "rows affected", cmd.DeletedRows) } } -func (service *CleanUpService) deleteOldLoginAttempts() { - if setting.DisableBruteForceLoginProtection { +func (srv *CleanUpService) deleteOldLoginAttempts() { + if srv.Cfg.DisableBruteForceLoginProtection { return } @@ -109,8 +111,8 @@ func (service *CleanUpService) deleteOldLoginAttempts() { OlderThan: time.Now().Add(time.Minute * -10), } if err := bus.Dispatch(&cmd); err != nil { - service.log.Error("Problem deleting expired login attempts", "error", err.Error()) + srv.log.Error("Problem deleting expired login attempts", "error", err.Error()) } else { - service.log.Debug("Deleted expired login attempts", "rows affected", cmd.DeletedRows) + srv.log.Debug("Deleted expired login attempts", "rows affected", cmd.DeletedRows) } } diff --git a/pkg/services/cleanup/cleanup_test.go b/pkg/services/cleanup/cleanup_test.go new file mode 100644 index 00000000000..54d29e32bf1 --- /dev/null +++ b/pkg/services/cleanup/cleanup_test.go @@ -0,0 +1,41 @@ +package cleanup + +import ( + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" + "testing" + "time" +) + +func TestCleanUpTmpFiles(t *testing.T) { + Convey("Cleanup service tests", t, func() { + cfg := setting.Cfg{} + cfg.TempDataLifetime, _ = time.ParseDuration("24h") + service := CleanUpService{ + Cfg: &cfg, + } + now := time.Now() + secondAgo := now.Add(-time.Second) + twoDaysAgo := now.Add(-time.Second * 3600 * 24 * 2) + weekAgo := now.Add(-time.Second * 3600 * 24 * 7) + + Convey("Should not cleanup recent files", func() { + So(service.shouldCleanupTempFile(secondAgo, now), ShouldBeFalse) + }) + + Convey("Should cleanup older files", func() { + So(service.shouldCleanupTempFile(twoDaysAgo, now), ShouldBeTrue) + }) + + Convey("After increasing temporary files lifetime, older files should be kept", func() { + cfg.TempDataLifetime, _ = time.ParseDuration("1000h") + So(service.shouldCleanupTempFile(weekAgo, now), ShouldBeFalse) + }) + + Convey("If lifetime is 0, files should never be cleaned up", func() { + cfg.TempDataLifetime = 0 + So(service.shouldCleanupTempFile(weekAgo, now), ShouldBeFalse) + }) + }) + +} diff --git a/pkg/services/dashboards/dashboard_service.go b/pkg/services/dashboards/dashboard_service.go index 02a6ffc8330..b52d1845a0b 100644 --- a/pkg/services/dashboards/dashboard_service.go +++ b/pkg/services/dashboards/dashboard_service.go @@ -5,6 +5,7 @@ import ( "time" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/guardian" "github.com/grafana/grafana/pkg/util" @@ -25,7 +26,9 @@ type DashboardProvisioningService interface { // NewService factory for creating a new dashboard service var NewService = func() DashboardService { - return &dashboardServiceImpl{} + return &dashboardServiceImpl{ + log: log.New("dashboard-service"), + } } // NewProvisioningService factory for creating a new dashboard provisioning service @@ -45,6 +48,7 @@ type SaveDashboardDTO struct { type dashboardServiceImpl struct { orgId int64 user *models.SignedInUser + log log.Logger } func (dr *dashboardServiceImpl) GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) { @@ -57,7 +61,7 @@ func (dr *dashboardServiceImpl) GetProvisionedDashboardData(name string) ([]*mod return cmd.Result, nil } -func (dr *dashboardServiceImpl) buildSaveDashboardCommand(dto *SaveDashboardDTO, validateAlerts bool) (*models.SaveDashboardCommand, error) { +func (dr *dashboardServiceImpl) buildSaveDashboardCommand(dto *SaveDashboardDTO, validateAlerts bool, validateProvisionedDashboard bool) (*models.SaveDashboardCommand, error) { dash := dto.Dashboard dash.Title = strings.TrimSpace(dash.Title) @@ -86,10 +90,11 @@ func (dr *dashboardServiceImpl) buildSaveDashboardCommand(dto *SaveDashboardDTO, validateAlertsCmd := models.ValidateDashboardAlertsCommand{ OrgId: dto.OrgId, Dashboard: dash, + User: dto.User, } if err := bus.Dispatch(&validateAlertsCmd); err != nil { - return nil, models.ErrDashboardContainsInvalidAlertData + return nil, err } } @@ -103,6 +108,29 @@ func (dr *dashboardServiceImpl) buildSaveDashboardCommand(dto *SaveDashboardDTO, return nil, err } + if validateBeforeSaveCmd.Result.IsParentFolderChanged { + folderGuardian := guardian.New(dash.FolderId, dto.OrgId, dto.User) + if canSave, err := folderGuardian.CanSave(); err != nil || !canSave { + if err != nil { + return nil, err + } + return nil, models.ErrDashboardUpdateAccessDenied + } + } + + if validateProvisionedDashboard { + isDashboardProvisioned := &models.IsDashboardProvisionedQuery{DashboardId: dash.Id} + err := bus.Dispatch(isDashboardProvisioned) + + if err != nil { + return nil, err + } + + if isDashboardProvisioned.Result { + return nil, models.ErrDashboardCannotSaveProvisionedDashboard + } + } + guard := guardian.New(dash.GetDashboardIdForSavePermissionCheck(), dto.OrgId, dto.User) if canSave, err := guard.CanSave(); err != nil || !canSave { if err != nil { @@ -132,8 +160,8 @@ func (dr *dashboardServiceImpl) buildSaveDashboardCommand(dto *SaveDashboardDTO, func (dr *dashboardServiceImpl) updateAlerting(cmd *models.SaveDashboardCommand, dto *SaveDashboardDTO) error { alertCmd := models.UpdateDashboardAlertsCommand{ OrgId: dto.OrgId, - UserId: dto.User.UserId, Dashboard: cmd.Result, + User: dto.User, } if err := bus.Dispatch(&alertCmd); err != nil { @@ -148,7 +176,7 @@ func (dr *dashboardServiceImpl) SaveProvisionedDashboard(dto *SaveDashboardDTO, UserId: 0, OrgRole: models.ROLE_ADMIN, } - cmd, err := dr.buildSaveDashboardCommand(dto, true) + cmd, err := dr.buildSaveDashboardCommand(dto, true, false) if err != nil { return nil, err } @@ -178,7 +206,7 @@ func (dr *dashboardServiceImpl) SaveFolderForProvisionedDashboards(dto *SaveDash UserId: 0, OrgRole: models.ROLE_ADMIN, } - cmd, err := dr.buildSaveDashboardCommand(dto, false) + cmd, err := dr.buildSaveDashboardCommand(dto, false, false) if err != nil { return nil, err } @@ -197,7 +225,7 @@ func (dr *dashboardServiceImpl) SaveFolderForProvisionedDashboards(dto *SaveDash } func (dr *dashboardServiceImpl) SaveDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) { - cmd, err := dr.buildSaveDashboardCommand(dto, true) + cmd, err := dr.buildSaveDashboardCommand(dto, true, true) if err != nil { return nil, err } @@ -216,7 +244,7 @@ func (dr *dashboardServiceImpl) SaveDashboard(dto *SaveDashboardDTO) (*models.Da } func (dr *dashboardServiceImpl) ImportDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) { - cmd, err := dr.buildSaveDashboardCommand(dto, false) + cmd, err := dr.buildSaveDashboardCommand(dto, false, true) if err != nil { return nil, err } diff --git a/pkg/services/dashboards/dashboard_service_test.go b/pkg/services/dashboards/dashboard_service_test.go index 965b10655b3..b8300a5af8d 100644 --- a/pkg/services/dashboards/dashboard_service_test.go +++ b/pkg/services/dashboards/dashboard_service_test.go @@ -14,7 +14,9 @@ import ( func TestDashboardService(t *testing.T) { Convey("Dashboard service tests", t, func() { - service := dashboardServiceImpl{} + bus.ClearBusHandlers() + + service := &dashboardServiceImpl{} origNewDashboardGuardian := guardian.New guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{CanSaveValue: true}) @@ -51,6 +53,12 @@ func TestDashboardService(t *testing.T) { }) bus.AddHandler("test", func(cmd *models.ValidateDashboardBeforeSaveCommand) error { + cmd.Result = &models.ValidateDashboardBeforeSaveResult{} + return nil + }) + + bus.AddHandler("test", func(cmd *models.IsDashboardProvisionedQuery) error { + cmd.Result = false return nil }) @@ -72,19 +80,123 @@ func TestDashboardService(t *testing.T) { dto.Dashboard.SetUid(tc.Uid) dto.User = &models.SignedInUser{} - _, err := service.buildSaveDashboardCommand(dto, true) + _, err := service.buildSaveDashboardCommand(dto, true, false) So(err, ShouldEqual, tc.Error) } }) - Convey("Should return validation error if alert data is invalid", func() { + Convey("Should return validation error if dashboard is provisioned", func() { + provisioningValidated := false + bus.AddHandler("test", func(cmd *models.IsDashboardProvisionedQuery) error { + provisioningValidated = true + cmd.Result = true + return nil + }) + bus.AddHandler("test", func(cmd *models.ValidateDashboardAlertsCommand) error { - return errors.New("error") + return nil + }) + + bus.AddHandler("test", func(cmd *models.ValidateDashboardBeforeSaveCommand) error { + cmd.Result = &models.ValidateDashboardBeforeSaveResult{} + return nil + }) + + dto.Dashboard = models.NewDashboard("Dash") + dto.Dashboard.SetId(3) + dto.User = &models.SignedInUser{UserId: 1} + _, err := service.SaveDashboard(dto) + So(provisioningValidated, ShouldBeTrue) + So(err, ShouldEqual, models.ErrDashboardCannotSaveProvisionedDashboard) + }) + + Convey("Should return validation error if alert data is invalid", func() { + bus.AddHandler("test", func(cmd *models.IsDashboardProvisionedQuery) error { + cmd.Result = false + return nil + }) + + bus.AddHandler("test", func(cmd *models.ValidateDashboardAlertsCommand) error { + return errors.New("Alert validation error") }) dto.Dashboard = models.NewDashboard("Dash") _, err := service.SaveDashboard(dto) - So(err, ShouldEqual, models.ErrDashboardContainsInvalidAlertData) + So(err.Error(), ShouldEqual, "Alert validation error") + }) + }) + + Convey("Save provisioned dashboard validation", func() { + dto := &SaveDashboardDTO{} + + Convey("Should not return validation error if dashboard is provisioned", func() { + provisioningValidated := false + bus.AddHandler("test", func(cmd *models.IsDashboardProvisionedQuery) error { + provisioningValidated = true + cmd.Result = true + return nil + }) + + bus.AddHandler("test", func(cmd *models.ValidateDashboardAlertsCommand) error { + return nil + }) + + bus.AddHandler("test", func(cmd *models.ValidateDashboardBeforeSaveCommand) error { + cmd.Result = &models.ValidateDashboardBeforeSaveResult{} + return nil + }) + + bus.AddHandler("test", func(cmd *models.SaveProvisionedDashboardCommand) error { + return nil + }) + + bus.AddHandler("test", func(cmd *models.UpdateDashboardAlertsCommand) error { + return nil + }) + + dto.Dashboard = models.NewDashboard("Dash") + dto.Dashboard.SetId(3) + dto.User = &models.SignedInUser{UserId: 1} + _, err := service.SaveProvisionedDashboard(dto, nil) + So(err, ShouldBeNil) + So(provisioningValidated, ShouldBeFalse) + }) + }) + + Convey("Import dashboard validation", func() { + dto := &SaveDashboardDTO{} + + Convey("Should return validation error if dashboard is provisioned", func() { + provisioningValidated := false + bus.AddHandler("test", func(cmd *models.IsDashboardProvisionedQuery) error { + provisioningValidated = true + cmd.Result = true + return nil + }) + + bus.AddHandler("test", func(cmd *models.ValidateDashboardAlertsCommand) error { + return nil + }) + + bus.AddHandler("test", func(cmd *models.ValidateDashboardBeforeSaveCommand) error { + cmd.Result = &models.ValidateDashboardBeforeSaveResult{} + return nil + }) + + bus.AddHandler("test", func(cmd *models.SaveProvisionedDashboardCommand) error { + return nil + }) + + bus.AddHandler("test", func(cmd *models.UpdateDashboardAlertsCommand) error { + return nil + }) + + dto.Dashboard = models.NewDashboard("Dash") + dto.Dashboard.SetId(3) + dto.User = &models.SignedInUser{UserId: 1} + _, err := service.ImportDashboard(dto) + So(provisioningValidated, ShouldBeTrue) + So(err, ShouldEqual, models.ErrDashboardCannotSaveProvisionedDashboard) }) }) diff --git a/pkg/services/dashboards/folder_service.go b/pkg/services/dashboards/folder_service.go index 66afa6306fb..b521b0e5213 100644 --- a/pkg/services/dashboards/folder_service.go +++ b/pkg/services/dashboards/folder_service.go @@ -10,8 +10,8 @@ import ( // FolderService service for operating on folders type FolderService interface { GetFolders(limit int) ([]*models.Folder, error) - GetFolderById(id int64) (*models.Folder, error) - GetFolderByUid(uid string) (*models.Folder, error) + GetFolderByID(id int64) (*models.Folder, error) + GetFolderByUID(uid string) (*models.Folder, error) CreateFolder(cmd *models.CreateFolderCommand) error UpdateFolder(uid string, cmd *models.UpdateFolderCommand) error DeleteFolder(uid string) (*models.Folder, error) @@ -57,7 +57,7 @@ func (dr *dashboardServiceImpl) GetFolders(limit int) ([]*models.Folder, error) return folders, nil } -func (dr *dashboardServiceImpl) GetFolderById(id int64) (*models.Folder, error) { +func (dr *dashboardServiceImpl) GetFolderByID(id int64) (*models.Folder, error) { query := models.GetDashboardQuery{OrgId: dr.orgId, Id: id} dashFolder, err := getFolder(query) @@ -76,7 +76,7 @@ func (dr *dashboardServiceImpl) GetFolderById(id int64) (*models.Folder, error) return dashToFolder(dashFolder), nil } -func (dr *dashboardServiceImpl) GetFolderByUid(uid string) (*models.Folder, error) { +func (dr *dashboardServiceImpl) GetFolderByUID(uid string) (*models.Folder, error) { query := models.GetDashboardQuery{OrgId: dr.orgId, Uid: uid} dashFolder, err := getFolder(query) @@ -104,7 +104,7 @@ func (dr *dashboardServiceImpl) CreateFolder(cmd *models.CreateFolderCommand) er User: dr.user, } - saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false) + saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false, false) if err != nil { return toFolderError(err) } @@ -141,7 +141,7 @@ func (dr *dashboardServiceImpl) UpdateFolder(existingUid string, cmd *models.Upd Overwrite: cmd.Overwrite, } - saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false) + saveDashboardCmd, err := dr.buildSaveDashboardCommand(dto, false, false) if err != nil { return toFolderError(err) } diff --git a/pkg/services/dashboards/folder_service_test.go b/pkg/services/dashboards/folder_service_test.go index 6357e84805a..4c9cecd3352 100644 --- a/pkg/services/dashboards/folder_service_test.go +++ b/pkg/services/dashboards/folder_service_test.go @@ -32,17 +32,18 @@ func TestFolderService(t *testing.T) { }) bus.AddHandler("test", func(cmd *models.ValidateDashboardBeforeSaveCommand) error { + cmd.Result = &models.ValidateDashboardBeforeSaveResult{} return models.ErrDashboardUpdateAccessDenied }) Convey("When get folder by id should return access denied error", func() { - _, err := service.GetFolderById(1) + _, err := service.GetFolderByID(1) So(err, ShouldNotBeNil) So(err, ShouldEqual, models.ErrFolderAccessDenied) }) Convey("When get folder by uid should return access denied error", func() { - _, err := service.GetFolderByUid("uid") + _, err := service.GetFolderByUID("uid") So(err, ShouldNotBeNil) So(err, ShouldEqual, models.ErrFolderAccessDenied) }) @@ -92,6 +93,7 @@ func TestFolderService(t *testing.T) { }) bus.AddHandler("test", func(cmd *models.ValidateDashboardBeforeSaveCommand) error { + cmd.Result = &models.ValidateDashboardBeforeSaveResult{} return nil }) @@ -108,11 +110,19 @@ func TestFolderService(t *testing.T) { return nil }) + provisioningValidated := false + + bus.AddHandler("test", func(query *models.IsDashboardProvisionedQuery) error { + provisioningValidated = true + return nil + }) + Convey("When creating folder should not return access denied error", func() { err := service.CreateFolder(&models.CreateFolderCommand{ Title: "Folder", }) So(err, ShouldBeNil) + So(provisioningValidated, ShouldBeFalse) }) Convey("When updating folder should not return access denied error", func() { @@ -121,6 +131,7 @@ func TestFolderService(t *testing.T) { Title: "Folder", }) So(err, ShouldBeNil) + So(provisioningValidated, ShouldBeFalse) }) Convey("When deleting folder by uid should not return access denied error", func() { @@ -147,14 +158,14 @@ func TestFolderService(t *testing.T) { }) Convey("When get folder by id should return folder", func() { - f, _ := service.GetFolderById(1) + f, _ := service.GetFolderByID(1) So(f.Id, ShouldEqual, dashFolder.Id) So(f.Uid, ShouldEqual, dashFolder.Uid) So(f.Title, ShouldEqual, dashFolder.Title) }) Convey("When get folder by uid should return folder", func() { - f, _ := service.GetFolderByUid("uid") + f, _ := service.GetFolderByUID("uid") So(f.Id, ShouldEqual, dashFolder.Id) So(f.Uid, ShouldEqual, dashFolder.Uid) So(f.Title, ShouldEqual, dashFolder.Title) diff --git a/pkg/services/datasources/cache.go b/pkg/services/datasources/cache.go new file mode 100644 index 00000000000..0cd2bae63b5 --- /dev/null +++ b/pkg/services/datasources/cache.go @@ -0,0 +1,53 @@ +package datasources + +import ( + "fmt" + "time" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/cache" +) + +type CacheService interface { + GetDatasource(datasourceID int64, user *m.SignedInUser, skipCache bool) (*m.DataSource, error) +} + +type CacheServiceImpl struct { + Bus bus.Bus `inject:""` + CacheService *cache.CacheService `inject:""` +} + +func init() { + registry.Register(®istry.Descriptor{ + Name: "DatasourceCacheService", + Instance: &CacheServiceImpl{}, + InitPriority: registry.Low, + }) +} + +func (dc *CacheServiceImpl) Init() error { + return nil +} + +func (dc *CacheServiceImpl) GetDatasource(datasourceID int64, user *m.SignedInUser, skipCache bool) (*m.DataSource, error) { + cacheKey := fmt.Sprintf("ds-%d", datasourceID) + + if !skipCache { + if cached, found := dc.CacheService.Get(cacheKey); found { + ds := cached.(*m.DataSource) + if ds.OrgId == user.OrgId { + return ds, nil + } + } + } + + query := m.GetDataSourceByIdQuery{Id: datasourceID, OrgId: user.OrgId} + if err := dc.Bus.Dispatch(&query); err != nil { + return nil, err + } + + dc.CacheService.Set(cacheKey, query.Result, time.Second*5) + return query.Result, nil +} diff --git a/pkg/services/guardian/guardian.go b/pkg/services/guardian/guardian.go index 811b38cac86..366bc90fc37 100644 --- a/pkg/services/guardian/guardian.go +++ b/pkg/services/guardian/guardian.go @@ -30,7 +30,7 @@ type dashboardGuardianImpl struct { dashId int64 orgId int64 acl []*m.DashboardAclInfoDTO - groups []*m.Team + teams []*m.TeamDTO log log.Logger } @@ -40,7 +40,7 @@ var New = func(dashId int64, orgId int64, user *m.SignedInUser) DashboardGuardia user: user, dashId: dashId, orgId: orgId, - log: log.New("guardians.dashboard"), + log: log.New("dashboard.permissions"), } } @@ -66,15 +66,30 @@ func (g *dashboardGuardianImpl) CanAdmin() (bool, error) { func (g *dashboardGuardianImpl) HasPermission(permission m.PermissionType) (bool, error) { if g.user.OrgRole == m.ROLE_ADMIN { - return true, nil + return g.logHasPermissionResult(permission, true, nil) } acl, err := g.GetAcl() if err != nil { - return false, err + return g.logHasPermissionResult(permission, false, err) } - return g.checkAcl(permission, acl) + result, err := g.checkAcl(permission, acl) + return g.logHasPermissionResult(permission, result, err) +} + +func (g *dashboardGuardianImpl) logHasPermissionResult(permission m.PermissionType, hasPermission bool, err error) (bool, error) { + if err != nil { + return hasPermission, err + } + + if hasPermission { + g.log.Debug("User granted access to execute action", "userId", g.user.UserId, "orgId", g.orgId, "uname", g.user.Login, "dashId", g.dashId, "action", permission) + } else { + g.log.Debug("User denied access to execute action", "userId", g.user.UserId, "orgId", g.orgId, "uname", g.user.Login, "dashId", g.dashId, "action", permission) + } + + return hasPermission, err } func (g *dashboardGuardianImpl) checkAcl(permission m.PermissionType, acl []*m.DashboardAclInfoDTO) (bool, error) { @@ -83,7 +98,7 @@ func (g *dashboardGuardianImpl) checkAcl(permission m.PermissionType, acl []*m.D for _, p := range acl { // user match - if !g.user.IsAnonymous { + if !g.user.IsAnonymous && p.UserId > 0 { if p.UserId == g.user.UserId && p.Permission >= permission { return true, nil } @@ -113,7 +128,7 @@ func (g *dashboardGuardianImpl) checkAcl(permission m.PermissionType, acl []*m.D return false, err } - // evalute team rules + // evaluate team rules for _, p := range acl { for _, ug := range teams { if ug.Id == p.TeamId && p.Permission >= permission { @@ -154,12 +169,7 @@ func (g *dashboardGuardianImpl) CheckPermissionBeforeUpdate(permission m.Permiss // validate overridden permissions to be higher for _, a := range acl { for _, existingPerm := range existingPermissions { - // handle default permissions - if existingPerm.DashboardId == -1 { - existingPerm.DashboardId = g.dashId - } - - if a.DashboardId == existingPerm.DashboardId { + if !existingPerm.Inherited { continue } @@ -173,7 +183,7 @@ func (g *dashboardGuardianImpl) CheckPermissionBeforeUpdate(permission m.Permiss return true, nil } - return g.checkAcl(permission, acl) + return g.checkAcl(permission, existingPermissions) } // GetAcl returns dashboard acl @@ -187,26 +197,19 @@ func (g *dashboardGuardianImpl) GetAcl() ([]*m.DashboardAclInfoDTO, error) { return nil, err } - for _, a := range query.Result { - // handle default permissions - if a.DashboardId == -1 { - a.DashboardId = g.dashId - } - } - g.acl = query.Result return g.acl, nil } -func (g *dashboardGuardianImpl) getTeams() ([]*m.Team, error) { - if g.groups != nil { - return g.groups, nil +func (g *dashboardGuardianImpl) getTeams() ([]*m.TeamDTO, error) { + if g.teams != nil { + return g.teams, nil } query := m.GetTeamsByUserQuery{OrgId: g.orgId, UserId: g.user.UserId} err := bus.Dispatch(&query) - g.groups = query.Result + g.teams = query.Result return query.Result, err } diff --git a/pkg/services/guardian/guardian_test.go b/pkg/services/guardian/guardian_test.go index bb7e6bd1a72..4704519b38d 100644 --- a/pkg/services/guardian/guardian_test.go +++ b/pkg/services/guardian/guardian_test.go @@ -2,710 +2,677 @@ package guardian import ( "fmt" + "runtime" "testing" - "github.com/grafana/grafana/pkg/bus" - m "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" ) -func TestGuardian(t *testing.T) { - Convey("Guardian permission tests", t, func() { - orgRoleScenario("Given user has admin org role", m.ROLE_ADMIN, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeTrue) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - - Convey("When trying to update permissions", func() { - Convey("With duplicate user permissions should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, UserId: 1, Permission: m.PERMISSION_VIEW}, - {OrgId: 1, DashboardId: 1, UserId: 1, Permission: m.PERMISSION_ADMIN}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianPermissionExists) - }) - - Convey("With duplicate team permissions should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, TeamId: 1, Permission: m.PERMISSION_VIEW}, - {OrgId: 1, DashboardId: 1, TeamId: 1, Permission: m.PERMISSION_ADMIN}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianPermissionExists) - }) - - Convey("With duplicate everyone with editor role permission should return error", func() { - r := m.ROLE_EDITOR - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, Role: &r, Permission: m.PERMISSION_VIEW}, - {OrgId: 1, DashboardId: 1, Role: &r, Permission: m.PERMISSION_ADMIN}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianPermissionExists) - }) - - Convey("With duplicate everyone with viewer role permission should return error", func() { - r := m.ROLE_VIEWER - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, Role: &r, Permission: m.PERMISSION_VIEW}, - {OrgId: 1, DashboardId: 1, Role: &r, Permission: m.PERMISSION_ADMIN}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianPermissionExists) - }) - - Convey("With everyone with admin role permission should return error", func() { - r := m.ROLE_ADMIN - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, Role: &r, Permission: m.PERMISSION_ADMIN}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianPermissionExists) - }) - }) - - Convey("Given default permissions", func() { - editor := m.ROLE_EDITOR - viewer := m.ROLE_VIEWER - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: -1, Role: &editor, Permission: m.PERMISSION_EDIT}, - {OrgId: 1, DashboardId: -1, Role: &viewer, Permission: m.PERMISSION_VIEW}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions without everyone with role editor can edit should be allowed", func() { - r := m.ROLE_VIEWER - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, Role: &r, Permission: m.PERMISSION_VIEW}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions without everyone with role viewer can view should be allowed", func() { - r := m.ROLE_EDITOR - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, Role: &r, Permission: m.PERMISSION_EDIT}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - }) - - Convey("Given parent folder has user admin permission", func() { - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, UserId: 1, Permission: m.PERMISSION_ADMIN}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with admin user permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_ADMIN}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - - Convey("When trying to update dashboard permissions with edit user permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_EDIT}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - - Convey("When trying to update dashboard permissions with view user permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - - Convey("Given parent folder has user edit permission", func() { - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, UserId: 1, Permission: m.PERMISSION_EDIT}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with admin user permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with edit user permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_EDIT}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - - Convey("When trying to update dashboard permissions with view user permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - - Convey("Given parent folder has user view permission", func() { - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, UserId: 1, Permission: m.PERMISSION_VIEW}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with admin user permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with edit user permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_EDIT}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with view user permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, UserId: 1, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - - Convey("Given parent folder has team admin permission", func() { - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, TeamId: 1, Permission: m.PERMISSION_ADMIN}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with admin team permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_ADMIN}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - - Convey("When trying to update dashboard permissions with edit team permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_EDIT}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - - Convey("When trying to update dashboard permissions with view team permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - - Convey("Given parent folder has team edit permission", func() { - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, TeamId: 1, Permission: m.PERMISSION_EDIT}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with admin team permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with edit team permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_EDIT}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - - Convey("When trying to update dashboard permissions with view team permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - - Convey("Given parent folder has team view permission", func() { - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, TeamId: 1, Permission: m.PERMISSION_VIEW}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with admin team permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with edit team permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_EDIT}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with view team permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, TeamId: 1, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - - Convey("Given parent folder has editor role with edit permission", func() { - r := m.ROLE_EDITOR - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, Role: &r, Permission: m.PERMISSION_EDIT}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with everyone with editor role can admin permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, Role: &r, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with everyone with editor role can edit permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, Role: &r, Permission: m.PERMISSION_EDIT}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - - Convey("When trying to update dashboard permissions with everyone with editor role can view permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, Role: &r, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - - Convey("Given parent folder has editor role with view permission", func() { - r := m.ROLE_EDITOR - existingPermissions := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 2, Role: &r, Permission: m.PERMISSION_VIEW}, - } - - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = existingPermissions - return nil - }) - - Convey("When trying to update dashboard permissions with everyone with viewer role can admin permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, Role: &r, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with everyone with viewer role can edit permission should be allowed", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, Role: &r, Permission: m.PERMISSION_EDIT}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeTrue) - }) - - Convey("When trying to update dashboard permissions with everyone with viewer role can view permission should return error", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 3, Role: &r, Permission: m.PERMISSION_VIEW}, - } - _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(err, ShouldEqual, ErrGuardianOverride) - }) - }) - }) - - orgRoleScenario("Given user has editor org role", m.ROLE_EDITOR, func(sc *scenarioContext) { - everyoneWithRoleScenario(m.ROLE_EDITOR, m.PERMISSION_ADMIN, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeTrue) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - everyoneWithRoleScenario(m.ROLE_EDITOR, m.PERMISSION_EDIT, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - everyoneWithRoleScenario(m.ROLE_EDITOR, m.PERMISSION_VIEW, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeTrue) - }) - - everyoneWithRoleScenario(m.ROLE_VIEWER, m.PERMISSION_ADMIN, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeFalse) - }) - - everyoneWithRoleScenario(m.ROLE_VIEWER, m.PERMISSION_EDIT, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeFalse) - }) - - everyoneWithRoleScenario(m.ROLE_VIEWER, m.PERMISSION_VIEW, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeFalse) - }) - - userWithPermissionScenario(m.PERMISSION_ADMIN, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeTrue) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - userWithPermissionScenario(m.PERMISSION_EDIT, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - userWithPermissionScenario(m.PERMISSION_VIEW, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeTrue) - }) - - teamWithPermissionScenario(m.PERMISSION_ADMIN, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeTrue) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - teamWithPermissionScenario(m.PERMISSION_EDIT, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - teamWithPermissionScenario(m.PERMISSION_VIEW, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeTrue) - }) - - Convey("When trying to update permissions should return false", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, UserId: 1, Permission: m.PERMISSION_VIEW}, - {OrgId: 1, DashboardId: 1, UserId: 1, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeFalse) - }) - }) - - orgRoleScenario("Given user has viewer org role", m.ROLE_VIEWER, func(sc *scenarioContext) { - everyoneWithRoleScenario(m.ROLE_EDITOR, m.PERMISSION_ADMIN, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeFalse) - }) - - everyoneWithRoleScenario(m.ROLE_EDITOR, m.PERMISSION_EDIT, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeFalse) - }) - - everyoneWithRoleScenario(m.ROLE_EDITOR, m.PERMISSION_VIEW, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeFalse) - }) - - everyoneWithRoleScenario(m.ROLE_VIEWER, m.PERMISSION_ADMIN, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeTrue) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - everyoneWithRoleScenario(m.ROLE_VIEWER, m.PERMISSION_EDIT, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - everyoneWithRoleScenario(m.ROLE_VIEWER, m.PERMISSION_VIEW, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeTrue) - }) - - userWithPermissionScenario(m.PERMISSION_ADMIN, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeTrue) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - userWithPermissionScenario(m.PERMISSION_EDIT, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeTrue) - So(canSave, ShouldBeTrue) - So(canView, ShouldBeTrue) - }) - - userWithPermissionScenario(m.PERMISSION_VIEW, sc, func(sc *scenarioContext) { - canAdmin, _ := sc.g.CanAdmin() - canEdit, _ := sc.g.CanEdit() - canSave, _ := sc.g.CanSave() - canView, _ := sc.g.CanView() - So(canAdmin, ShouldBeFalse) - So(canEdit, ShouldBeFalse) - So(canSave, ShouldBeFalse) - So(canView, ShouldBeTrue) - }) - - Convey("When trying to update permissions should return false", func() { - p := []*m.DashboardAcl{ - {OrgId: 1, DashboardId: 1, UserId: 1, Permission: m.PERMISSION_VIEW}, - {OrgId: 1, DashboardId: 1, UserId: 1, Permission: m.PERMISSION_ADMIN}, - } - ok, _ := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) - So(ok, ShouldBeFalse) - }) +var ( + orgID = int64(1) + defaultDashboardID = int64(-1) + dashboardID = int64(1) + parentFolderID = int64(2) + childDashboardID = int64(3) + userID = int64(1) + otherUserID = int64(2) + teamID = int64(1) + otherTeamID = int64(2) + adminRole = m.ROLE_ADMIN + editorRole = m.ROLE_EDITOR + viewerRole = m.ROLE_VIEWER +) + +func TestGuardianAdmin(t *testing.T) { + Convey("Guardian admin org role tests", t, func() { + orgRoleScenario("Given user has admin org role", t, m.ROLE_ADMIN, func(sc *scenarioContext) { + // dashboard has default permissions + sc.defaultPermissionScenario(USER, FULL_ACCESS) + + // dashboard has user with permission + sc.dashboardPermissionScenario(USER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(USER, m.PERMISSION_EDIT, FULL_ACCESS) + sc.dashboardPermissionScenario(USER, m.PERMISSION_VIEW, FULL_ACCESS) + + // dashboard has team with permission + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_EDIT, FULL_ACCESS) + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_VIEW, FULL_ACCESS) + + // dashboard has editor role with permission + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_EDIT, FULL_ACCESS) + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_VIEW, FULL_ACCESS) + + // dashboard has viewer role with permission + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_EDIT, FULL_ACCESS) + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_VIEW, FULL_ACCESS) + + // parent folder has user with permission + sc.parentFolderPermissionScenario(USER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(USER, m.PERMISSION_EDIT, FULL_ACCESS) + sc.parentFolderPermissionScenario(USER, m.PERMISSION_VIEW, FULL_ACCESS) + + // parent folder has team with permission + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_EDIT, FULL_ACCESS) + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_VIEW, FULL_ACCESS) + + // parent folder has editor role with permission + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_EDIT, FULL_ACCESS) + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_VIEW, FULL_ACCESS) + + // parent folder has viweer role with permission + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_EDIT, FULL_ACCESS) + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_VIEW, FULL_ACCESS) }) }) } -type scenarioContext struct { - g DashboardGuardian +func TestGuardianEditor(t *testing.T) { + Convey("Guardian editor org role tests", t, func() { + orgRoleScenario("Given user has editor org role", t, m.ROLE_EDITOR, func(sc *scenarioContext) { + // dashboard has default permissions + sc.defaultPermissionScenario(USER, EDITOR_ACCESS) + + // dashboard has user with permission + sc.dashboardPermissionScenario(USER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(USER, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.dashboardPermissionScenario(USER, m.PERMISSION_VIEW, CAN_VIEW) + + // dashboard has team with permission + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_VIEW, CAN_VIEW) + + // dashboard has editor role with permission + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // dashboard has viewer role with permission + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_ADMIN, NO_ACCESS) + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_EDIT, NO_ACCESS) + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_VIEW, NO_ACCESS) + + // parent folder has user with permission + sc.parentFolderPermissionScenario(USER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(USER, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.parentFolderPermissionScenario(USER, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // parent folder has team with permission + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // parent folder has editor role with permission + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // parent folder has viweer role with permission + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_ADMIN, NO_ACCESS) + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_EDIT, NO_ACCESS) + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_VIEW, NO_ACCESS) + }) + }) } -type scenarioFunc func(c *scenarioContext) +func TestGuardianViewer(t *testing.T) { + Convey("Guardian viewer org role tests", t, func() { + orgRoleScenario("Given user has viewer org role", t, m.ROLE_VIEWER, func(sc *scenarioContext) { + // dashboard has default permissions + sc.defaultPermissionScenario(USER, VIEWER_ACCESS) -func orgRoleScenario(desc string, role m.RoleType, fn scenarioFunc) { - user := &m.SignedInUser{ - UserId: 1, - OrgId: 1, - OrgRole: role, - } - guard := New(1, 1, user) - sc := &scenarioContext{ - g: guard, + // dashboard has user with permission + sc.dashboardPermissionScenario(USER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(USER, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.dashboardPermissionScenario(USER, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // dashboard has team with permission + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.dashboardPermissionScenario(TEAM, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // dashboard has editor role with permission + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_ADMIN, NO_ACCESS) + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_EDIT, NO_ACCESS) + sc.dashboardPermissionScenario(EDITOR, m.PERMISSION_VIEW, NO_ACCESS) + + // dashboard has viewer role with permission + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.dashboardPermissionScenario(VIEWER, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // parent folder has user with permission + sc.parentFolderPermissionScenario(USER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(USER, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.parentFolderPermissionScenario(USER, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // parent folder has team with permission + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.parentFolderPermissionScenario(TEAM, m.PERMISSION_VIEW, VIEWER_ACCESS) + + // parent folder has editor role with permission + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_ADMIN, NO_ACCESS) + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_EDIT, NO_ACCESS) + sc.parentFolderPermissionScenario(EDITOR, m.PERMISSION_VIEW, NO_ACCESS) + + // parent folder has viweer role with permission + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_ADMIN, FULL_ACCESS) + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_EDIT, EDITOR_ACCESS) + sc.parentFolderPermissionScenario(VIEWER, m.PERMISSION_VIEW, VIEWER_ACCESS) + }) + + apiKeyScenario("Given api key with viewer role", t, m.ROLE_VIEWER, func(sc *scenarioContext) { + // dashboard has default permissions + sc.defaultPermissionScenario(VIEWER, VIEWER_ACCESS) + }) + }) +} + +func (sc *scenarioContext) defaultPermissionScenario(pt permissionType, flag permissionFlags) { + _, callerFile, callerLine, _ := runtime.Caller(1) + sc.callerFile = callerFile + sc.callerLine = callerLine + existingPermissions := []*m.DashboardAclInfoDTO{ + toDto(newEditorRolePermission(defaultDashboardID, m.PERMISSION_EDIT)), + toDto(newViewerRolePermission(defaultDashboardID, m.PERMISSION_VIEW)), } - Convey(desc, func() { - fn(sc) + permissionScenario("and existing permissions is the default permissions (everyone with editor role can edit, everyone with viewer role can view)", dashboardID, sc, existingPermissions, func(sc *scenarioContext) { + sc.expectedFlags = flag + sc.verifyExpectedPermissionsFlags() + sc.verifyDuplicatePermissionsShouldNotBeAllowed() + sc.verifyUpdateDashboardPermissionsShouldBeAllowed(pt) + sc.verifyUpdateDashboardPermissionsShouldNotBeAllowed(pt) }) } -func permissionScenario(desc string, sc *scenarioContext, permissions []*m.DashboardAclInfoDTO, fn scenarioFunc) { - bus.ClearBusHandlers() +func (sc *scenarioContext) dashboardPermissionScenario(pt permissionType, permission m.PermissionType, flag permissionFlags) { + _, callerFile, callerLine, _ := runtime.Caller(1) + sc.callerFile = callerFile + sc.callerLine = callerLine + var existingPermissions []*m.DashboardAclInfoDTO - bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { - query.Result = permissions - return nil + switch pt { + case USER: + existingPermissions = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: dashboardID, UserId: userID, Permission: permission}} + case TEAM: + existingPermissions = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: dashboardID, TeamId: teamID, Permission: permission}} + case EDITOR: + existingPermissions = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: dashboardID, Role: &editorRole, Permission: permission}} + case VIEWER: + existingPermissions = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: dashboardID, Role: &viewerRole, Permission: permission}} + } + + permissionScenario(fmt.Sprintf("and %s has permission to %s dashboard", pt.String(), permission.String()), dashboardID, sc, existingPermissions, func(sc *scenarioContext) { + sc.expectedFlags = flag + sc.verifyExpectedPermissionsFlags() + sc.verifyDuplicatePermissionsShouldNotBeAllowed() + sc.verifyUpdateDashboardPermissionsShouldBeAllowed(pt) + sc.verifyUpdateDashboardPermissionsShouldNotBeAllowed(pt) }) +} - teams := []*m.Team{} +func (sc *scenarioContext) parentFolderPermissionScenario(pt permissionType, permission m.PermissionType, flag permissionFlags) { + _, callerFile, callerLine, _ := runtime.Caller(1) + sc.callerFile = callerFile + sc.callerLine = callerLine + var folderPermissionList []*m.DashboardAclInfoDTO - for _, p := range permissions { - if p.TeamId > 0 { - teams = append(teams, &m.Team{Id: p.TeamId}) + switch pt { + case USER: + folderPermissionList = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: parentFolderID, UserId: userID, Permission: permission, Inherited: true}} + case TEAM: + folderPermissionList = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: parentFolderID, TeamId: teamID, Permission: permission, Inherited: true}} + case EDITOR: + folderPermissionList = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: parentFolderID, Role: &editorRole, Permission: permission, Inherited: true}} + case VIEWER: + folderPermissionList = []*m.DashboardAclInfoDTO{{OrgId: orgID, DashboardId: parentFolderID, Role: &viewerRole, Permission: permission, Inherited: true}} + } + + permissionScenario(fmt.Sprintf("and parent folder has %s with permission to %s", pt.String(), permission.String()), childDashboardID, sc, folderPermissionList, func(sc *scenarioContext) { + sc.expectedFlags = flag + sc.verifyExpectedPermissionsFlags() + sc.verifyDuplicatePermissionsShouldNotBeAllowed() + sc.verifyUpdateChildDashboardPermissionsShouldBeAllowed(pt, permission) + sc.verifyUpdateChildDashboardPermissionsShouldNotBeAllowed(pt, permission) + sc.verifyUpdateChildDashboardPermissionsWithOverrideShouldBeAllowed(pt, permission) + sc.verifyUpdateChildDashboardPermissionsWithOverrideShouldNotBeAllowed(pt, permission) + }) +} + +func (sc *scenarioContext) verifyExpectedPermissionsFlags() { + canAdmin, _ := sc.g.CanAdmin() + canEdit, _ := sc.g.CanEdit() + canSave, _ := sc.g.CanSave() + canView, _ := sc.g.CanView() + + tc := fmt.Sprintf("should have permissions to %s", sc.expectedFlags.String()) + Convey(tc, func() { + var actualFlag permissionFlags + + if canAdmin { + actualFlag |= CAN_ADMIN } - } - bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error { - query.Result = teams - return nil - }) + if canEdit { + actualFlag |= CAN_EDIT + } - Convey(desc, func() { - fn(sc) + if canSave { + actualFlag |= CAN_SAVE + } + + if canView { + actualFlag |= CAN_VIEW + } + + if actualFlag.noAccess() { + actualFlag = NO_ACCESS + } + + if actualFlag&sc.expectedFlags != actualFlag { + sc.reportFailure(tc, sc.expectedFlags.String(), actualFlag.String()) + } + + sc.reportSuccess() }) } -func userWithPermissionScenario(permission m.PermissionType, sc *scenarioContext, fn scenarioFunc) { - p := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 1, UserId: 1, Permission: permission}, +func (sc *scenarioContext) verifyDuplicatePermissionsShouldNotBeAllowed() { + if !sc.expectedFlags.canAdmin() { + return } - permissionScenario(fmt.Sprintf("and user has permission to %s item", permission), sc, p, fn) + + tc := "When updating dashboard permissions with duplicate permission for user should not be allowed" + Convey(tc, func() { + p := []*m.DashboardAcl{ + newDefaultUserPermission(dashboardID, m.PERMISSION_VIEW), + newDefaultUserPermission(dashboardID, m.PERMISSION_ADMIN), + } + sc.updatePermissions = p + _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) + + if err != ErrGuardianPermissionExists { + sc.reportFailure(tc, ErrGuardianPermissionExists, err) + } + sc.reportSuccess() + }) + + tc = "When updating dashboard permissions with duplicate permission for team should not be allowed" + Convey(tc, func() { + p := []*m.DashboardAcl{ + newDefaultTeamPermission(dashboardID, m.PERMISSION_VIEW), + newDefaultTeamPermission(dashboardID, m.PERMISSION_ADMIN), + } + sc.updatePermissions = p + _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) + + if err != ErrGuardianPermissionExists { + sc.reportFailure(tc, ErrGuardianPermissionExists, err) + } + sc.reportSuccess() + }) + + tc = "When updating dashboard permissions with duplicate permission for editor role should not be allowed" + Convey(tc, func() { + p := []*m.DashboardAcl{ + newEditorRolePermission(dashboardID, m.PERMISSION_VIEW), + newEditorRolePermission(dashboardID, m.PERMISSION_ADMIN), + } + sc.updatePermissions = p + _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) + + if err != ErrGuardianPermissionExists { + sc.reportFailure(tc, ErrGuardianPermissionExists, err) + } + sc.reportSuccess() + }) + + tc = "When updating dashboard permissions with duplicate permission for viewer role should not be allowed" + Convey(tc, func() { + p := []*m.DashboardAcl{ + newViewerRolePermission(dashboardID, m.PERMISSION_VIEW), + newViewerRolePermission(dashboardID, m.PERMISSION_ADMIN), + } + sc.updatePermissions = p + _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) + + if err != ErrGuardianPermissionExists { + sc.reportFailure(tc, ErrGuardianPermissionExists, err) + } + sc.reportSuccess() + }) + + tc = "When updating dashboard permissions with duplicate permission for admin role should not be allowed" + Convey(tc, func() { + p := []*m.DashboardAcl{ + newAdminRolePermission(dashboardID, m.PERMISSION_ADMIN), + } + sc.updatePermissions = p + _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, p) + + if err != ErrGuardianPermissionExists { + sc.reportFailure(tc, ErrGuardianPermissionExists, err) + } + sc.reportSuccess() + }) } -func teamWithPermissionScenario(permission m.PermissionType, sc *scenarioContext, fn scenarioFunc) { - p := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 1, TeamId: 1, Permission: permission}, +func (sc *scenarioContext) verifyUpdateDashboardPermissionsShouldBeAllowed(pt permissionType) { + if !sc.expectedFlags.canAdmin() { + return + } + + for _, p := range []m.PermissionType{m.PERMISSION_ADMIN, m.PERMISSION_EDIT, m.PERMISSION_VIEW} { + tc := fmt.Sprintf("When updating dashboard permissions with %s permissions should be allowed", p.String()) + + Convey(tc, func() { + permissionList := []*m.DashboardAcl{} + switch pt { + case USER: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(dashboardID, p), + newViewerRolePermission(dashboardID, p), + newCustomUserPermission(dashboardID, otherUserID, p), + newDefaultTeamPermission(dashboardID, p), + } + case TEAM: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(dashboardID, p), + newViewerRolePermission(dashboardID, p), + newDefaultUserPermission(dashboardID, p), + newCustomTeamPermission(dashboardID, otherTeamID, p), + } + case EDITOR, VIEWER: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(dashboardID, p), + newViewerRolePermission(dashboardID, p), + newDefaultUserPermission(dashboardID, p), + newDefaultTeamPermission(dashboardID, p), + } + } + + sc.updatePermissions = permissionList + ok, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, permissionList) + + if err != nil { + sc.reportFailure(tc, nil, err) + } + if !ok { + sc.reportFailure(tc, false, true) + } + sc.reportSuccess() + }) } - permissionScenario(fmt.Sprintf("and team has permission to %s item", permission), sc, p, fn) } -func everyoneWithRoleScenario(role m.RoleType, permission m.PermissionType, sc *scenarioContext, fn scenarioFunc) { - p := []*m.DashboardAclInfoDTO{ - {OrgId: 1, DashboardId: 1, UserId: -1, Role: &role, Permission: permission}, +func (sc *scenarioContext) verifyUpdateDashboardPermissionsShouldNotBeAllowed(pt permissionType) { + if sc.expectedFlags.canAdmin() { + return + } + + for _, p := range []m.PermissionType{m.PERMISSION_ADMIN, m.PERMISSION_EDIT, m.PERMISSION_VIEW} { + tc := fmt.Sprintf("When updating dashboard permissions with %s permissions should NOT be allowed", p.String()) + + Convey(tc, func() { + permissionList := []*m.DashboardAcl{ + newEditorRolePermission(dashboardID, p), + newViewerRolePermission(dashboardID, p), + } + switch pt { + case USER: + permissionList = append(permissionList, []*m.DashboardAcl{ + newCustomUserPermission(dashboardID, otherUserID, p), + newDefaultTeamPermission(dashboardID, p), + }...) + case TEAM: + permissionList = append(permissionList, []*m.DashboardAcl{ + newDefaultUserPermission(dashboardID, p), + newCustomTeamPermission(dashboardID, otherTeamID, p), + }...) + } + + sc.updatePermissions = permissionList + ok, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, permissionList) + + if err != nil { + sc.reportFailure(tc, nil, err) + } + if ok { + sc.reportFailure(tc, true, false) + } + sc.reportSuccess() + }) + } +} + +func (sc *scenarioContext) verifyUpdateChildDashboardPermissionsShouldBeAllowed(pt permissionType, parentFolderPermission m.PermissionType) { + if !sc.expectedFlags.canAdmin() { + return + } + + for _, p := range []m.PermissionType{m.PERMISSION_ADMIN, m.PERMISSION_EDIT, m.PERMISSION_VIEW} { + tc := fmt.Sprintf("When updating child dashboard permissions with %s permissions should be allowed", p.String()) + + Convey(tc, func() { + permissionList := []*m.DashboardAcl{} + switch pt { + case USER: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + newViewerRolePermission(childDashboardID, p), + newCustomUserPermission(childDashboardID, otherUserID, p), + newDefaultTeamPermission(childDashboardID, p), + } + case TEAM: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + newViewerRolePermission(childDashboardID, p), + newDefaultUserPermission(childDashboardID, p), + newCustomTeamPermission(childDashboardID, otherTeamID, p), + } + case EDITOR: + permissionList = []*m.DashboardAcl{ + newViewerRolePermission(childDashboardID, p), + newDefaultUserPermission(childDashboardID, p), + newDefaultTeamPermission(childDashboardID, p), + } + + // permission to update is higher than parent folder permission + if p > parentFolderPermission { + permissionList = append(permissionList, newEditorRolePermission(childDashboardID, p)) + } + case VIEWER: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + newDefaultUserPermission(childDashboardID, p), + newDefaultTeamPermission(childDashboardID, p), + } + + // permission to update is higher than parent folder permission + if p > parentFolderPermission { + permissionList = append(permissionList, newViewerRolePermission(childDashboardID, p)) + } + } + + sc.updatePermissions = permissionList + ok, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, permissionList) + + if err != nil { + sc.reportFailure(tc, nil, err) + } + if !ok { + sc.reportFailure(tc, false, true) + } + sc.reportSuccess() + }) + } +} + +func (sc *scenarioContext) verifyUpdateChildDashboardPermissionsShouldNotBeAllowed(pt permissionType, parentFolderPermission m.PermissionType) { + if sc.expectedFlags.canAdmin() { + return + } + + for _, p := range []m.PermissionType{m.PERMISSION_ADMIN, m.PERMISSION_EDIT, m.PERMISSION_VIEW} { + tc := fmt.Sprintf("When updating child dashboard permissions with %s permissions should NOT be allowed", p.String()) + + Convey(tc, func() { + permissionList := []*m.DashboardAcl{} + switch pt { + case USER: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + newViewerRolePermission(childDashboardID, p), + newCustomUserPermission(childDashboardID, otherUserID, p), + newDefaultTeamPermission(childDashboardID, p), + } + case TEAM: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + newViewerRolePermission(childDashboardID, p), + newDefaultUserPermission(childDashboardID, p), + newCustomTeamPermission(childDashboardID, otherTeamID, p), + } + case EDITOR: + permissionList = []*m.DashboardAcl{ + newViewerRolePermission(childDashboardID, p), + newDefaultUserPermission(childDashboardID, p), + newDefaultTeamPermission(childDashboardID, p), + } + + // perminssion to update is higher than parent folder permission + if p > parentFolderPermission { + permissionList = append(permissionList, newEditorRolePermission(childDashboardID, p)) + } + case VIEWER: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + newDefaultUserPermission(childDashboardID, p), + newDefaultTeamPermission(childDashboardID, p), + } + + // perminssion to update is higher than parent folder permission + if p > parentFolderPermission { + permissionList = append(permissionList, newViewerRolePermission(childDashboardID, p)) + } + } + + sc.updatePermissions = permissionList + ok, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, permissionList) + + if err != nil { + sc.reportFailure(tc, nil, err) + } + if ok { + sc.reportFailure(tc, true, false) + } + sc.reportSuccess() + }) + } +} + +func (sc *scenarioContext) verifyUpdateChildDashboardPermissionsWithOverrideShouldBeAllowed(pt permissionType, parentFolderPermission m.PermissionType) { + if !sc.expectedFlags.canAdmin() { + return + } + + for _, p := range []m.PermissionType{m.PERMISSION_ADMIN, m.PERMISSION_EDIT, m.PERMISSION_VIEW} { + // perminssion to update is higher tban parent folder permission + if p > parentFolderPermission { + continue + } + + tc := fmt.Sprintf("When updating child dashboard permissions overriding parent %s permission with %s permission should NOT be allowed", pt.String(), p.String()) + + Convey(tc, func() { + permissionList := []*m.DashboardAcl{} + switch pt { + case USER: + permissionList = []*m.DashboardAcl{ + newDefaultUserPermission(childDashboardID, p), + } + case TEAM: + permissionList = []*m.DashboardAcl{ + newDefaultTeamPermission(childDashboardID, p), + } + case EDITOR: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + } + case VIEWER: + permissionList = []*m.DashboardAcl{ + newViewerRolePermission(childDashboardID, p), + } + } + + sc.updatePermissions = permissionList + _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, permissionList) + + if err != ErrGuardianOverride { + sc.reportFailure(tc, ErrGuardianOverride, err) + } + sc.reportSuccess() + }) + } +} + +func (sc *scenarioContext) verifyUpdateChildDashboardPermissionsWithOverrideShouldNotBeAllowed(pt permissionType, parentFolderPermission m.PermissionType) { + if !sc.expectedFlags.canAdmin() { + return + } + + for _, p := range []m.PermissionType{m.PERMISSION_ADMIN, m.PERMISSION_EDIT, m.PERMISSION_VIEW} { + // perminssion to update is lower than/equal parent folder permission + if p <= parentFolderPermission { + continue + } + + tc := fmt.Sprintf("When updating child dashboard permissions overriding parent %s permission with %s permission should be allowed", pt.String(), p.String()) + + Convey(tc, func() { + permissionList := []*m.DashboardAcl{} + switch pt { + case USER: + permissionList = []*m.DashboardAcl{ + newDefaultUserPermission(childDashboardID, p), + } + case TEAM: + permissionList = []*m.DashboardAcl{ + newDefaultTeamPermission(childDashboardID, p), + } + case EDITOR: + permissionList = []*m.DashboardAcl{ + newEditorRolePermission(childDashboardID, p), + } + case VIEWER: + permissionList = []*m.DashboardAcl{ + newViewerRolePermission(childDashboardID, p), + } + } + + _, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, permissionList) + if err != nil { + sc.reportFailure(tc, nil, err) + } + sc.updatePermissions = permissionList + ok, err := sc.g.CheckPermissionBeforeUpdate(m.PERMISSION_ADMIN, permissionList) + + if err != nil { + sc.reportFailure(tc, nil, err) + } + if !ok { + sc.reportFailure(tc, false, true) + } + sc.reportSuccess() + }) } - permissionScenario(fmt.Sprintf("and everyone with %s role can %s item", role, permission), sc, p, fn) } diff --git a/pkg/services/guardian/guardian_util_test.go b/pkg/services/guardian/guardian_util_test.go new file mode 100644 index 00000000000..d85548ecb8c --- /dev/null +++ b/pkg/services/guardian/guardian_util_test.go @@ -0,0 +1,277 @@ +package guardian + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +type scenarioContext struct { + t *testing.T + orgRoleScenario string + permissionScenario string + g DashboardGuardian + givenUser *m.SignedInUser + givenDashboardID int64 + givenPermissions []*m.DashboardAclInfoDTO + givenTeams []*m.TeamDTO + updatePermissions []*m.DashboardAcl + expectedFlags permissionFlags + callerFile string + callerLine int +} + +type scenarioFunc func(c *scenarioContext) + +func orgRoleScenario(desc string, t *testing.T, role m.RoleType, fn scenarioFunc) { + user := &m.SignedInUser{ + UserId: userID, + OrgId: orgID, + OrgRole: role, + } + guard := New(dashboardID, orgID, user) + sc := &scenarioContext{ + t: t, + orgRoleScenario: desc, + givenUser: user, + givenDashboardID: dashboardID, + g: guard, + } + + Convey(desc, func() { + fn(sc) + }) +} + +func apiKeyScenario(desc string, t *testing.T, role m.RoleType, fn scenarioFunc) { + user := &m.SignedInUser{ + UserId: 0, + OrgId: orgID, + OrgRole: role, + ApiKeyId: 10, + } + guard := New(dashboardID, orgID, user) + sc := &scenarioContext{ + t: t, + orgRoleScenario: desc, + givenUser: user, + givenDashboardID: dashboardID, + g: guard, + } + + Convey(desc, func() { + fn(sc) + }) +} + +func permissionScenario(desc string, dashboardID int64, sc *scenarioContext, permissions []*m.DashboardAclInfoDTO, fn scenarioFunc) { + bus.ClearBusHandlers() + + bus.AddHandler("test", func(query *m.GetDashboardAclInfoListQuery) error { + if query.OrgId != sc.givenUser.OrgId { + sc.reportFailure("Invalid organization id for GetDashboardAclInfoListQuery", sc.givenUser.OrgId, query.OrgId) + } + if query.DashboardId != sc.givenDashboardID { + sc.reportFailure("Invalid dashboard id for GetDashboardAclInfoListQuery", sc.givenDashboardID, query.DashboardId) + } + + query.Result = permissions + return nil + }) + + teams := []*m.TeamDTO{} + + for _, p := range permissions { + if p.TeamId > 0 { + teams = append(teams, &m.TeamDTO{Id: p.TeamId}) + } + } + + bus.AddHandler("test", func(query *m.GetTeamsByUserQuery) error { + if query.OrgId != sc.givenUser.OrgId { + sc.reportFailure("Invalid organization id for GetTeamsByUserQuery", sc.givenUser.OrgId, query.OrgId) + } + if query.UserId != sc.givenUser.UserId { + sc.reportFailure("Invalid user id for GetTeamsByUserQuery", sc.givenUser.UserId, query.UserId) + } + + query.Result = teams + return nil + }) + + sc.permissionScenario = desc + sc.g = New(dashboardID, sc.givenUser.OrgId, sc.givenUser) + sc.givenDashboardID = dashboardID + sc.givenPermissions = permissions + sc.givenTeams = teams + + Convey(desc, func() { + fn(sc) + }) +} + +type permissionType uint8 + +const ( + USER permissionType = 1 << iota + TEAM + EDITOR + VIEWER +) + +func (p permissionType) String() string { + names := map[uint8]string{ + uint8(USER): "user", + uint8(TEAM): "team", + uint8(EDITOR): "editor role", + uint8(VIEWER): "viewer role", + } + return names[uint8(p)] +} + +type permissionFlags uint8 + +const ( + NO_ACCESS permissionFlags = 1 << iota + CAN_ADMIN + CAN_EDIT + CAN_SAVE + CAN_VIEW + FULL_ACCESS = CAN_ADMIN | CAN_EDIT | CAN_SAVE | CAN_VIEW + EDITOR_ACCESS = CAN_EDIT | CAN_SAVE | CAN_VIEW + VIEWER_ACCESS = CAN_VIEW +) + +func (flag permissionFlags) canAdmin() bool { + return flag&CAN_ADMIN != 0 +} + +func (flag permissionFlags) canEdit() bool { + return flag&CAN_EDIT != 0 +} + +func (flag permissionFlags) canSave() bool { + return flag&CAN_SAVE != 0 +} + +func (flag permissionFlags) canView() bool { + return flag&CAN_VIEW != 0 +} + +func (flag permissionFlags) noAccess() bool { + return flag&(CAN_ADMIN|CAN_EDIT|CAN_SAVE|CAN_VIEW) == 0 +} + +func (f permissionFlags) String() string { + r := []string{} + + if f.canAdmin() { + r = append(r, "admin") + } + + if f.canEdit() { + r = append(r, "edit") + } + + if f.canSave() { + r = append(r, "save") + } + + if f.canView() { + r = append(r, "view") + } + + if f.noAccess() { + r = append(r, "") + } + + return strings.Join(r[:], ", ") +} + +func (sc *scenarioContext) reportSuccess() { + So(true, ShouldBeTrue) +} + +func (sc *scenarioContext) reportFailure(desc string, expected interface{}, actual interface{}) { + var buf bytes.Buffer + buf.WriteString("\n") + buf.WriteString(sc.orgRoleScenario) + buf.WriteString(" ") + buf.WriteString(sc.permissionScenario) + buf.WriteString("\n ") + buf.WriteString(desc) + buf.WriteString("\n") + buf.WriteString(fmt.Sprintf("Source test: %s:%d\n", sc.callerFile, sc.callerLine)) + buf.WriteString(fmt.Sprintf("Expected: %v\n", expected)) + buf.WriteString(fmt.Sprintf("Actual: %v\n", actual)) + buf.WriteString("Context:") + buf.WriteString(fmt.Sprintf("\n Given user: orgRole=%s, id=%d, orgId=%d", sc.givenUser.OrgRole, sc.givenUser.UserId, sc.givenUser.OrgId)) + buf.WriteString(fmt.Sprintf("\n Given dashboard id: %d", sc.givenDashboardID)) + + for i, p := range sc.givenPermissions { + r := "" + if p.Role != nil { + r = string(*p.Role) + } + buf.WriteString(fmt.Sprintf("\n Given permission (%d): dashboardId=%d, userId=%d, teamId=%d, role=%v, permission=%s", i, p.DashboardId, p.UserId, p.TeamId, r, p.Permission.String())) + } + + for i, t := range sc.givenTeams { + buf.WriteString(fmt.Sprintf("\n Given team (%d): id=%d", i, t.Id)) + } + + for i, p := range sc.updatePermissions { + r := "" + if p.Role != nil { + r = string(*p.Role) + } + buf.WriteString(fmt.Sprintf("\n Update permission (%d): dashboardId=%d, userId=%d, teamId=%d, role=%v, permission=%s", i, p.DashboardId, p.UserId, p.TeamId, r, p.Permission.String())) + } + + sc.t.Fatalf(buf.String()) +} + +func newCustomUserPermission(dashboardID int64, userID int64, permission m.PermissionType) *m.DashboardAcl { + return &m.DashboardAcl{OrgId: orgID, DashboardId: dashboardID, UserId: userID, Permission: permission} +} + +func newDefaultUserPermission(dashboardID int64, permission m.PermissionType) *m.DashboardAcl { + return newCustomUserPermission(dashboardID, userID, permission) +} + +func newCustomTeamPermission(dashboardID int64, teamID int64, permission m.PermissionType) *m.DashboardAcl { + return &m.DashboardAcl{OrgId: orgID, DashboardId: dashboardID, TeamId: teamID, Permission: permission} +} + +func newDefaultTeamPermission(dashboardID int64, permission m.PermissionType) *m.DashboardAcl { + return newCustomTeamPermission(dashboardID, teamID, permission) +} + +func newAdminRolePermission(dashboardID int64, permission m.PermissionType) *m.DashboardAcl { + return &m.DashboardAcl{OrgId: orgID, DashboardId: dashboardID, Role: &adminRole, Permission: permission} +} + +func newEditorRolePermission(dashboardID int64, permission m.PermissionType) *m.DashboardAcl { + return &m.DashboardAcl{OrgId: orgID, DashboardId: dashboardID, Role: &editorRole, Permission: permission} +} + +func newViewerRolePermission(dashboardID int64, permission m.PermissionType) *m.DashboardAcl { + return &m.DashboardAcl{OrgId: orgID, DashboardId: dashboardID, Role: &viewerRole, Permission: permission} +} + +func toDto(acl *m.DashboardAcl) *m.DashboardAclInfoDTO { + return &m.DashboardAclInfoDTO{ + OrgId: acl.OrgId, + DashboardId: acl.DashboardId, + UserId: acl.UserId, + TeamId: acl.TeamId, + Role: acl.Role, + Permission: acl.Permission, + PermissionName: acl.Permission.String(), + } +} diff --git a/pkg/services/hooks/hooks.go b/pkg/services/hooks/hooks.go new file mode 100644 index 00000000000..c51650cf6c9 --- /dev/null +++ b/pkg/services/hooks/hooks.go @@ -0,0 +1,30 @@ +package hooks + +import ( + "github.com/grafana/grafana/pkg/api/dtos" + "github.com/grafana/grafana/pkg/registry" +) + +type IndexDataHook func(indexData *dtos.IndexViewData) + +type HooksService struct { + indexDataHooks []IndexDataHook +} + +func init() { + registry.RegisterService(&HooksService{}) +} + +func (srv *HooksService) Init() error { + return nil +} + +func (srv *HooksService) AddIndexDataHook(hook IndexDataHook) { + srv.indexDataHooks = append(srv.indexDataHooks, hook) +} + +func (srv *HooksService) RunIndexDataHooks(indexData *dtos.IndexViewData) { + for _, hook := range srv.indexDataHooks { + hook(indexData) + } +} diff --git a/pkg/services/notifications/mailer.go b/pkg/services/notifications/mailer.go index 7fbf39ee41d..4730ef7f0f1 100644 --- a/pkg/services/notifications/mailer.go +++ b/pkg/services/notifications/mailer.go @@ -7,51 +7,18 @@ package notifications import ( "bytes" "crypto/tls" - "errors" "fmt" "html/template" "net" "strconv" - "strings" - "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" - "gopkg.in/gomail.v2" + gomail "gopkg.in/mail.v2" ) -var mailQueue chan *Message - -func initMailQueue() { - mailQueue = make(chan *Message, 10) - go processMailQueue() -} - -func processMailQueue() { - for { - select { - case msg := <-mailQueue: - num, err := send(msg) - tos := strings.Join(msg.To, "; ") - info := "" - if err != nil { - if len(msg.Info) > 0 { - info = ", info: " + msg.Info - } - log.Error(4, fmt.Sprintf("Async sent email %d succeed, not send emails: %s%s err: %s", num, tos, info, err)) - } else { - log.Trace(fmt.Sprintf("Async sent email %d succeed, sent emails: %s%s", num, tos, info)) - } - } - } -} - -var addToMailQueue = func(msg *Message) { - mailQueue <- msg -} - -func send(msg *Message) (int, error) { - dialer, err := createDialer() +func (ns *NotificationService) send(msg *Message) (int, error) { + dialer, err := ns.createDialer() if err != nil { return 0, err } @@ -75,8 +42,8 @@ func send(msg *Message) (int, error) { return len(msg.To), nil } -func createDialer() (*gomail.Dialer, error) { - host, port, err := net.SplitHostPort(setting.Smtp.Host) +func (ns *NotificationService) createDialer() (*gomail.Dialer, error) { + host, port, err := net.SplitHostPort(ns.Cfg.Smtp.Host) if err != nil { return nil, err @@ -87,30 +54,31 @@ func createDialer() (*gomail.Dialer, error) { } tlsconfig := &tls.Config{ - InsecureSkipVerify: setting.Smtp.SkipVerify, + InsecureSkipVerify: ns.Cfg.Smtp.SkipVerify, ServerName: host, } - if setting.Smtp.CertFile != "" { - cert, err := tls.LoadX509KeyPair(setting.Smtp.CertFile, setting.Smtp.KeyFile) + if ns.Cfg.Smtp.CertFile != "" { + cert, err := tls.LoadX509KeyPair(ns.Cfg.Smtp.CertFile, ns.Cfg.Smtp.KeyFile) if err != nil { return nil, fmt.Errorf("Could not load cert or key file. error: %v", err) } tlsconfig.Certificates = []tls.Certificate{cert} } - d := gomail.NewDialer(host, iPort, setting.Smtp.User, setting.Smtp.Password) + d := gomail.NewDialer(host, iPort, ns.Cfg.Smtp.User, ns.Cfg.Smtp.Password) d.TLSConfig = tlsconfig - if setting.Smtp.EhloIdentity != "" { - d.LocalName = setting.Smtp.EhloIdentity + + if ns.Cfg.Smtp.EhloIdentity != "" { + d.LocalName = ns.Cfg.Smtp.EhloIdentity } else { d.LocalName = setting.InstanceName } return d, nil } -func buildEmailMessage(cmd *m.SendEmailCommand) (*Message, error) { - if !setting.Smtp.Enabled { +func (ns *NotificationService) buildEmailMessage(cmd *m.SendEmailCommand) (*Message, error) { + if !ns.Cfg.Smtp.Enabled { return nil, m.ErrSmtpNotEnabled } @@ -135,7 +103,7 @@ func buildEmailMessage(cmd *m.SendEmailCommand) (*Message, error) { subjectText, hasSubject := subjectData["value"] if !hasSubject { - return nil, errors.New(fmt.Sprintf("Missing subject in Template %s", cmd.Template)) + return nil, fmt.Errorf("Missing subject in Template %s", cmd.Template) } subjectTmpl, err := template.New("subject").Parse(subjectText.(string)) @@ -154,7 +122,7 @@ func buildEmailMessage(cmd *m.SendEmailCommand) (*Message, error) { return &Message{ To: cmd.To, - From: fmt.Sprintf("%s <%s>", setting.Smtp.FromName, setting.Smtp.FromAddress), + From: fmt.Sprintf("%s <%s>", ns.Cfg.Smtp.FromName, ns.Cfg.Smtp.FromAddress), Subject: subject, Body: buffer.String(), EmbededFiles: cmd.EmbededFiles, diff --git a/pkg/services/notifications/notifications.go b/pkg/services/notifications/notifications.go index 25eb2b5936a..769fdd06fd0 100644 --- a/pkg/services/notifications/notifications.go +++ b/pkg/services/notifications/notifications.go @@ -7,11 +7,13 @@ import ( "html/template" "net/url" "path/filepath" + "strings" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/events" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -21,33 +23,46 @@ var tmplResetPassword = "reset_password.html" var tmplSignUpStarted = "signup_started.html" var tmplWelcomeOnSignUp = "welcome_on_signup.html" -func Init() error { - initMailQueue() - initWebhookQueue() +func init() { + registry.RegisterService(&NotificationService{}) +} - bus.AddHandler("email", sendResetPasswordEmail) - bus.AddHandler("email", validateResetPasswordCode) - bus.AddHandler("email", sendEmailCommandHandler) +type NotificationService struct { + Bus bus.Bus `inject:""` + Cfg *setting.Cfg `inject:""` - bus.AddCtxHandler("email", sendEmailCommandHandlerSync) + mailQueue chan *Message + webhookQueue chan *Webhook + log log.Logger +} - bus.AddCtxHandler("webhook", SendWebhookSync) +func (ns *NotificationService) Init() error { + ns.log = log.New("notifications") + ns.mailQueue = make(chan *Message, 10) + ns.webhookQueue = make(chan *Webhook, 10) - bus.AddEventListener(signUpStartedHandler) - bus.AddEventListener(signUpCompletedHandler) + ns.Bus.AddHandler(ns.sendResetPasswordEmail) + ns.Bus.AddHandler(ns.validateResetPasswordCode) + ns.Bus.AddHandler(ns.sendEmailCommandHandler) + + ns.Bus.AddHandlerCtx(ns.sendEmailCommandHandlerSync) + ns.Bus.AddHandlerCtx(ns.SendWebhookSync) + + ns.Bus.AddEventListener(ns.signUpStartedHandler) + ns.Bus.AddEventListener(ns.signUpCompletedHandler) mailTemplates = template.New("name") mailTemplates.Funcs(template.FuncMap{ "Subject": subjectTemplateFunc, }) - templatePattern := filepath.Join(setting.StaticRootPath, setting.Smtp.TemplatesPattern) + templatePattern := filepath.Join(setting.StaticRootPath, ns.Cfg.Smtp.TemplatesPattern) _, err := mailTemplates.ParseGlob(templatePattern) if err != nil { return err } - if !util.IsEmail(setting.Smtp.FromAddress) { + if !util.IsEmail(ns.Cfg.Smtp.FromAddress) { return errors.New("Invalid email address for SMTP from_address config") } @@ -58,14 +73,42 @@ func Init() error { return nil } -func SendWebhookSync(ctx context.Context, cmd *m.SendWebhookSync) error { - return sendWebRequestSync(ctx, &Webhook{ - Url: cmd.Url, - User: cmd.User, - Password: cmd.Password, - Body: cmd.Body, - HttpMethod: cmd.HttpMethod, - HttpHeader: cmd.HttpHeader, +func (ns *NotificationService) Run(ctx context.Context) error { + for { + select { + case webhook := <-ns.webhookQueue: + err := ns.sendWebRequestSync(context.Background(), webhook) + + if err != nil { + ns.log.Error("Failed to send webrequest ", "error", err) + } + case msg := <-ns.mailQueue: + num, err := ns.send(msg) + tos := strings.Join(msg.To, "; ") + info := "" + if err != nil { + if len(msg.Info) > 0 { + info = ", info: " + msg.Info + } + ns.log.Error(fmt.Sprintf("Async sent email %d succeed, not send emails: %s%s err: %s", num, tos, info, err)) + } else { + ns.log.Debug(fmt.Sprintf("Async sent email %d succeed, sent emails: %s%s", num, tos, info)) + } + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (ns *NotificationService) SendWebhookSync(ctx context.Context, cmd *m.SendWebhookSync) error { + return ns.sendWebRequestSync(ctx, &Webhook{ + Url: cmd.Url, + User: cmd.User, + Password: cmd.Password, + Body: cmd.Body, + HttpMethod: cmd.HttpMethod, + HttpHeader: cmd.HttpHeader, + ContentType: cmd.ContentType, }) } @@ -74,8 +117,8 @@ func subjectTemplateFunc(obj map[string]interface{}, value string) string { return "" } -func sendEmailCommandHandlerSync(ctx context.Context, cmd *m.SendEmailCommandSync) error { - message, err := buildEmailMessage(&m.SendEmailCommand{ +func (ns *NotificationService) sendEmailCommandHandlerSync(ctx context.Context, cmd *m.SendEmailCommandSync) error { + message, err := ns.buildEmailMessage(&m.SendEmailCommand{ Data: cmd.Data, Info: cmd.Info, Template: cmd.Template, @@ -88,25 +131,23 @@ func sendEmailCommandHandlerSync(ctx context.Context, cmd *m.SendEmailCommandSyn return err } - _, err = send(message) - + _, err = ns.send(message) return err } -func sendEmailCommandHandler(cmd *m.SendEmailCommand) error { - message, err := buildEmailMessage(cmd) +func (ns *NotificationService) sendEmailCommandHandler(cmd *m.SendEmailCommand) error { + message, err := ns.buildEmailMessage(cmd) if err != nil { return err } - addToMailQueue(message) - + ns.mailQueue <- message return nil } -func sendResetPasswordEmail(cmd *m.SendResetPasswordEmailCommand) error { - return sendEmailCommandHandler(&m.SendEmailCommand{ +func (ns *NotificationService) sendResetPasswordEmail(cmd *m.SendResetPasswordEmailCommand) error { + return ns.sendEmailCommandHandler(&m.SendEmailCommand{ To: []string{cmd.User.Email}, Template: tmplResetPassword, Data: map[string]interface{}{ @@ -116,7 +157,7 @@ func sendResetPasswordEmail(cmd *m.SendResetPasswordEmailCommand) error { }) } -func validateResetPasswordCode(query *m.ValidateResetPasswordCodeQuery) error { +func (ns *NotificationService) validateResetPasswordCode(query *m.ValidateResetPasswordCodeQuery) error { login := getLoginForEmailCode(query.Code) if login == "" { return m.ErrInvalidEmailCode @@ -135,18 +176,18 @@ func validateResetPasswordCode(query *m.ValidateResetPasswordCodeQuery) error { return nil } -func signUpStartedHandler(evt *events.SignUpStarted) error { +func (ns *NotificationService) signUpStartedHandler(evt *events.SignUpStarted) error { if !setting.VerifyEmailEnabled { return nil } - log.Info("User signup started: %s", evt.Email) + ns.log.Info("User signup started", "email", evt.Email) if evt.Email == "" { return nil } - err := sendEmailCommandHandler(&m.SendEmailCommand{ + err := ns.sendEmailCommandHandler(&m.SendEmailCommand{ To: []string{evt.Email}, Template: tmplSignUpStarted, Data: map[string]interface{}{ @@ -155,6 +196,7 @@ func signUpStartedHandler(evt *events.SignUpStarted) error { "SignUpUrl": setting.ToAbsUrl(fmt.Sprintf("signup/?email=%s&code=%s", url.QueryEscape(evt.Email), url.QueryEscape(evt.Code))), }, }) + if err != nil { return err } @@ -163,12 +205,12 @@ func signUpStartedHandler(evt *events.SignUpStarted) error { return bus.Dispatch(&emailSentCmd) } -func signUpCompletedHandler(evt *events.SignUpCompleted) error { - if evt.Email == "" || !setting.Smtp.SendWelcomeEmailOnSignUp { +func (ns *NotificationService) signUpCompletedHandler(evt *events.SignUpCompleted) error { + if evt.Email == "" || !ns.Cfg.Smtp.SendWelcomeEmailOnSignUp { return nil } - return sendEmailCommandHandler(&m.SendEmailCommand{ + return ns.sendEmailCommandHandler(&m.SendEmailCommand{ To: []string{evt.Email}, Template: tmplWelcomeOnSignUp, Data: map[string]interface{}{ diff --git a/pkg/services/notifications/notifications_test.go b/pkg/services/notifications/notifications_test.go index 3a5ff5fedb7..d54b70e704f 100644 --- a/pkg/services/notifications/notifications_test.go +++ b/pkg/services/notifications/notifications_test.go @@ -3,39 +3,33 @@ package notifications import ( "testing" + "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" . "github.com/smartystreets/goconvey/convey" ) -type testTriggeredAlert struct { - ActualValue float64 - Name string - State string -} - func TestNotifications(t *testing.T) { Convey("Given the notifications service", t, func() { - //bus.ClearBusHandlers() - setting.StaticRootPath = "../../../public/" - setting.Smtp.Enabled = true - setting.Smtp.TemplatesPattern = "emails/*.html" - setting.Smtp.FromAddress = "from@address.com" - setting.Smtp.FromName = "Grafana Admin" - err := Init() + ns := &NotificationService{} + ns.Bus = bus.New() + ns.Cfg = setting.NewCfg() + ns.Cfg.Smtp.Enabled = true + ns.Cfg.Smtp.TemplatesPattern = "emails/*.html" + ns.Cfg.Smtp.FromAddress = "from@address.com" + ns.Cfg.Smtp.FromName = "Grafana Admin" + + err := ns.Init() So(err, ShouldBeNil) - var sentMsg *Message - addToMailQueue = func(msg *Message) { - sentMsg = msg - } - Convey("When sending reset email password", func() { - err := sendResetPasswordEmail(&m.SendResetPasswordEmailCommand{User: &m.User{Email: "asd@asd.com"}}) + err := ns.sendResetPasswordEmail(&m.SendResetPasswordEmailCommand{User: &m.User{Email: "asd@asd.com"}}) So(err, ShouldBeNil) + + sentMsg := <-ns.mailQueue So(sentMsg.Body, ShouldContainSubstring, "body") So(sentMsg.Subject, ShouldEqual, "Reset your Grafana password - asd@asd.com") So(sentMsg.Body, ShouldNotContainSubstring, "Subject") diff --git a/pkg/services/notifications/send_email_integration_test.go b/pkg/services/notifications/send_email_integration_test.go index a9a5215d3ca..201f86036d3 100644 --- a/pkg/services/notifications/send_email_integration_test.go +++ b/pkg/services/notifications/send_email_integration_test.go @@ -12,23 +12,19 @@ import ( func TestEmailIntegrationTest(t *testing.T) { SkipConvey("Given the notifications service", t, func() { - bus.ClearBusHandlers() - setting.StaticRootPath = "../../../public/" - setting.Smtp.Enabled = true - setting.Smtp.TemplatesPattern = "emails/*.html" - setting.Smtp.FromAddress = "from@address.com" - setting.Smtp.FromName = "Grafana Admin" setting.BuildVersion = "4.0.0" - err := Init() - So(err, ShouldBeNil) + ns := &NotificationService{} + ns.Bus = bus.New() + ns.Cfg = setting.NewCfg() + ns.Cfg.Smtp.Enabled = true + ns.Cfg.Smtp.TemplatesPattern = "emails/*.html" + ns.Cfg.Smtp.FromAddress = "from@address.com" + ns.Cfg.Smtp.FromName = "Grafana Admin" - addToMailQueue = func(msg *Message) { - So(msg.From, ShouldEqual, "Grafana Admin ") - So(msg.To[0], ShouldEqual, "asdf@asdf.com") - ioutil.WriteFile("../../../tmp/test_email.html", []byte(msg.Body), 0777) - } + err := ns.Init() + So(err, ShouldBeNil) Convey("When sending reset email password", func() { cmd := &m.SendEmailCommand{ @@ -59,8 +55,13 @@ func TestEmailIntegrationTest(t *testing.T) { Template: "alert_notification.html", } - err := sendEmailCommandHandler(cmd) + err := ns.sendEmailCommandHandler(cmd) So(err, ShouldBeNil) + + sentMsg := <-ns.mailQueue + So(sentMsg.From, ShouldEqual, "Grafana Admin ") + So(sentMsg.To[0], ShouldEqual, "asdf@asdf.com") + ioutil.WriteFile("../../../tmp/test_email.html", []byte(sentMsg.Body), 0777) }) }) } diff --git a/pkg/services/notifications/webhook.go b/pkg/services/notifications/webhook.go index dff5aa4924a..a236a1d1c4e 100644 --- a/pkg/services/notifications/webhook.go +++ b/pkg/services/notifications/webhook.go @@ -11,17 +11,17 @@ import ( "golang.org/x/net/context/ctxhttp" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/util" ) type Webhook struct { - Url string - User string - Password string - Body string - HttpMethod string - HttpHeader map[string]string + Url string + User string + Password string + Body string + HttpMethod string + HttpHeader map[string]string + ContentType string } var netTransport = &http.Transport{ @@ -37,32 +37,8 @@ var netClient = &http.Client{ Transport: netTransport, } -var ( - webhookQueue chan *Webhook - webhookLog log.Logger -) - -func initWebhookQueue() { - webhookLog = log.New("notifications.webhook") - webhookQueue = make(chan *Webhook, 10) - go processWebhookQueue() -} - -func processWebhookQueue() { - for { - select { - case webhook := <-webhookQueue: - err := sendWebRequestSync(context.Background(), webhook) - - if err != nil { - webhookLog.Error("Failed to send webrequest ", "error", err) - } - } - } -} - -func sendWebRequestSync(ctx context.Context, webhook *Webhook) error { - webhookLog.Debug("Sending webhook", "url", webhook.Url, "http method", webhook.HttpMethod) +func (ns *NotificationService) sendWebRequestSync(ctx context.Context, webhook *Webhook) error { + ns.log.Debug("Sending webhook", "url", webhook.Url, "http method", webhook.HttpMethod) if webhook.HttpMethod == "" { webhook.HttpMethod = http.MethodPost @@ -73,8 +49,13 @@ func sendWebRequestSync(ctx context.Context, webhook *Webhook) error { return err } - request.Header.Add("Content-Type", "application/json") + if webhook.ContentType == "" { + webhook.ContentType = "application/json" + } + + request.Header.Add("Content-Type", webhook.ContentType) request.Header.Add("User-Agent", "Grafana") + if webhook.User != "" && webhook.Password != "" { request.Header.Add("Authorization", util.GetBasicAuthHeader(webhook.User, webhook.Password)) } @@ -98,10 +79,6 @@ func sendWebRequestSync(ctx context.Context, webhook *Webhook) error { return err } - webhookLog.Debug("Webhook failed", "statuscode", resp.Status, "body", string(body)) + ns.log.Debug("Webhook failed", "statuscode", resp.Status, "body", string(body)) return fmt.Errorf("Webhook response status %v", resp.Status) } - -var addToWebhookQueue = func(msg *Webhook) { - webhookQueue <- msg -} diff --git a/pkg/services/provisioning/dashboards/config_reader.go b/pkg/services/provisioning/dashboards/config_reader.go index 9030ba609b9..bfef06b558e 100644 --- a/pkg/services/provisioning/dashboards/config_reader.go +++ b/pkg/services/provisioning/dashboards/config_reader.go @@ -58,7 +58,7 @@ func (cr *configReader) readConfig() ([]*DashboardsAsConfig, error) { files, err := ioutil.ReadDir(cr.path) if err != nil { - cr.log.Error("cant read dashboard provisioning files from directory", "path", cr.path) + cr.log.Error("can't read dashboard provisioning files from directory", "path", cr.path) return dashboards, nil } @@ -69,7 +69,7 @@ func (cr *configReader) readConfig() ([]*DashboardsAsConfig, error) { parsedDashboards, err := cr.parseConfigs(file) if err != nil { - + return nil, err } if len(parsedDashboards) > 0 { @@ -81,6 +81,10 @@ func (cr *configReader) readConfig() ([]*DashboardsAsConfig, error) { if dashboards[i].OrgId == 0 { dashboards[i].OrgId = 1 } + + if dashboards[i].UpdateIntervalSeconds == 0 { + dashboards[i].UpdateIntervalSeconds = 10 + } } return dashboards, nil diff --git a/pkg/services/provisioning/dashboards/config_reader_test.go b/pkg/services/provisioning/dashboards/config_reader_test.go index ecbf6435c36..d386e42349d 100644 --- a/pkg/services/provisioning/dashboards/config_reader_test.go +++ b/pkg/services/provisioning/dashboards/config_reader_test.go @@ -8,9 +8,9 @@ import ( ) var ( - simpleDashboardConfig string = "./test-configs/dashboards-from-disk" - oldVersion string = "./test-configs/version-0" - brokenConfigs string = "./test-configs/broken-configs" + simpleDashboardConfig = "./testdata/test-configs/dashboards-from-disk" + oldVersion = "./testdata/test-configs/version-0" + brokenConfigs = "./testdata/test-configs/broken-configs" ) func TestDashboardsAsConfig(t *testing.T) { @@ -22,7 +22,7 @@ func TestDashboardsAsConfig(t *testing.T) { cfg, err := cfgProvider.readConfig() So(err, ShouldBeNil) - validateDashboardAsConfig(cfg) + validateDashboardAsConfig(t, cfg) }) Convey("Can read config file in version 0 format", func() { @@ -30,7 +30,7 @@ func TestDashboardsAsConfig(t *testing.T) { cfg, err := cfgProvider.readConfig() So(err, ShouldBeNil) - validateDashboardAsConfig(cfg) + validateDashboardAsConfig(t, cfg) }) Convey("Should skip invalid path", func() { @@ -56,7 +56,9 @@ func TestDashboardsAsConfig(t *testing.T) { }) }) } -func validateDashboardAsConfig(cfg []*DashboardsAsConfig) { +func validateDashboardAsConfig(t *testing.T, cfg []*DashboardsAsConfig) { + t.Helper() + So(len(cfg), ShouldEqual, 2) ds := cfg[0] @@ -68,6 +70,7 @@ func validateDashboardAsConfig(cfg []*DashboardsAsConfig) { So(len(ds.Options), ShouldEqual, 1) So(ds.Options["path"], ShouldEqual, "/var/lib/grafana/dashboards") So(ds.DisableDeletion, ShouldBeTrue) + So(ds.UpdateIntervalSeconds, ShouldEqual, 15) ds2 := cfg[1] So(ds2.Name, ShouldEqual, "default") @@ -78,4 +81,5 @@ func validateDashboardAsConfig(cfg []*DashboardsAsConfig) { So(len(ds2.Options), ShouldEqual, 1) So(ds2.Options["path"], ShouldEqual, "/var/lib/grafana/dashboards") So(ds2.DisableDeletion, ShouldBeFalse) + So(ds2.UpdateIntervalSeconds, ShouldEqual, 10) } diff --git a/pkg/services/provisioning/dashboards/dashboard.go b/pkg/services/provisioning/dashboards/dashboard.go index a5349517bbe..a856565bf01 100644 --- a/pkg/services/provisioning/dashboards/dashboard.go +++ b/pkg/services/provisioning/dashboards/dashboard.go @@ -10,19 +10,16 @@ import ( type DashboardProvisioner struct { cfgReader *configReader log log.Logger - ctx context.Context } -func Provision(ctx context.Context, configDirectory string) (*DashboardProvisioner, error) { +func NewDashboardProvisioner(configDirectory string) *DashboardProvisioner { log := log.New("provisioning.dashboard") d := &DashboardProvisioner{ cfgReader: &configReader{path: configDirectory, log: log}, log: log, - ctx: ctx, } - err := d.Provision(ctx) - return d, err + return d } func (provider *DashboardProvisioner) Provision(ctx context.Context) error { diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index d3e9892c8f5..ea093860f3e 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" + "io/ioutil" "os" "path/filepath" "strings" "time" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/bus" @@ -19,9 +21,7 @@ import ( ) var ( - checkDiskForChangesInterval time.Duration = time.Second * 3 - - ErrFolderNameMissing error = errors.New("Folder name missing") + ErrFolderNameMissing = errors.New("Folder name missing") ) type fileReader struct { @@ -43,10 +43,6 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade log.Warn("[Deprecated] The folder property is deprecated. Please use path instead.") } - if _, err := os.Stat(path); os.IsNotExist(err) { - log.Error("Cannot read directory", "error", err) - } - return &fileReader{ Cfg: cfg, Path: path, @@ -60,7 +56,7 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error { fr.log.Error("failed to search for dashboards", "error", err) } - ticker := time.NewTicker(checkDiskForChangesInterval) + ticker := time.NewTicker(time.Duration(int64(time.Second) * fr.Cfg.UpdateIntervalSeconds)) running := false @@ -83,7 +79,8 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error { } func (fr *fileReader) startWalkingDisk() error { - if _, err := os.Stat(fr.Path); err != nil { + resolvedPath := fr.resolvePath(fr.Path) + if _, err := os.Stat(resolvedPath); err != nil { if os.IsNotExist(err) { return err } @@ -100,7 +97,7 @@ func (fr *fileReader) startWalkingDisk() error { } filesFoundOnDisk := map[string]os.FileInfo{} - err = filepath.Walk(fr.Path, createWalkFn(filesFoundOnDisk)) + err = filepath.Walk(resolvedPath, createWalkFn(filesFoundOnDisk)) if err != nil { return err } @@ -140,7 +137,7 @@ func (fr *fileReader) deleteDashboardIfFileIsMissing(provisionedDashboardRefs ma cmd := &models.DeleteDashboardCommand{OrgId: fr.Cfg.OrgId, Id: dashboardId} err := bus.Dispatch(cmd) if err != nil { - fr.log.Error("failed to delete dashboard", "id", cmd.Id) + fr.log.Error("failed to delete dashboard", "id", cmd.Id, "error", err) } } } @@ -153,15 +150,20 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil } provisionedData, alreadyProvisioned := provisionedDashboardRefs[path] - upToDate := alreadyProvisioned && provisionedData.Updated == resolvedFileInfo.ModTime().Unix() + upToDate := alreadyProvisioned && provisionedData.Updated >= resolvedFileInfo.ModTime().Unix() - dash, err := fr.readDashboardFromFile(path, resolvedFileInfo.ModTime(), folderId) + jsonFile, err := fr.readDashboardFromFile(path, resolvedFileInfo.ModTime(), folderId) if err != nil { fr.log.Error("failed to load dashboard from ", "file", path, "error", err) return provisioningMetadata, nil } + if provisionedData != nil && jsonFile.checkSum == provisionedData.CheckSum { + upToDate = true + } + // keeps track of what uid's and title's we have already provisioned + dash := jsonFile.dashboard provisioningMetadata.uid = dash.Dashboard.Uid provisioningMetadata.title = dash.Dashboard.Title @@ -170,8 +172,8 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil } if dash.Dashboard.Id != 0 { - fr.log.Error("provisioned dashboard json files cannot contain id") - return provisioningMetadata, nil + dash.Dashboard.Data.Set("id", nil) + dash.Dashboard.Id = 0 } if alreadyProvisioned { @@ -179,7 +181,13 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil } fr.log.Debug("saving new dashboard", "file", path) - dp := &models.DashboardProvisioning{ExternalId: path, Name: fr.Cfg.Name, Updated: resolvedFileInfo.ModTime().Unix()} + dp := &models.DashboardProvisioning{ + ExternalId: path, + Name: fr.Cfg.Name, + Updated: resolvedFileInfo.ModTime().Unix(), + CheckSum: jsonFile.checkSum, + } + _, err = fr.dashboardService.SaveProvisionedDashboard(dash, dp) return provisioningMetadata, err } @@ -235,7 +243,6 @@ func getOrCreateFolderId(cfg *DashboardsAsConfig, service dashboards.DashboardPr func resolveSymlink(fileinfo os.FileInfo, path string) (os.FileInfo, error) { checkFilepath, err := filepath.EvalSymlinks(path) if path != checkFilepath { - path = checkFilepath fi, err := os.Lstat(checkFilepath) if err != nil { return nil, err @@ -278,14 +285,30 @@ func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { return true, nil } -func (fr *fileReader) readDashboardFromFile(path string, lastModified time.Time, folderId int64) (*dashboards.SaveDashboardDTO, error) { +type dashboardJsonFile struct { + dashboard *dashboards.SaveDashboardDTO + checkSum string + lastModified time.Time +} + +func (fr *fileReader) readDashboardFromFile(path string, lastModified time.Time, folderId int64) (*dashboardJsonFile, error) { reader, err := os.Open(path) if err != nil { return nil, err } defer reader.Close() - data, err := simplejson.NewFromReader(reader) + all, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + checkSum, err := util.Md5SumString(string(all)) + if err != nil { + return nil, err + } + + data, err := simplejson.NewJson(all) if err != nil { return nil, err } @@ -295,7 +318,34 @@ func (fr *fileReader) readDashboardFromFile(path string, lastModified time.Time, return nil, err } - return dash, nil + return &dashboardJsonFile{ + dashboard: dash, + checkSum: checkSum, + lastModified: lastModified, + }, nil +} + +func (fr *fileReader) resolvePath(path string) string { + if _, err := os.Stat(path); os.IsNotExist(err) { + fr.log.Error("Cannot read directory", "error", err) + } + + copy := path + path, err := filepath.Abs(path) + if err != nil { + fr.log.Error("Could not create absolute path ", "path", path) + } + + path, err = filepath.EvalSymlinks(path) + if err != nil { + fr.log.Error("Failed to read content of symlinked path: %s", path) + } + + if path == "" { + path = copy + fr.log.Info("falling back to original path due to EvalSymlink/Abs failure") + } + return path } type provisioningMetadata struct { @@ -323,7 +373,6 @@ func (checker provisioningSanityChecker) track(pm provisioningMetadata) { if len(pm.title) > 0 { checker.titleUsage[pm.title] += 1 } - } func (checker provisioningSanityChecker) logWarnings(log log.Logger) { @@ -338,5 +387,4 @@ func (checker provisioningSanityChecker) logWarnings(log log.Logger) { log.Error("the same 'title' is used more than once", "title", title, "provider", checker.provisioningProvider) } } - } diff --git a/pkg/services/provisioning/dashboards/file_reader_linux_test.go b/pkg/services/provisioning/dashboards/file_reader_linux_test.go new file mode 100644 index 00000000000..77f488ebcfb --- /dev/null +++ b/pkg/services/provisioning/dashboards/file_reader_linux_test.go @@ -0,0 +1,40 @@ +// +build linux + +package dashboards + +import ( + "path/filepath" + "testing" + + "github.com/grafana/grafana/pkg/log" +) + +var ( + symlinkedFolder = "testdata/test-dashboards/symlink" +) + +func TestProvsionedSymlinkedFolder(t *testing.T) { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{"path": symlinkedFolder}, + } + + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + if err != nil { + t.Error("expected err to be nil") + } + + want, err := filepath.Abs(containingId) + + if err != nil { + t.Errorf("expected err to be nil") + } + + resolvedPath := reader.resolvePath(reader.Path) + if resolvedPath != want { + t.Errorf("got %s want %s", resolvedPath, want) + } +} diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index cd5e3456734..fe849816553 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -3,6 +3,7 @@ package dashboards import ( "os" "path/filepath" + "runtime" "testing" "time" @@ -15,13 +16,63 @@ import ( ) var ( - defaultDashboards string = "./test-dashboards/folder-one" - brokenDashboards string = "./test-dashboards/broken-dashboards" - oneDashboard string = "./test-dashboards/one-dashboard" + defaultDashboards = "testdata/test-dashboards/folder-one" + brokenDashboards = "testdata/test-dashboards/broken-dashboards" + oneDashboard = "testdata/test-dashboards/one-dashboard" + containingId = "testdata/test-dashboards/containing-id" fakeService *fakeDashboardProvisioningService ) +func TestCreatingNewDashboardFileReader(t *testing.T) { + Convey("creating new dashboard file reader", t, func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{}, + } + + Convey("using path parameter", func() { + cfg.Options["path"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + So(reader.Path, ShouldNotEqual, "") + }) + + Convey("using folder as options", func() { + cfg.Options["folder"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + So(reader.Path, ShouldNotEqual, "") + }) + + Convey("using full path", func() { + fullPath := "/var/lib/grafana/dashboards" + if runtime.GOOS == "windows" { + fullPath = `c:\var\lib\grafana` + } + + cfg.Options["folder"] = fullPath + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + + So(reader.Path, ShouldEqual, fullPath) + So(filepath.IsAbs(reader.Path), ShouldBeTrue) + }) + + Convey("using relative path", func() { + cfg.Options["folder"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + + resolvedPath := reader.resolvePath(reader.Path) + So(filepath.IsAbs(resolvedPath), ShouldBeTrue) + }) + }) +} + func TestDashboardFileReader(t *testing.T) { Convey("Dashboard file reader", t, func() { bus.ClearBusHandlers() @@ -85,6 +136,18 @@ func TestDashboardFileReader(t *testing.T) { So(len(fakeService.inserted), ShouldEqual, 1) }) + Convey("Overrides id from dashboard.json files", func() { + cfg.Options["path"] = containingId + + reader, err := NewDashboardFileReader(cfg, logger) + So(err, ShouldBeNil) + + err = reader.startWalkingDisk() + So(err, ShouldBeNil) + + So(len(fakeService.inserted), ShouldEqual, 1) + }) + Convey("Invalid configuration should return error", func() { cfg := &DashboardsAsConfig{ Name: "Default", @@ -157,30 +220,6 @@ func TestDashboardFileReader(t *testing.T) { }) }) - Convey("Can use bpth path and folder as dashboard path", func() { - cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", - Options: map[string]interface{}{}, - } - - Convey("using path parameter", func() { - cfg.Options["path"] = defaultDashboards - reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) - So(err, ShouldBeNil) - So(reader.Path, ShouldEqual, defaultDashboards) - }) - - Convey("using folder as options", func() { - cfg.Options["folder"] = defaultDashboards - reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) - So(err, ShouldBeNil) - So(reader.Path, ShouldEqual, defaultDashboards) - }) - }) - Reset(func() { dashboards.NewProvisioningService = origNewDashboardProvisioningService }) diff --git a/pkg/services/provisioning/dashboards/test-configs/broken-configs/commented.yaml b/pkg/services/provisioning/dashboards/testdata/test-configs/broken-configs/commented.yaml similarity index 100% rename from pkg/services/provisioning/dashboards/test-configs/broken-configs/commented.yaml rename to pkg/services/provisioning/dashboards/testdata/test-configs/broken-configs/commented.yaml diff --git a/pkg/services/provisioning/dashboards/test-configs/dashboards-from-disk/dev-dashboards.yaml b/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml similarity index 90% rename from pkg/services/provisioning/dashboards/test-configs/dashboards-from-disk/dev-dashboards.yaml rename to pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml index e9776d69010..c43c4a14c53 100644 --- a/pkg/services/provisioning/dashboards/test-configs/dashboards-from-disk/dev-dashboards.yaml +++ b/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml @@ -6,6 +6,7 @@ providers: folder: 'developers' editable: true disableDeletion: true + updateIntervalSeconds: 15 type: file options: path: /var/lib/grafana/dashboards diff --git a/pkg/services/provisioning/dashboards/test-configs/dashboards-from-disk/sample.yaml b/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/sample.yaml similarity index 100% rename from pkg/services/provisioning/dashboards/test-configs/dashboards-from-disk/sample.yaml rename to pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/sample.yaml diff --git a/pkg/services/provisioning/dashboards/test-configs/version-0/version-0.yaml b/pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml similarity index 89% rename from pkg/services/provisioning/dashboards/test-configs/version-0/version-0.yaml rename to pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml index 979e762d4d4..8b7b8991759 100644 --- a/pkg/services/provisioning/dashboards/test-configs/version-0/version-0.yaml +++ b/pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml @@ -3,6 +3,7 @@ folder: 'developers' editable: true disableDeletion: true + updateIntervalSeconds: 15 type: file options: path: /var/lib/grafana/dashboards diff --git a/pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/empty-json.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/broken-dashboards/empty-json.json similarity index 100% rename from pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/empty-json.json rename to pkg/services/provisioning/dashboards/testdata/test-dashboards/broken-dashboards/empty-json.json diff --git a/pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/invalid.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/broken-dashboards/invalid.json similarity index 100% rename from pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/invalid.json rename to pkg/services/provisioning/dashboards/testdata/test-dashboards/broken-dashboards/invalid.json diff --git a/pkg/services/provisioning/dashboards/testdata/test-dashboards/containing-id/dashboard1.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/containing-id/dashboard1.json new file mode 100644 index 00000000000..12a8b81eee6 --- /dev/null +++ b/pkg/services/provisioning/dashboards/testdata/test-dashboards/containing-id/dashboard1.json @@ -0,0 +1,68 @@ +{ + "title": "Grafana1", + "tags": [], + "id": 3, + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5 + } diff --git a/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/folder-one/dashboard1.json similarity index 100% rename from pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json rename to pkg/services/provisioning/dashboards/testdata/test-dashboards/folder-one/dashboard1.json diff --git a/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/folder-one/dashboard2.json similarity index 100% rename from pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json rename to pkg/services/provisioning/dashboards/testdata/test-dashboards/folder-one/dashboard2.json diff --git a/pkg/services/provisioning/dashboards/test-dashboards/one-dashboard/dashboard1.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/one-dashboard/dashboard1.json similarity index 100% rename from pkg/services/provisioning/dashboards/test-dashboards/one-dashboard/dashboard1.json rename to pkg/services/provisioning/dashboards/testdata/test-dashboards/one-dashboard/dashboard1.json diff --git a/pkg/services/provisioning/dashboards/testdata/test-dashboards/symlink b/pkg/services/provisioning/dashboards/testdata/test-dashboards/symlink new file mode 120000 index 00000000000..42e166e6959 --- /dev/null +++ b/pkg/services/provisioning/dashboards/testdata/test-dashboards/symlink @@ -0,0 +1 @@ +containing-id/ \ No newline at end of file diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index f742b321552..a658b816c7d 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -10,23 +10,25 @@ import ( ) type DashboardsAsConfig struct { - Name string - Type string - OrgId int64 - Folder string - Editable bool - Options map[string]interface{} - DisableDeletion bool + Name string + Type string + OrgId int64 + Folder string + Editable bool + Options map[string]interface{} + DisableDeletion bool + UpdateIntervalSeconds int64 } type DashboardsAsConfigV0 struct { - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - OrgId int64 `json:"org_id" yaml:"org_id"` - Folder string `json:"folder" yaml:"folder"` - Editable bool `json:"editable" yaml:"editable"` - Options map[string]interface{} `json:"options" yaml:"options"` - DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + OrgId int64 `json:"org_id" yaml:"org_id"` + Folder string `json:"folder" yaml:"folder"` + Editable bool `json:"editable" yaml:"editable"` + Options map[string]interface{} `json:"options" yaml:"options"` + DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` + UpdateIntervalSeconds int64 `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"` } type ConfigVersion struct { @@ -38,13 +40,14 @@ type DashboardAsConfigV1 struct { } type DashboardProviderConfigs struct { - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - OrgId int64 `json:"orgId" yaml:"orgId"` - Folder string `json:"folder" yaml:"folder"` - Editable bool `json:"editable" yaml:"editable"` - Options map[string]interface{} `json:"options" yaml:"options"` - DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + OrgId int64 `json:"orgId" yaml:"orgId"` + Folder string `json:"folder" yaml:"folder"` + Editable bool `json:"editable" yaml:"editable"` + Options map[string]interface{} `json:"options" yaml:"options"` + DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` + UpdateIntervalSeconds int64 `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"` } func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardDTO, error) { @@ -55,9 +58,6 @@ func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *Das dash.OrgId = cfg.OrgId dash.Dashboard.OrgId = cfg.OrgId dash.Dashboard.FolderId = folderId - if !cfg.Editable { - dash.Dashboard.Data.Set("editable", cfg.Editable) - } if dash.Dashboard.Title == "" { return nil, models.ErrDashboardTitleEmpty @@ -71,13 +71,14 @@ func mapV0ToDashboardAsConfig(v0 []*DashboardsAsConfigV0) []*DashboardsAsConfig for _, v := range v0 { r = append(r, &DashboardsAsConfig{ - Name: v.Name, - Type: v.Type, - OrgId: v.OrgId, - Folder: v.Folder, - Editable: v.Editable, - Options: v.Options, - DisableDeletion: v.DisableDeletion, + Name: v.Name, + Type: v.Type, + OrgId: v.OrgId, + Folder: v.Folder, + Editable: v.Editable, + Options: v.Options, + DisableDeletion: v.DisableDeletion, + UpdateIntervalSeconds: v.UpdateIntervalSeconds, }) } @@ -89,13 +90,14 @@ func (dc *DashboardAsConfigV1) mapToDashboardAsConfig() []*DashboardsAsConfig { for _, v := range dc.Providers { r = append(r, &DashboardsAsConfig{ - Name: v.Name, - Type: v.Type, - OrgId: v.OrgId, - Folder: v.Folder, - Editable: v.Editable, - Options: v.Options, - DisableDeletion: v.DisableDeletion, + Name: v.Name, + Type: v.Type, + OrgId: v.OrgId, + Folder: v.Folder, + Editable: v.Editable, + Options: v.Options, + DisableDeletion: v.DisableDeletion, + UpdateIntervalSeconds: v.UpdateIntervalSeconds, }) } diff --git a/pkg/services/provisioning/datasources/config_reader.go b/pkg/services/provisioning/datasources/config_reader.go index 58ed5472a6b..b2930c2b679 100644 --- a/pkg/services/provisioning/datasources/config_reader.go +++ b/pkg/services/provisioning/datasources/config_reader.go @@ -19,7 +19,7 @@ func (cr *configReader) readConfig(path string) ([]*DatasourcesAsConfig, error) files, err := ioutil.ReadDir(path) if err != nil { - cr.log.Error("cant read datasource provisioning files from directory", "path", path) + cr.log.Error("can't read datasource provisioning files from directory", "path", path) return datasources, nil } @@ -83,7 +83,7 @@ func (cr *configReader) parseDatasourceConfig(path string, file os.FileInfo) (*D } func validateDefaultUniqueness(datasources []*DatasourcesAsConfig) error { - defaultCount := 0 + defaultCount := map[int64]int{} for i := range datasources { if datasources[i].Datasources == nil { continue @@ -95,8 +95,8 @@ func validateDefaultUniqueness(datasources []*DatasourcesAsConfig) error { } if ds.IsDefault { - defaultCount++ - if defaultCount > 1 { + defaultCount[ds.OrgId] = defaultCount[ds.OrgId] + 1 + if defaultCount[ds.OrgId] > 1 { return ErrInvalidConfigToManyDefault } } diff --git a/pkg/services/provisioning/datasources/config_reader_test.go b/pkg/services/provisioning/datasources/config_reader_test.go index 3198329e0ae..07c8d68e75c 100644 --- a/pkg/services/provisioning/datasources/config_reader_test.go +++ b/pkg/services/provisioning/datasources/config_reader_test.go @@ -11,14 +11,15 @@ import ( ) var ( - logger log.Logger = log.New("fake.log") - oneDatasourcesConfig string = "" - twoDatasourcesConfig string = "./test-configs/two-datasources" - twoDatasourcesConfigPurgeOthers string = "./test-configs/insert-two-delete-two" - doubleDatasourcesConfig string = "./test-configs/double-default" - allProperties string = "./test-configs/all-properties" - versionZero string = "./test-configs/version-0" - brokenYaml string = "./test-configs/broken-yaml" + logger log.Logger = log.New("fake.log") + + twoDatasourcesConfig = "testdata/two-datasources" + twoDatasourcesConfigPurgeOthers = "testdata/insert-two-delete-two" + doubleDatasourcesConfig = "testdata/double-default" + allProperties = "testdata/all-properties" + versionZero = "testdata/version-0" + brokenYaml = "testdata/broken-yaml" + multipleOrgsWithDefault = "testdata/multiple-org-default" fakeRepo *fakeRepository ) @@ -73,6 +74,19 @@ func TestDatasourceAsConfig(t *testing.T) { }) }) + Convey("Multiple datasources in different organizations with isDefault in each organization", func() { + dc := newDatasourceProvisioner(logger) + err := dc.applyChanges(multipleOrgsWithDefault) + Convey("should not raise error", func() { + So(err, ShouldBeNil) + So(len(fakeRepo.inserted), ShouldEqual, 4) + So(fakeRepo.inserted[0].IsDefault, ShouldBeTrue) + So(fakeRepo.inserted[0].OrgId, ShouldEqual, 1) + So(fakeRepo.inserted[2].IsDefault, ShouldBeTrue) + So(fakeRepo.inserted[2].OrgId, ShouldEqual, 2) + }) + }) + Convey("Two configured datasource and purge others ", func() { Convey("two other datasources in database", func() { fakeRepo.loadAll = []*models.DataSource{ diff --git a/pkg/services/provisioning/datasources/datasources.go b/pkg/services/provisioning/datasources/datasources.go index 1fa0a3b3173..de6c876baad 100644 --- a/pkg/services/provisioning/datasources/datasources.go +++ b/pkg/services/provisioning/datasources/datasources.go @@ -11,7 +11,7 @@ import ( ) var ( - ErrInvalidConfigToManyDefault = errors.New("datasource.yaml config is invalid. Only one datasource can be marked as default") + ErrInvalidConfigToManyDefault = errors.New("datasource.yaml config is invalid. Only one datasource per organization can be marked as default") ) func Provision(configDirectory string) error { diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt b/pkg/services/provisioning/datasources/testdata/all-properties/not.yaml.txt similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt rename to pkg/services/provisioning/datasources/testdata/all-properties/not.yaml.txt diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/sample.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/sample.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/sample.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/sample.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/second.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/second.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml b/pkg/services/provisioning/datasources/testdata/broken-yaml/broken.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml rename to pkg/services/provisioning/datasources/testdata/broken-yaml/broken.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml b/pkg/services/provisioning/datasources/testdata/broken-yaml/commented.yaml similarity index 97% rename from pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml rename to pkg/services/provisioning/datasources/testdata/broken-yaml/commented.yaml index 1bb9cb53b45..b532c9012ec 100644 --- a/pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml +++ b/pkg/services/provisioning/datasources/testdata/broken-yaml/commented.yaml @@ -4,7 +4,7 @@ # org_id: 1 # # list of datasources to insert/update depending -# # whats available in the datbase +# # what's available in the database #datasources: # # name of the datasource. Required # - name: Graphite diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml b/pkg/services/provisioning/datasources/testdata/double-default/default-1.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml rename to pkg/services/provisioning/datasources/testdata/double-default/default-1.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml b/pkg/services/provisioning/datasources/testdata/double-default/default-2.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml rename to pkg/services/provisioning/datasources/testdata/double-default/default-2.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml b/pkg/services/provisioning/datasources/testdata/insert-two-delete-two/one-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml rename to pkg/services/provisioning/datasources/testdata/insert-two-delete-two/one-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml b/pkg/services/provisioning/datasources/testdata/insert-two-delete-two/two-datasources.yml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml rename to pkg/services/provisioning/datasources/testdata/insert-two-delete-two/two-datasources.yml diff --git a/pkg/services/provisioning/datasources/testdata/multiple-org-default/config.yaml b/pkg/services/provisioning/datasources/testdata/multiple-org-default/config.yaml new file mode 100644 index 00000000000..f185abb6f53 --- /dev/null +++ b/pkg/services/provisioning/datasources/testdata/multiple-org-default/config.yaml @@ -0,0 +1,25 @@ +apiVersion: 1 + +datasources: + - orgId: 1 + name: prometheus + type: prometheus + isDefault: True + access: proxy + url: http://prometheus.example.com:9090 + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + - orgId: 2 + name: prometheus + type: prometheus + isDefault: True + access: proxy + url: http://prometheus.example.com:9090 + - orgId: 2 + name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + diff --git a/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml b/pkg/services/provisioning/datasources/testdata/two-datasources/two-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml rename to pkg/services/provisioning/datasources/testdata/two-datasources/two-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/version-0/version-0.yaml b/pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/version-0/version-0.yaml rename to pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git rename to pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index b41ec37b797..9044ae97389 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -2,34 +2,40 @@ package provisioning import ( "context" + "fmt" "path" - "path/filepath" + "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/provisioning/dashboards" "github.com/grafana/grafana/pkg/services/provisioning/datasources" - ini "gopkg.in/ini.v1" + "github.com/grafana/grafana/pkg/setting" ) -func Init(ctx context.Context, homePath string, cfg *ini.File) error { - provisioningPath := makeAbsolute(cfg.Section("paths").Key("provisioning").String(), homePath) +func init() { + registry.RegisterService(&ProvisioningService{}) +} - datasourcePath := path.Join(provisioningPath, "datasources") +type ProvisioningService struct { + Cfg *setting.Cfg `inject:""` +} + +func (ps *ProvisioningService) Init() error { + datasourcePath := path.Join(ps.Cfg.ProvisioningPath, "datasources") if err := datasources.Provision(datasourcePath); err != nil { - return err - } - - dashboardPath := path.Join(provisioningPath, "dashboards") - _, err := dashboards.Provision(ctx, dashboardPath) - if err != nil { - return err + return fmt.Errorf("Datasource provisioning error: %v", err) } return nil } -func makeAbsolute(path string, root string) string { - if filepath.IsAbs(path) { - return path +func (ps *ProvisioningService) Run(ctx context.Context) error { + dashboardPath := path.Join(ps.Cfg.ProvisioningPath, "dashboards") + dashProvisioner := dashboards.NewDashboardProvisioner(dashboardPath) + + if err := dashProvisioner.Provision(ctx); err != nil { + return err } - return filepath.Join(root, path) + + <-ctx.Done() + return ctx.Err() } diff --git a/pkg/services/rendering/http_mode.go b/pkg/services/rendering/http_mode.go new file mode 100644 index 00000000000..40259c44746 --- /dev/null +++ b/pkg/services/rendering/http_mode.go @@ -0,0 +1,96 @@ +package rendering + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strconv" + "time" +) + +var netTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + DualStack: true, + }).Dial, + TLSHandshakeTimeout: 5 * time.Second, +} + +var netClient = &http.Client{ + Transport: netTransport, +} + +func (rs *RenderingService) renderViaHttp(ctx context.Context, opts Opts) (*RenderResult, error) { + filePath := rs.getFilePathForNewImage() + + rendererUrl, err := url.Parse(rs.Cfg.RendererUrl) + if err != nil { + return nil, err + } + + queryParams := rendererUrl.Query() + queryParams.Add("url", rs.getURL(opts.Path)) + queryParams.Add("renderKey", rs.getRenderKey(opts.OrgId, opts.UserId, opts.OrgRole)) + queryParams.Add("width", strconv.Itoa(opts.Width)) + queryParams.Add("height", strconv.Itoa(opts.Height)) + queryParams.Add("domain", rs.domain) + queryParams.Add("timezone", isoTimeOffsetToPosixTz(opts.Timezone)) + queryParams.Add("encoding", opts.Encoding) + queryParams.Add("timeout", strconv.Itoa(int(opts.Timeout.Seconds()))) + rendererUrl.RawQuery = queryParams.Encode() + + req, err := http.NewRequest("GET", rendererUrl.String(), nil) + if err != nil { + return nil, err + } + + reqContext, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2) + defer cancel() + + req = req.WithContext(reqContext) + + // make request to renderer server + resp, err := netClient.Do(req) + if err != nil { + rs.log.Error("Failed to send request to remote rendering service.", "error", err) + return nil, fmt.Errorf("Failed to send request to remote rendering service. %s", err) + } + + // save response to file + defer resp.Body.Close() + + // check for timeout first + if reqContext.Err() == context.DeadlineExceeded { + rs.log.Info("Rendering timed out") + return nil, ErrTimeout + } + + // if we didn't get a 200 response, something went wrong. + if resp.StatusCode != http.StatusOK { + rs.log.Error("Remote rendering request failed", "error", resp.Status) + return nil, fmt.Errorf("Remote rendering request failed. %d: %s", resp.StatusCode, resp.Status) + } + + out, err := os.Create(filePath) + if err != nil { + return nil, err + } + defer out.Close() + _, err = io.Copy(out, resp.Body) + if err != nil { + // check that we didn't timeout while receiving the response. + if reqContext.Err() == context.DeadlineExceeded { + rs.log.Info("Rendering timed out") + return nil, ErrTimeout + } + rs.log.Error("Remote rendering request failed", "error", err) + return nil, fmt.Errorf("Remote rendering request failed. %s", err) + } + + return &RenderResult{FilePath: filePath}, err +} diff --git a/pkg/services/rendering/interface.go b/pkg/services/rendering/interface.go new file mode 100644 index 00000000000..39cb1ada0f5 --- /dev/null +++ b/pkg/services/rendering/interface.go @@ -0,0 +1,36 @@ +package rendering + +import ( + "context" + "errors" + "time" + + "github.com/grafana/grafana/pkg/models" +) + +var ErrTimeout = errors.New("Timeout error. You can set timeout in seconds with &timeout url parameter") +var ErrNoRenderer = errors.New("No renderer plugin found nor is an external render server configured") +var ErrPhantomJSNotInstalled = errors.New("PhantomJS executable not found") + +type Opts struct { + Width int + Height int + Timeout time.Duration + OrgId int64 + UserId int64 + OrgRole models.RoleType + Path string + Encoding string + Timezone string + ConcurrentLimit int +} + +type RenderResult struct { + FilePath string +} + +type renderFunc func(ctx context.Context, options Opts) (*RenderResult, error) + +type Service interface { + Render(ctx context.Context, opts Opts) (*RenderResult, error) +} diff --git a/pkg/services/rendering/phantomjs.go b/pkg/services/rendering/phantomjs.go new file mode 100644 index 00000000000..1bd7489c153 --- /dev/null +++ b/pkg/services/rendering/phantomjs.go @@ -0,0 +1,111 @@ +package rendering + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/middleware" +) + +func (rs *RenderingService) renderViaPhantomJS(ctx context.Context, opts Opts) (*RenderResult, error) { + rs.log.Info("Rendering", "path", opts.Path) + + var executable = "phantomjs" + if runtime.GOOS == "windows" { + executable = executable + ".exe" + } + + url := rs.getURL(opts.Path) + binPath, _ := filepath.Abs(filepath.Join(rs.Cfg.PhantomDir, executable)) + if _, err := os.Stat(binPath); os.IsNotExist(err) { + rs.log.Error("executable not found", "executable", binPath) + return nil, ErrPhantomJSNotInstalled + } + + scriptPath, _ := filepath.Abs(filepath.Join(rs.Cfg.PhantomDir, "render.js")) + pngPath := rs.getFilePathForNewImage() + + renderKey := middleware.AddRenderAuthKey(opts.OrgId, opts.UserId, opts.OrgRole) + defer middleware.RemoveRenderAuthKey(renderKey) + + phantomDebugArg := "--debug=false" + if log.GetLogLevelFor("renderer") >= log.LvlDebug { + phantomDebugArg = "--debug=true" + } + + cmdArgs := []string{ + "--ignore-ssl-errors=true", + "--web-security=false", + phantomDebugArg, + scriptPath, + fmt.Sprintf("url=%v", url), + fmt.Sprintf("width=%v", opts.Width), + fmt.Sprintf("height=%v", opts.Height), + fmt.Sprintf("png=%v", pngPath), + fmt.Sprintf("domain=%v", rs.domain), + fmt.Sprintf("timeout=%v", opts.Timeout.Seconds()), + fmt.Sprintf("renderKey=%v", renderKey), + } + + if opts.Encoding != "" { + cmdArgs = append([]string{fmt.Sprintf("--output-encoding=%s", opts.Encoding)}, cmdArgs...) + } + + commandCtx, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2) + defer cancel() + + cmd := exec.CommandContext(commandCtx, binPath, cmdArgs...) + cmd.Stderr = cmd.Stdout + + if opts.Timezone != "" { + baseEnviron := os.Environ() + cmd.Env = appendEnviron(baseEnviron, "TZ", isoTimeOffsetToPosixTz(opts.Timezone)) + } + + out, err := cmd.Output() + + // check for timeout first + if commandCtx.Err() == context.DeadlineExceeded { + rs.log.Info("Rendering timed out") + return nil, ErrTimeout + } + + if err != nil { + rs.log.Error("Phantomjs exited with non zero exit code", "error", err) + return nil, err + } + + rs.log.Debug("Phantomjs output", "out", string(out)) + + rs.log.Debug("Image rendered", "path", pngPath) + return &RenderResult{FilePath: pngPath}, nil +} + +func isoTimeOffsetToPosixTz(isoOffset string) string { + // invert offset + if strings.HasPrefix(isoOffset, "UTC+") { + return strings.Replace(isoOffset, "UTC+", "UTC-", 1) + } + if strings.HasPrefix(isoOffset, "UTC-") { + return strings.Replace(isoOffset, "UTC-", "UTC+", 1) + } + return isoOffset +} + +func appendEnviron(baseEnviron []string, name string, value string) []string { + results := make([]string, 0) + prefix := fmt.Sprintf("%s=", name) + for _, v := range baseEnviron { + if !strings.HasPrefix(v, prefix) { + results = append(results, v) + } + } + return append(results, fmt.Sprintf("%s=%s", name, value)) +} diff --git a/pkg/services/rendering/plugin_mode.go b/pkg/services/rendering/plugin_mode.go new file mode 100644 index 00000000000..58fef2b095f --- /dev/null +++ b/pkg/services/rendering/plugin_mode.go @@ -0,0 +1,95 @@ +package rendering + +import ( + "context" + "fmt" + "os/exec" + "path" + "time" + + pluginModel "github.com/grafana/grafana-plugin-model/go/renderer" + "github.com/grafana/grafana/pkg/plugins" + plugin "github.com/hashicorp/go-plugin" +) + +func (rs *RenderingService) startPlugin(ctx context.Context) error { + cmd := plugins.ComposePluginStartCommmand("plugin_start") + fullpath := path.Join(rs.pluginInfo.PluginDir, cmd) + + var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "grafana_plugin_type", + MagicCookieValue: "renderer", + } + + rs.log.Info("Renderer plugin found, starting", "cmd", cmd) + + rs.pluginClient = plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: handshakeConfig, + Plugins: map[string]plugin.Plugin{ + plugins.Renderer.Id: &pluginModel.RendererPluginImpl{}, + }, + Cmd: exec.Command(fullpath), + AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, + Logger: plugins.LogWrapper{Logger: rs.log}, + }) + + rpcClient, err := rs.pluginClient.Client() + if err != nil { + return err + } + + raw, err := rpcClient.Dispense(rs.pluginInfo.Id) + if err != nil { + return err + } + + rs.grpcPlugin = raw.(pluginModel.RendererPlugin) + + return nil +} + +func (rs *RenderingService) watchAndRestartPlugin(ctx context.Context) error { + ticker := time.NewTicker(time.Second * 1) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if rs.pluginClient.Exited() { + err := rs.startPlugin(ctx) + rs.log.Debug("Render plugin existed, restarting...") + if err != nil { + rs.log.Error("Failed to start render plugin", err) + } + } + } + } +} + +func (rs *RenderingService) renderViaPlugin(ctx context.Context, opts Opts) (*RenderResult, error) { + pngPath := rs.getFilePathForNewImage() + + rsp, err := rs.grpcPlugin.Render(ctx, &pluginModel.RenderRequest{ + Url: rs.getURL(opts.Path), + Width: int32(opts.Width), + Height: int32(opts.Height), + FilePath: pngPath, + Timeout: int32(opts.Timeout.Seconds()), + RenderKey: rs.getRenderKey(opts.OrgId, opts.UserId, opts.OrgRole), + Encoding: opts.Encoding, + Timezone: isoTimeOffsetToPosixTz(opts.Timezone), + Domain: rs.domain, + }) + + if err != nil { + return nil, err + } + + if rsp.Error != "" { + return nil, fmt.Errorf("Rendering failed: %v", rsp.Error) + } + + return &RenderResult{FilePath: pngPath}, err +} diff --git a/pkg/services/rendering/rendering.go b/pkg/services/rendering/rendering.go new file mode 100644 index 00000000000..0b4f23e93b4 --- /dev/null +++ b/pkg/services/rendering/rendering.go @@ -0,0 +1,134 @@ +package rendering + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + + plugin "github.com/hashicorp/go-plugin" + + pluginModel "github.com/grafana/grafana-plugin-model/go/renderer" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/middleware" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" +) + +func init() { + registry.RegisterService(&RenderingService{}) +} + +type RenderingService struct { + log log.Logger + pluginClient *plugin.Client + grpcPlugin pluginModel.RendererPlugin + pluginInfo *plugins.RendererPlugin + renderAction renderFunc + domain string + inProgressCount int + + Cfg *setting.Cfg `inject:""` +} + +func (rs *RenderingService) Init() error { + rs.log = log.New("rendering") + + // ensure ImagesDir exists + err := os.MkdirAll(rs.Cfg.ImagesDir, 0700) + if err != nil { + return err + } + + // set value used for domain attribute of renderKey cookie + if rs.Cfg.RendererUrl != "" { + // RendererCallbackUrl has already been passed, it won't generate an error. + u, _ := url.Parse(rs.Cfg.RendererCallbackUrl) + rs.domain = u.Hostname() + } else if setting.HttpAddr != setting.DEFAULT_HTTP_ADDR { + rs.domain = setting.HttpAddr + } else { + rs.domain = "localhost" + } + + return nil +} + +func (rs *RenderingService) Run(ctx context.Context) error { + if rs.Cfg.RendererUrl != "" { + rs.log.Info("Backend rendering via external http server") + rs.renderAction = rs.renderViaHttp + <-ctx.Done() + return nil + } + + if plugins.Renderer == nil { + rs.renderAction = rs.renderViaPhantomJS + <-ctx.Done() + return nil + } + + rs.pluginInfo = plugins.Renderer + + if err := rs.startPlugin(ctx); err != nil { + return err + } + + rs.renderAction = rs.renderViaPlugin + + err := rs.watchAndRestartPlugin(ctx) + + if rs.pluginClient != nil { + rs.log.Debug("Killing renderer plugin process") + rs.pluginClient.Kill() + } + + return err +} + +func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResult, error) { + if rs.inProgressCount > opts.ConcurrentLimit { + return &RenderResult{ + FilePath: filepath.Join(setting.HomePath, "public/img/rendering_limit.png"), + }, nil + } + + defer func() { + rs.inProgressCount -= 1 + }() + + rs.inProgressCount += 1 + + if rs.renderAction != nil { + return rs.renderAction(ctx, opts) + } else { + return nil, fmt.Errorf("No renderer found") + } +} + +func (rs *RenderingService) getFilePathForNewImage() string { + pngPath, _ := filepath.Abs(filepath.Join(rs.Cfg.ImagesDir, util.GetRandomString(20))) + return pngPath + ".png" +} + +func (rs *RenderingService) getURL(path string) string { + if rs.Cfg.RendererUrl != "" { + // The backend rendering service can potentially be remote. + // So we need to use the root_url to ensure the rendering service + // can reach this Grafana instance. + + // &render=1 signals to the legacy redirect layer to + return fmt.Sprintf("%s%s&render=1", rs.Cfg.RendererCallbackUrl, path) + + } + // &render=1 signals to the legacy redirect layer to + return fmt.Sprintf("%s://%s:%s/%s&render=1", setting.Protocol, rs.domain, setting.HttpPort, path) +} + +func (rs *RenderingService) getRenderKey(orgId, userId int64, orgRole models.RoleType) string { + return middleware.AddRenderAuthKey(orgId, userId, orgRole) +} diff --git a/pkg/services/search/handlers.go b/pkg/services/search/handlers.go index cf194c320bb..9d40697f489 100644 --- a/pkg/services/search/handlers.go +++ b/pkg/services/search/handlers.go @@ -5,13 +5,23 @@ import ( "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/registry" ) -func Init() { - bus.AddHandler("search", searchHandler) +func init() { + registry.RegisterService(&SearchService{}) } -func searchHandler(query *Query) error { +type SearchService struct { + Bus bus.Bus `inject:""` +} + +func (s *SearchService) Init() error { + s.Bus.AddHandler(s.searchHandler) + return nil +} + +func (s *SearchService) searchHandler(query *Query) error { dashQuery := FindPersistedDashboardsQuery{ Title: query.Title, SignedInUser: query.SignedInUser, diff --git a/pkg/services/search/handlers_test.go b/pkg/services/search/handlers_test.go index fc223b2ef4b..5cf934cbc92 100644 --- a/pkg/services/search/handlers_test.go +++ b/pkg/services/search/handlers_test.go @@ -12,6 +12,7 @@ func TestSearch(t *testing.T) { Convey("Given search query", t, func() { query := Query{Limit: 2000, SignedInUser: &m.SignedInUser{IsGrafanaAdmin: true}} + ss := &SearchService{} bus.AddHandler("test", func(query *FindPersistedDashboardsQuery) error { query.Result = HitList{ @@ -35,7 +36,7 @@ func TestSearch(t *testing.T) { }) Convey("That is empty", func() { - err := searchHandler(&query) + err := ss.searchHandler(&query) So(err, ShouldBeNil) Convey("should return sorted results", func() { diff --git a/vendor/github.com/go-macaron/session/mysql/mysql.go b/pkg/services/session/mysql.go similarity index 87% rename from vendor/github.com/go-macaron/session/mysql/mysql.go rename to pkg/services/session/mysql.go index a16db779c63..f8c5d828cfa 100644 --- a/vendor/github.com/go-macaron/session/mysql/mysql.go +++ b/pkg/services/session/mysql.go @@ -108,6 +108,7 @@ func (p *MysqlProvider) Init(expire int64, connStr string) (err error) { p.expire = expire p.c, err = sql.Open("mysql", connStr) + p.c.SetConnMaxLifetime(time.Second * time.Duration(sessionConnMaxLifetime)) if err != nil { return err } @@ -141,12 +142,29 @@ func (p *MysqlProvider) Read(sid string) (session.RawStore, error) { // Exist returns true if session with given ID exists. func (p *MysqlProvider) Exist(sid string) bool { + exists, err := p.queryExists(sid) + + if err != nil { + exists, err = p.queryExists(sid) + } + + if err != nil { + log.Printf("session/mysql: error checking if session exists: %v", err) + return false + } + + return exists +} + +func (p *MysqlProvider) queryExists(sid string) (bool, error) { var data []byte err := p.c.QueryRow("SELECT data FROM session WHERE `key`=?", sid).Scan(&data) + if err != nil && err != sql.ErrNoRows { - panic("session/mysql: error checking existence: " + err.Error()) + return false, err } - return err != sql.ErrNoRows + + return err != sql.ErrNoRows, nil } // Destory deletes a session by session ID. @@ -185,7 +203,12 @@ func (p *MysqlProvider) Count() (total int) { // GC calls GC to clean expired sessions. func (p *MysqlProvider) GC() { - if _, err := p.c.Exec("DELETE FROM session WHERE expiry + ? <= UNIX_TIMESTAMP(NOW())", p.expire); err != nil { + var err error + if _, err = p.c.Exec("DELETE FROM session WHERE expiry + ? <= UNIX_TIMESTAMP(NOW())", p.expire); err != nil { + _, err = p.c.Exec("DELETE FROM session WHERE expiry + ? <= UNIX_TIMESTAMP(NOW())", p.expire) + } + + if err != nil { log.Printf("session/mysql: error garbage collecting: %v", err) } } diff --git a/pkg/services/session/session.go b/pkg/services/session/session.go index bfdc58bc5cc..5873a6a5b72 100644 --- a/pkg/services/session/session.go +++ b/pkg/services/session/session.go @@ -6,7 +6,6 @@ import ( ms "github.com/go-macaron/session" _ "github.com/go-macaron/session/memcache" - _ "github.com/go-macaron/session/mysql" _ "github.com/go-macaron/session/postgres" _ "github.com/go-macaron/session/redis" "github.com/grafana/grafana/pkg/log" @@ -25,6 +24,7 @@ var sessionOptions *ms.Options var StartSessionGC func() var GetSessionCount func() int var sessionLogger = log.New("session") +var sessionConnMaxLifetime int64 func init() { StartSessionGC = func() { @@ -37,9 +37,10 @@ func init() { } } -func Init(options *ms.Options) { +func Init(options *ms.Options, connMaxLifetime int64) { var err error sessionOptions = prepareOptions(options) + sessionConnMaxLifetime = connMaxLifetime sessionManager, err = ms.NewManager(options.Provider, *options) if err != nil { panic(err) diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index f449bec5849..2f17402b80c 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -40,7 +40,7 @@ func GetAlertById(query *m.GetAlertByIdQuery) error { func GetAllAlertQueryHandler(query *m.GetAllAlertsQuery) error { var alerts []*m.Alert - err := x.Sql("select * from alert").Find(&alerts) + err := x.SQL("select * from alert").Find(&alerts) if err != nil { return err } @@ -60,6 +60,10 @@ func deleteAlertByIdInternal(alertId int64, reason string, sess *DBSession) erro return err } + if _, err := sess.Exec("DELETE FROM alert_notification_state WHERE alert_id = ?", alertId); err != nil { + return err + } + return nil } @@ -73,6 +77,7 @@ func HandleAlertsQuery(query *m.GetAlertsQuery) error { alert.name, alert.state, alert.new_state_date, + alert.eval_data, alert.eval_date, alert.execution_error, dashboard.uid as dashboard_uid, @@ -82,8 +87,16 @@ func HandleAlertsQuery(query *m.GetAlertsQuery) error { builder.Write(`WHERE alert.org_id = ?`, query.OrgId) - if query.DashboardId != 0 { - builder.Write(` AND alert.dashboard_id = ?`, query.DashboardId) + if len(strings.TrimSpace(query.Query)) > 0 { + builder.Write(" AND alert.name "+dialect.LikeStr()+" ?", "%"+query.Query+"%") + } + + if len(query.DashboardIDs) > 0 { + builder.sql.WriteString(` AND alert.dashboard_id IN (?` + strings.Repeat(",?", len(query.DashboardIDs)-1) + `) `) + + for _, dbID := range query.DashboardIDs { + builder.AddParams(dbID) + } } if query.PanelId != 0 { @@ -108,13 +121,13 @@ func HandleAlertsQuery(query *m.GetAlertsQuery) error { } if query.User.OrgRole != m.ROLE_ADMIN { - builder.writeDashboardPermissionFilter(query.User, m.PERMISSION_EDIT) + builder.writeDashboardPermissionFilter(query.User, m.PERMISSION_VIEW) } builder.Write(" ORDER BY name ASC") if query.Limit != 0 { - builder.Write(" LIMIT ?", query.Limit) + builder.Write(dialect.Limit(query.Limit)) } alerts := make([]*m.AlertListItemDTO, 0) @@ -181,7 +194,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS alert.Updated = timeNow() alert.State = alertToUpdate.State sess.MustCols("message") - _, err := sess.Id(alert.Id).Update(alert) + _, err := sess.ID(alert.Id).Update(alert) if err != nil { return err } @@ -240,7 +253,7 @@ func SetAlertState(cmd *m.SetAlertStateCommand) error { return inTransaction(func(sess *DBSession) error { alert := m.Alert{} - if has, err := sess.Id(cmd.AlertId).Get(&alert); err != nil { + if has, err := sess.ID(cmd.AlertId).Get(&alert); err != nil { return err } else if !has { return fmt.Errorf("Could not find alert") @@ -255,7 +268,7 @@ func SetAlertState(cmd *m.SetAlertStateCommand) error { } alert.State = cmd.State - alert.StateChanges += 1 + alert.StateChanges++ alert.NewStateDate = timeNow() alert.EvalData = cmd.EvalData @@ -266,6 +279,8 @@ func SetAlertState(cmd *m.SetAlertStateCommand) error { } sess.ID(alert.Id).Update(&alert) + + cmd.Result = alert return nil }) } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index ae691c7166c..afe6269510f 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -2,6 +2,8 @@ package sqlstore import ( "bytes" + "context" + "errors" "fmt" "strings" "time" @@ -17,14 +19,19 @@ func init() { bus.AddHandler("sql", DeleteAlertNotification) bus.AddHandler("sql", GetAlertNotificationsToSend) bus.AddHandler("sql", GetAllAlertNotifications) + bus.AddHandlerCtx("sql", GetOrCreateAlertNotificationState) + bus.AddHandlerCtx("sql", SetAlertNotificationStateToCompleteCommand) + bus.AddHandlerCtx("sql", SetAlertNotificationStateToPendingCommand) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { return inTransaction(func(sess *DBSession) error { sql := "DELETE FROM alert_notification WHERE alert_notification.org_id = ? AND alert_notification.id = ?" - _, err := sess.Exec(sql, cmd.OrgId, cmd.Id) + if _, err := sess.Exec(sql, cmd.OrgId, cmd.Id); err != nil { + return err + } - if err != nil { + if _, err := sess.Exec("DELETE FROM alert_notification_state WHERE alert_notification_state.org_id = ? AND alert_notification_state.notifier_id = ?", cmd.OrgId, cmd.Id); err != nil { return err } @@ -58,7 +65,10 @@ func GetAlertNotificationsToSend(query *m.GetAlertNotificationsToSendQuery) erro alert_notification.created, alert_notification.updated, alert_notification.settings, - alert_notification.is_default + alert_notification.is_default, + alert_notification.disable_resolve_message, + alert_notification.send_reminder, + alert_notification.frequency FROM alert_notification `) @@ -96,7 +106,10 @@ func getAlertNotificationInternal(query *m.GetAlertNotificationsQuery, sess *DBS alert_notification.created, alert_notification.updated, alert_notification.settings, - alert_notification.is_default + alert_notification.is_default, + alert_notification.disable_resolve_message, + alert_notification.send_reminder, + alert_notification.frequency FROM alert_notification `) @@ -116,7 +129,7 @@ func getAlertNotificationInternal(query *m.GetAlertNotificationsQuery, sess *DBS } results := make([]*m.AlertNotification, 0) - if err := sess.Sql(sql.String(), params...).Find(&results); err != nil { + if err := sess.SQL(sql.String(), params...).Find(&results); err != nil { return err } @@ -142,17 +155,32 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification name %s already exists", cmd.Name) } - alertNotification := &m.AlertNotification{ - OrgId: cmd.OrgId, - Name: cmd.Name, - Type: cmd.Type, - Settings: cmd.Settings, - Created: time.Now(), - Updated: time.Now(), - IsDefault: cmd.IsDefault, + var frequency time.Duration + if cmd.SendReminder { + if cmd.Frequency == "" { + return m.ErrNotificationFrequencyNotFound + } + + frequency, err = time.ParseDuration(cmd.Frequency) + if err != nil { + return err + } } - if _, err = sess.Insert(alertNotification); err != nil { + alertNotification := &m.AlertNotification{ + OrgId: cmd.OrgId, + Name: cmd.Name, + Type: cmd.Type, + Settings: cmd.Settings, + SendReminder: cmd.SendReminder, + DisableResolveMessage: cmd.DisableResolveMessage, + Frequency: frequency, + Created: time.Now(), + Updated: time.Now(), + IsDefault: cmd.IsDefault, + } + + if _, err = sess.MustCols("send_reminder").Insert(alertNotification); err != nil { return err } @@ -184,16 +212,152 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.Name = cmd.Name current.Type = cmd.Type current.IsDefault = cmd.IsDefault + current.SendReminder = cmd.SendReminder + current.DisableResolveMessage = cmd.DisableResolveMessage - sess.UseBool("is_default") + if current.SendReminder { + if cmd.Frequency == "" { + return m.ErrNotificationFrequencyNotFound + } + + frequency, err := time.ParseDuration(cmd.Frequency) + if err != nil { + return err + } + + current.Frequency = frequency + } + + sess.UseBool("is_default", "send_reminder", "disable_resolve_message") if affected, err := sess.ID(cmd.Id).Update(current); err != nil { return err } else if affected == 0 { - return fmt.Errorf("Could not find alert notification") + return fmt.Errorf("Could not update alert notification") } cmd.Result = ¤t return nil }) } + +func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { + version := cmd.Version + var current m.AlertNotificationState + sess.ID(cmd.Id).Get(¤t) + + newVersion := cmd.Version + 1 + + sql := `UPDATE alert_notification_state SET + state = ?, + version = ?, + updated_at = ? + WHERE + id = ?` + + _, err := sess.Exec(sql, m.AlertNotificationStateCompleted, newVersion, timeNow().Unix(), cmd.Id) + + if err != nil { + return err + } + + if current.Version != version { + sqlog.Error("notification state out of sync. the notification is marked as complete but has been modified between set as pending and completion.", "notifierId", current.NotifierId) + } + + return nil + }) +} + +func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error { + return withDbSession(ctx, func(sess *DBSession) error { + newVersion := cmd.Version + 1 + sql := `UPDATE alert_notification_state SET + state = ?, + version = ?, + updated_at = ?, + alert_rule_state_updated_version = ? + WHERE + id = ? AND + (version = ? OR alert_rule_state_updated_version < ?)` + + res, err := sess.Exec(sql, + m.AlertNotificationStatePending, + newVersion, + timeNow().Unix(), + cmd.AlertRuleStateUpdatedVersion, + cmd.Id, + cmd.Version, + cmd.AlertRuleStateUpdatedVersion) + + if err != nil { + return err + } + + affected, _ := res.RowsAffected() + if affected == 0 { + return m.ErrAlertNotificationStateVersionConflict + } + + cmd.ResultVersion = newVersion + + return nil + }) +} + +func GetOrCreateAlertNotificationState(ctx context.Context, cmd *m.GetOrCreateNotificationStateQuery) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { + nj := &m.AlertNotificationState{} + + exist, err := getAlertNotificationState(sess, cmd, nj) + + // if exists, return it, otherwise create it with default values + if err != nil { + return err + } + + if exist { + cmd.Result = nj + return nil + } + + notificationState := &m.AlertNotificationState{ + OrgId: cmd.OrgId, + AlertId: cmd.AlertId, + NotifierId: cmd.NotifierId, + State: m.AlertNotificationStateUnknown, + UpdatedAt: timeNow().Unix(), + } + + if _, err := sess.Insert(notificationState); err != nil { + if dialect.IsUniqueConstraintViolation(err) { + exist, err = getAlertNotificationState(sess, cmd, nj) + + if err != nil { + return err + } + + if !exist { + return errors.New("Should not happen") + } + + cmd.Result = nj + return nil + } + + return err + } + + cmd.Result = notificationState + return nil + }) +} + +func getAlertNotificationState(sess *DBSession, cmd *m.GetOrCreateNotificationStateQuery, nj *m.AlertNotificationState) (bool, error) { + return sess. + Where("alert_notification_state.org_id = ?", cmd.OrgId). + Where("alert_notification_state.alert_id = ?", cmd.AlertId). + Where("alert_notification_state.notifier_id = ?", cmd.NotifierId). + Get(nj) +} diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index d37062fb58f..629a6292eb5 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -1,44 +1,225 @@ package sqlstore import ( - "fmt" + "context" "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" - m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" ) func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Testing Alert notification sql access", t, func() { InitTestDB(t) - var err error + + Convey("Alert notification state", func() { + var alertID int64 = 7 + var orgID int64 = 5 + var notifierID int64 = 10 + oldTimeNow := timeNow + now := time.Date(2018, 9, 30, 0, 0, 0, 0, time.UTC) + timeNow = func() time.Time { return now } + + Convey("Get no existing state should create a new state", func() { + query := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err := GetOrCreateAlertNotificationState(context.Background(), query) + So(err, ShouldBeNil) + So(query.Result, ShouldNotBeNil) + So(query.Result.State, ShouldEqual, "unknown") + So(query.Result.Version, ShouldEqual, 0) + So(query.Result.UpdatedAt, ShouldEqual, now.Unix()) + + Convey("Get existing state should not create a new state", func() { + query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err := GetOrCreateAlertNotificationState(context.Background(), query2) + So(err, ShouldBeNil) + So(query2.Result, ShouldNotBeNil) + So(query2.Result.Id, ShouldEqual, query.Result.Id) + So(query2.Result.UpdatedAt, ShouldEqual, now.Unix()) + }) + + Convey("Update existing state to pending with correct version should update database", func() { + s := *query.Result + + cmd := models.SetAlertNotificationStateToPendingCommand{ + Id: s.Id, + Version: s.Version, + AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion, + } + + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldBeNil) + So(cmd.ResultVersion, ShouldEqual, 1) + + query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetOrCreateAlertNotificationState(context.Background(), query2) + So(err, ShouldBeNil) + So(query2.Result.Version, ShouldEqual, 1) + So(query2.Result.State, ShouldEqual, models.AlertNotificationStatePending) + So(query2.Result.UpdatedAt, ShouldEqual, now.Unix()) + + Convey("Update existing state to completed should update database", func() { + s := *query.Result + setStateCmd := models.SetAlertNotificationStateToCompleteCommand{ + Id: s.Id, + Version: cmd.ResultVersion, + } + err := SetAlertNotificationStateToCompleteCommand(context.Background(), &setStateCmd) + So(err, ShouldBeNil) + + query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetOrCreateAlertNotificationState(context.Background(), query3) + So(err, ShouldBeNil) + So(query3.Result.Version, ShouldEqual, 2) + So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) + So(query3.Result.UpdatedAt, ShouldEqual, now.Unix()) + }) + + Convey("Update existing state to completed should update database. regardless of version", func() { + s := *query.Result + unknownVersion := int64(1000) + cmd := models.SetAlertNotificationStateToCompleteCommand{ + Id: s.Id, + Version: unknownVersion, + } + err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) + So(err, ShouldBeNil) + + query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} + err = GetOrCreateAlertNotificationState(context.Background(), query3) + So(err, ShouldBeNil) + So(query3.Result.Version, ShouldEqual, unknownVersion+1) + So(query3.Result.State, ShouldEqual, models.AlertNotificationStateCompleted) + So(query3.Result.UpdatedAt, ShouldEqual, now.Unix()) + }) + }) + + Convey("Update existing state to pending with incorrect version should return version mismatch error", func() { + s := *query.Result + s.Version = 1000 + cmd := models.SetAlertNotificationStateToPendingCommand{ + Id: s.NotifierId, + Version: s.Version, + AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion, + } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) + }) + + Convey("Updating existing state to pending with incorrect version since alert rule state update version is higher", func() { + s := *query.Result + cmd := models.SetAlertNotificationStateToPendingCommand{ + Id: s.Id, + Version: s.Version, + AlertRuleStateUpdatedVersion: 1000, + } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldBeNil) + + So(cmd.ResultVersion, ShouldEqual, 1) + }) + + Convey("different version and same alert state change version should return error", func() { + s := *query.Result + s.Version = 1000 + cmd := models.SetAlertNotificationStateToPendingCommand{ + Id: s.Id, + Version: s.Version, + AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion, + } + err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) + So(err, ShouldNotBeNil) + }) + }) + + Reset(func() { + timeNow = oldTimeNow + }) + }) Convey("Alert notifications should be empty", func() { - cmd := &m.GetAlertNotificationsQuery{ + cmd := &models.GetAlertNotificationsQuery{ OrgId: 2, Name: "email", } err := GetAlertNotifications(cmd) - fmt.Printf("errror %v", err) So(err, ShouldBeNil) So(cmd.Result, ShouldBeNil) }) - Convey("Can save Alert Notification", func() { - cmd := &m.CreateAlertNotificationCommand{ - Name: "ops", - Type: "email", - OrgId: 1, - Settings: simplejson.New(), + Convey("Cannot save alert notifier with send reminder = true", func() { + cmd := &models.CreateAlertNotificationCommand{ + Name: "ops", + Type: "email", + OrgId: 1, + SendReminder: true, + Settings: simplejson.New(), } - err = CreateAlertNotificationCommand(cmd) + Convey("and missing frequency", func() { + err := CreateAlertNotificationCommand(cmd) + So(err, ShouldEqual, models.ErrNotificationFrequencyNotFound) + }) + + Convey("invalid frequency", func() { + cmd.Frequency = "invalid duration" + + err := CreateAlertNotificationCommand(cmd) + So(err.Error(), ShouldEqual, "time: invalid duration invalid duration") + }) + }) + + Convey("Cannot update alert notifier with send reminder = false", func() { + cmd := &models.CreateAlertNotificationCommand{ + Name: "ops update", + Type: "email", + OrgId: 1, + SendReminder: false, + Settings: simplejson.New(), + } + + err := CreateAlertNotificationCommand(cmd) + So(err, ShouldBeNil) + + updateCmd := &models.UpdateAlertNotificationCommand{ + Id: cmd.Result.Id, + SendReminder: true, + } + + Convey("and missing frequency", func() { + err := UpdateAlertNotification(updateCmd) + So(err, ShouldEqual, models.ErrNotificationFrequencyNotFound) + }) + + Convey("invalid frequency", func() { + updateCmd.Frequency = "invalid duration" + + err := UpdateAlertNotification(updateCmd) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "time: invalid duration invalid duration") + }) + }) + + Convey("Can save Alert Notification", func() { + cmd := &models.CreateAlertNotificationCommand{ + Name: "ops", + Type: "email", + OrgId: 1, + SendReminder: true, + Frequency: "10s", + Settings: simplejson.New(), + } + + err := CreateAlertNotificationCommand(cmd) So(err, ShouldBeNil) So(cmd.Result.Id, ShouldNotEqual, 0) So(cmd.Result.OrgId, ShouldNotEqual, 0) So(cmd.Result.Type, ShouldEqual, "email") + So(cmd.Result.Frequency, ShouldEqual, 10*time.Second) + So(cmd.Result.DisableResolveMessage, ShouldBeFalse) Convey("Cannot save Alert Notification with the same name", func() { err = CreateAlertNotificationCommand(cmd) @@ -46,26 +227,45 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can update alert notification", func() { - newCmd := &m.UpdateAlertNotificationCommand{ - Name: "NewName", - Type: "webhook", - OrgId: cmd.Result.OrgId, - Settings: simplejson.New(), - Id: cmd.Result.Id, + newCmd := &models.UpdateAlertNotificationCommand{ + Name: "NewName", + Type: "webhook", + OrgId: cmd.Result.OrgId, + SendReminder: true, + DisableResolveMessage: true, + Frequency: "60s", + Settings: simplejson.New(), + Id: cmd.Result.Id, } err := UpdateAlertNotification(newCmd) So(err, ShouldBeNil) So(newCmd.Result.Name, ShouldEqual, "NewName") + So(newCmd.Result.Frequency, ShouldEqual, 60*time.Second) + So(newCmd.Result.DisableResolveMessage, ShouldBeTrue) + }) + + Convey("Can update alert notification to disable sending of reminders", func() { + newCmd := &models.UpdateAlertNotificationCommand{ + Name: "NewName", + Type: "webhook", + OrgId: cmd.Result.OrgId, + SendReminder: false, + Settings: simplejson.New(), + Id: cmd.Result.Id, + } + err := UpdateAlertNotification(newCmd) + So(err, ShouldBeNil) + So(newCmd.Result.SendReminder, ShouldBeFalse) }) }) Convey("Can search using an array of ids", func() { - cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, Settings: simplejson.New()} - cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, Settings: simplejson.New()} - cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, Settings: simplejson.New()} - cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, Settings: simplejson.New()} + cmd1 := models.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd2 := models.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd3 := models.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd4 := models.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} - otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, Settings: simplejson.New()} + otherOrg := models.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil) So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil) @@ -74,7 +274,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(CreateAlertNotificationCommand(&otherOrg), ShouldBeNil) Convey("search", func() { - query := &m.GetAlertNotificationsToSendQuery{ + query := &models.GetAlertNotificationsToSendQuery{ Ids: []int64{cmd1.Result.Id, cmd2.Result.Id, 112341231}, OrgId: 1, } @@ -85,7 +285,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("all", func() { - query := &m.GetAllAlertNotificationsQuery{ + query := &models.GetAllAlertNotificationsQuery{ OrgId: 1, } diff --git a/pkg/services/sqlstore/alert_test.go b/pkg/services/sqlstore/alert_test.go index 296d16c2f45..d97deb45f0e 100644 --- a/pkg/services/sqlstore/alert_test.go +++ b/pkg/services/sqlstore/alert_test.go @@ -2,18 +2,18 @@ package sqlstore import ( "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" - "time" ) func mockTimeNow() { var timeSeed int64 timeNow = func() time.Time { fakeNow := time.Unix(timeSeed, 0) - timeSeed += 1 + timeSeed++ return fakeNow } } @@ -30,7 +30,7 @@ func TestAlertingDataAccess(t *testing.T) { InitTestDB(t) testDash := insertTestDashboard("dashboard with alerts", 1, 0, false, "alert") - + evalData, _ := simplejson.NewJson([]byte(`{"test": "test"}`)) items := []*m.Alert{ { PanelId: 1, @@ -40,6 +40,7 @@ func TestAlertingDataAccess(t *testing.T) { Message: "Alerting message", Settings: simplejson.New(), Frequency: 1, + EvalData: evalData, }, } @@ -99,21 +100,32 @@ func TestAlertingDataAccess(t *testing.T) { }) Convey("Can read properties", func() { - alertQuery := m.GetAlertsQuery{DashboardId: testDash.Id, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + alertQuery := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&alertQuery) alert := alertQuery.Result[0] So(err2, ShouldBeNil) + So(alert.Id, ShouldBeGreaterThan, 0) + So(alert.DashboardId, ShouldEqual, testDash.Id) + So(alert.PanelId, ShouldEqual, 1) So(alert.Name, ShouldEqual, "Alerting title") So(alert.State, ShouldEqual, "pending") + So(alert.NewStateDate, ShouldNotBeNil) + So(alert.EvalData, ShouldNotBeNil) + So(alert.EvalData.Get("test").MustString(), ShouldEqual, "test") + So(alert.EvalDate, ShouldNotBeNil) + So(alert.ExecutionError, ShouldEqual, "") + So(alert.DashboardUid, ShouldNotBeNil) + So(alert.DashboardSlug, ShouldEqual, "dashboard-with-alerts") }) Convey("Viewer cannot read alerts", func() { - alertQuery := m.GetAlertsQuery{DashboardId: testDash.Id, PanelId: 1, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_VIEWER}} + viewerUser := &m.SignedInUser{OrgRole: m.ROLE_VIEWER, OrgId: 1} + alertQuery := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, PanelId: 1, OrgId: 1, User: viewerUser} err2 := HandleAlertsQuery(&alertQuery) So(err2, ShouldBeNil) - So(alertQuery.Result, ShouldHaveLength, 0) + So(alertQuery.Result, ShouldHaveLength, 1) }) Convey("Alerts with same dashboard id and panel id should update", func() { @@ -134,7 +146,7 @@ func TestAlertingDataAccess(t *testing.T) { }) Convey("Alerts should be updated", func() { - query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + query := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&query) So(err2, ShouldBeNil) @@ -183,7 +195,7 @@ func TestAlertingDataAccess(t *testing.T) { Convey("Should save 3 dashboards", func() { So(err, ShouldBeNil) - queryForDashboard := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + queryForDashboard := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&queryForDashboard) So(err2, ShouldBeNil) @@ -197,7 +209,7 @@ func TestAlertingDataAccess(t *testing.T) { err = SaveAlerts(&cmd) Convey("should delete the missing alert", func() { - query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + query := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&query) So(err2, ShouldBeNil) So(len(query.Result), ShouldEqual, 2) @@ -232,7 +244,7 @@ func TestAlertingDataAccess(t *testing.T) { So(err, ShouldBeNil) Convey("Alerts should be removed", func() { - query := m.GetAlertsQuery{DashboardId: testDash.Id, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} + query := m.GetAlertsQuery{DashboardIDs: []int64{testDash.Id}, OrgId: 1, User: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}} err2 := HandleAlertsQuery(&query) So(testDash.Id, ShouldEqual, 1) diff --git a/pkg/services/sqlstore/annotation.go b/pkg/services/sqlstore/annotation.go index 76f1819a18c..274481baeca 100644 --- a/pkg/services/sqlstore/annotation.go +++ b/pkg/services/sqlstore/annotation.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/annotations" @@ -17,18 +18,24 @@ func (r *SqlAnnotationRepo) Save(item *annotations.Item) error { return inTransaction(func(sess *DBSession) error { tags := models.ParseTagPairs(item.Tags) item.Tags = models.JoinTagPairs(tags) + item.Created = time.Now().UnixNano() / int64(time.Millisecond) + item.Updated = item.Created + if item.Epoch == 0 { + item.Epoch = item.Created + } + if _, err := sess.Table("annotation").Insert(item); err != nil { return err } if item.Tags != nil { - if tags, err := r.ensureTagsExist(sess, tags); err != nil { + tags, err := r.ensureTagsExist(sess, tags) + if err != nil { return err - } else { - for _, tag := range tags { - if _, err := sess.Exec("INSERT INTO annotation_tag (annotation_id, tag_id) VALUES(?,?)", item.Id, tag.Id); err != nil { - return err - } + } + for _, tag := range tags { + if _, err := sess.Exec("INSERT INTO annotation_tag (annotation_id, tag_id) VALUES(?,?)", item.Id, tag.Id); err != nil { + return err } } } @@ -43,7 +50,7 @@ func (r *SqlAnnotationRepo) ensureTagsExist(sess *DBSession, tags []*models.Tag) var existingTag models.Tag // check if it exists - if exists, err := sess.Table("tag").Where("`key`=? AND `value`=?", tag.Key, tag.Value).Get(&existingTag); err != nil { + if exists, err := sess.Table("tag").Where(dialect.Quote("key")+"=? AND "+dialect.Quote("value")+"=?", tag.Key, tag.Value).Get(&existingTag); err != nil { return nil, err } else if exists { tag.Id = existingTag.Id @@ -79,6 +86,7 @@ func (r *SqlAnnotationRepo) Update(item *annotations.Item) error { return errors.New("Annotation not found") } + existing.Updated = time.Now().UnixNano() / int64(time.Millisecond) existing.Epoch = item.Epoch existing.Text = item.Text if item.RegionId != 0 { @@ -86,27 +94,24 @@ func (r *SqlAnnotationRepo) Update(item *annotations.Item) error { } if item.Tags != nil { - if tags, err := r.ensureTagsExist(sess, models.ParseTagPairs(item.Tags)); err != nil { + tags, err := r.ensureTagsExist(sess, models.ParseTagPairs(item.Tags)) + if err != nil { return err - } else { - if _, err := sess.Exec("DELETE FROM annotation_tag WHERE annotation_id = ?", existing.Id); err != nil { + } + if _, err := sess.Exec("DELETE FROM annotation_tag WHERE annotation_id = ?", existing.Id); err != nil { + return err + } + for _, tag := range tags { + if _, err := sess.Exec("INSERT INTO annotation_tag (annotation_id, tag_id) VALUES(?,?)", existing.Id, tag.Id); err != nil { return err } - for _, tag := range tags { - if _, err := sess.Exec("INSERT INTO annotation_tag (annotation_id, tag_id) VALUES(?,?)", existing.Id, tag.Id); err != nil { - return err - } - } } } existing.Tags = item.Tags - if _, err := sess.Table("annotation").Id(existing.Id).Cols("epoch", "text", "region_id", "tags").Update(existing); err != nil { - return err - } - - return nil + _, err = sess.Table("annotation").ID(existing.Id).Cols("epoch", "text", "region_id", "updated", "tags").Update(existing) + return err }) } @@ -127,6 +132,8 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I annotation.text, annotation.tags, annotation.data, + annotation.created, + annotation.updated, usr.email, usr.login, alert.name as alert_name @@ -139,7 +146,7 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I params = append(params, query.OrgId) if query.AnnotationId != 0 { - fmt.Print("annotation query") + // fmt.Print("annotation query") sql.WriteString(` AND annotation.id = ?`) params = append(params, query.AnnotationId) } @@ -164,6 +171,11 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I params = append(params, query.PanelId) } + if query.UserId != 0 { + sql.WriteString(` AND annotation.user_id = ?`) + params = append(params, query.UserId) + } + if query.From > 0 && query.To > 0 { sql.WriteString(` AND annotation.epoch BETWEEN ? AND ?`) params = append(params, query.From, query.To) @@ -171,6 +183,8 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I if query.Type == "alert" { sql.WriteString(` AND annotation.alert_id > 0`) + } else if query.Type == "annotation" { + sql.WriteString(` AND annotation.alert_id = 0`) } if len(query.Tags) > 0 { @@ -179,10 +193,10 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I tags := models.ParseTagPairs(query.Tags) for _, tag := range tags { if tag.Value == "" { - keyValueFilters = append(keyValueFilters, "(tag.key = ?)") + keyValueFilters = append(keyValueFilters, "(tag."+dialect.Quote("key")+" = ?)") params = append(params, tag.Key) } else { - keyValueFilters = append(keyValueFilters, "(tag.key = ? AND tag.value = ?)") + keyValueFilters = append(keyValueFilters, "(tag."+dialect.Quote("key")+" = ? AND tag."+dialect.Quote("value")+" = ?)") params = append(params, tag.Key, tag.Value) } } @@ -197,19 +211,24 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I ) `, strings.Join(keyValueFilters, " OR ")) - sql.WriteString(fmt.Sprintf(" AND (%s) = %d ", tagsSubQuery, len(tags))) + if query.MatchAny { + sql.WriteString(fmt.Sprintf(" AND (%s) > 0 ", tagsSubQuery)) + } else { + sql.WriteString(fmt.Sprintf(" AND (%s) = %d ", tagsSubQuery, len(tags))) + } + } } if query.Limit == 0 { - query.Limit = 10 + query.Limit = 100 } - sql.WriteString(fmt.Sprintf(" ORDER BY epoch DESC LIMIT %v", query.Limit)) + sql.WriteString(" ORDER BY epoch DESC" + dialect.Limit(query.Limit)) items := make([]*annotations.ItemDTO, 0) - if err := x.Sql(sql.String(), params...).Find(&items); err != nil { + if err := x.SQL(sql.String(), params...).Find(&items); err != nil { return nil, err } @@ -224,18 +243,19 @@ func (r *SqlAnnotationRepo) Delete(params *annotations.DeleteParams) error { queryParams []interface{} ) + sqlog.Info("delete", "orgId", params.OrgId) if params.RegionId != 0 { - annoTagSql = "DELETE FROM annotation_tag WHERE annotation_id IN (SELECT id FROM annotation WHERE region_id = ?)" - sql = "DELETE FROM annotation WHERE region_id = ?" - queryParams = []interface{}{params.RegionId} + annoTagSql = "DELETE FROM annotation_tag WHERE annotation_id IN (SELECT id FROM annotation WHERE region_id = ? AND org_id = ?)" + sql = "DELETE FROM annotation WHERE region_id = ? AND org_id = ?" + queryParams = []interface{}{params.RegionId, params.OrgId} } else if params.Id != 0 { - annoTagSql = "DELETE FROM annotation_tag WHERE annotation_id IN (SELECT id FROM annotation WHERE id = ?)" - sql = "DELETE FROM annotation WHERE id = ?" - queryParams = []interface{}{params.Id} + annoTagSql = "DELETE FROM annotation_tag WHERE annotation_id IN (SELECT id FROM annotation WHERE id = ? AND org_id = ?)" + sql = "DELETE FROM annotation WHERE id = ? AND org_id = ?" + queryParams = []interface{}{params.Id, params.OrgId} } else { - annoTagSql = "DELETE FROM annotation_tag WHERE annotation_id IN (SELECT id FROM annotation WHERE dashboard_id = ? AND panel_id = ?)" - sql = "DELETE FROM annotation WHERE dashboard_id = ? AND panel_id = ?" - queryParams = []interface{}{params.DashboardId, params.PanelId} + annoTagSql = "DELETE FROM annotation_tag WHERE annotation_id IN (SELECT id FROM annotation WHERE dashboard_id = ? AND panel_id = ? AND org_id = ?)" + sql = "DELETE FROM annotation WHERE dashboard_id = ? AND panel_id = ? AND org_id = ?" + queryParams = []interface{}{params.DashboardId, params.PanelId, params.OrgId} } if _, err := sess.Exec(annoTagSql, queryParams...); err != nil { diff --git a/pkg/services/sqlstore/annotation_test.go b/pkg/services/sqlstore/annotation_test.go index d5cee110b9a..d3459527e7d 100644 --- a/pkg/services/sqlstore/annotation_test.go +++ b/pkg/services/sqlstore/annotation_test.go @@ -10,12 +10,18 @@ import ( ) func TestSavingTags(t *testing.T) { + InitTestDB(t) + Convey("Testing annotation saving/loading", t, func() { - InitTestDB(t) repo := SqlAnnotationRepo{} Convey("Can save tags", func() { + Reset(func() { + _, err := x.Exec("DELETE FROM annotation_tag WHERE 1=1") + So(err, ShouldBeNil) + }) + tagPairs := []*models.Tag{ {Key: "outage"}, {Key: "type", Value: "outage"}, @@ -31,12 +37,19 @@ func TestSavingTags(t *testing.T) { } func TestAnnotations(t *testing.T) { - Convey("Testing annotation saving/loading", t, func() { - InitTestDB(t) + InitTestDB(t) + Convey("Testing annotation saving/loading", t, func() { repo := SqlAnnotationRepo{} Convey("Can save annotation", func() { + Reset(func() { + _, err := x.Exec("DELETE FROM annotation WHERE 1=1") + So(err, ShouldBeNil) + _, err = x.Exec("DELETE FROM annotation_tag WHERE 1=1") + So(err, ShouldBeNil) + }) + annotation := &annotations.Item{ OrgId: 1, UserId: 1, @@ -65,7 +78,31 @@ func TestAnnotations(t *testing.T) { So(err, ShouldBeNil) So(annotation2.Id, ShouldBeGreaterThan, 0) - Convey("Can query for annotation", func() { + globalAnnotation1 := &annotations.Item{ + OrgId: 1, + UserId: 1, + Text: "deploy", + Type: "", + Epoch: 15, + Tags: []string{"deploy"}, + } + err = repo.Save(globalAnnotation1) + So(err, ShouldBeNil) + So(globalAnnotation1.Id, ShouldBeGreaterThan, 0) + + globalAnnotation2 := &annotations.Item{ + OrgId: 1, + UserId: 1, + Text: "rollback", + Type: "", + Epoch: 17, + Tags: []string{"rollback"}, + } + err = repo.Save(globalAnnotation2) + So(err, ShouldBeNil) + So(globalAnnotation2.Id, ShouldBeGreaterThan, 0) + + Convey("Can query for annotation by dashboard id", func() { items, err := repo.Find(&annotations.ItemQuery{ OrgId: 1, DashboardId: 1, @@ -79,6 +116,12 @@ func TestAnnotations(t *testing.T) { Convey("Can read tags", func() { So(items[0].Tags, ShouldResemble, []string{"outage", "error", "type:outage", "server:server-1"}) }) + + Convey("Has created and updated values", func() { + So(items[0].Created, ShouldBeGreaterThan, 0) + So(items[0].Updated, ShouldBeGreaterThan, 0) + So(items[0].Updated, ShouldEqual, items[0].Created) + }) }) Convey("Can query for annotation by id", func() { @@ -146,7 +189,7 @@ func TestAnnotations(t *testing.T) { OrgId: 1, DashboardId: 1, From: 1, - To: 15, + To: 15, //this will exclude the second test annotation Tags: []string{"outage", "error"}, }) @@ -154,6 +197,19 @@ func TestAnnotations(t *testing.T) { So(items, ShouldHaveLength, 1) }) + Convey("Should find two annotations using partial match", func() { + items, err := repo.Find(&annotations.ItemQuery{ + OrgId: 1, + From: 1, + To: 25, + MatchAny: true, + Tags: []string{"rollback", "deploy"}, + }) + + So(err, ShouldBeNil) + So(items, ShouldHaveLength, 2) + }) + Convey("Should find one when all key value tag filters does match", func() { items, err := repo.Find(&annotations.ItemQuery{ OrgId: 1, @@ -231,6 +287,10 @@ func TestAnnotations(t *testing.T) { So(items[0].Tags, ShouldResemble, []string{"newtag1", "newtag2"}) So(items[0].Text, ShouldEqual, "something new") }) + + Convey("Updated time has increased", func() { + So(items[0].Updated, ShouldBeGreaterThan, items[0].Created) + }) }) Convey("Can delete annotation", func() { @@ -245,7 +305,8 @@ func TestAnnotations(t *testing.T) { annotationId := items[0].Id - err = repo.Delete(&annotations.DeleteParams{Id: annotationId}) + err = repo.Delete(&annotations.DeleteParams{Id: annotationId, OrgId: 1}) + So(err, ShouldBeNil) items, err = repo.Find(query) So(err, ShouldBeNil) diff --git a/pkg/services/sqlstore/apikey.go b/pkg/services/sqlstore/apikey.go index 0532f636625..775d4cf6447 100644 --- a/pkg/services/sqlstore/apikey.go +++ b/pkg/services/sqlstore/apikey.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "time" "github.com/grafana/grafana/pkg/bus" @@ -11,7 +12,7 @@ func init() { bus.AddHandler("sql", GetApiKeys) bus.AddHandler("sql", GetApiKeyById) bus.AddHandler("sql", GetApiKeyByName) - bus.AddHandler("sql", DeleteApiKey) + bus.AddHandlerCtx("sql", DeleteApiKeyCtx) bus.AddHandler("sql", AddApiKey) } @@ -22,8 +23,8 @@ func GetApiKeys(query *m.GetApiKeysQuery) error { return sess.Find(&query.Result) } -func DeleteApiKey(cmd *m.DeleteApiKeyCommand) error { - return inTransaction(func(sess *DBSession) error { +func DeleteApiKeyCtx(ctx context.Context, cmd *m.DeleteApiKeyCommand) error { + return withDbSession(ctx, func(sess *DBSession) error { var rawSql = "DELETE FROM api_key WHERE id=? and org_id=?" _, err := sess.Exec(rawSql, cmd.Id, cmd.OrgId) return err @@ -55,7 +56,7 @@ func GetApiKeyById(query *m.GetApiKeyByIdQuery) error { if err != nil { return err - } else if has == false { + } else if !has { return m.ErrInvalidApiKey } @@ -69,7 +70,7 @@ func GetApiKeyByName(query *m.GetApiKeyByNameQuery) error { if err != nil { return err - } else if has == false { + } else if !has { return m.ErrInvalidApiKey } diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index 8a89c3d942c..bad46c10af4 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -24,6 +24,7 @@ func init() { bus.AddHandler("sql", GetDashboardPermissionsForUser) bus.AddHandler("sql", GetDashboardsBySlug) bus.AddHandler("sql", ValidateDashboardBeforeSave) + bus.AddHandler("sql", HasEditPermissionInFolders) } var generateNewUid func() string = util.GenerateShortUid @@ -63,7 +64,7 @@ func saveDashboard(sess *DBSession, cmd *m.SaveDashboardCommand) error { } // do not allow plugin dashboard updates without overwrite flag - if existing.PluginId != "" && cmd.Overwrite == false { + if existing.PluginId != "" && !cmd.Overwrite { return m.UpdatePluginDashboardError{PluginId: existing.PluginId} } } @@ -77,7 +78,7 @@ func saveDashboard(sess *DBSession, cmd *m.SaveDashboardCommand) error { } parentVersion := dash.Version - affectedRows := int64(0) + var affectedRows int64 var err error if dash.Id == 0 { @@ -172,7 +173,7 @@ func GetDashboard(query *m.GetDashboardQuery) error { if err != nil { return err - } else if has == false { + } else if !has { return m.ErrDashboardNotFound } @@ -224,7 +225,7 @@ func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSear var res []DashboardSearchProjection sql, params := sb.ToSql() - err := x.Sql(sql, params...).Find(&res) + err := x.SQL(sql, params...).Find(&res) if err != nil { return nil, err } @@ -294,10 +295,11 @@ func GetDashboardTags(query *m.GetDashboardTagsQuery) error { FROM dashboard INNER JOIN dashboard_tag on dashboard_tag.dashboard_id = dashboard.id WHERE dashboard.org_id=? - GROUP BY term` + GROUP BY term + ORDER BY term` query.Result = make([]*m.DashboardTagCloudItem, 0) - sess := x.Sql(sql, query.OrgId) + sess := x.SQL(sql, query.OrgId) err := sess.Find(&query.Result) return err } @@ -308,7 +310,7 @@ func DeleteDashboard(cmd *m.DeleteDashboardCommand) error { has, err := sess.Get(&dashboard) if err != nil { return err - } else if has == false { + } else if !has { return m.ErrDashboardNotFound } @@ -318,22 +320,41 @@ func DeleteDashboard(cmd *m.DeleteDashboardCommand) error { "DELETE FROM dashboard WHERE id = ?", "DELETE FROM playlist_item WHERE type = 'dashboard_by_id' AND value = ?", "DELETE FROM dashboard_version WHERE dashboard_id = ?", - "DELETE FROM dashboard WHERE folder_id = ?", "DELETE FROM annotation WHERE dashboard_id = ?", "DELETE FROM dashboard_provisioning WHERE dashboard_id = ?", } - for _, sql := range deletes { - _, err := sess.Exec(sql, dashboard.Id) + if dashboard.IsFolder { + deletes = append(deletes, "DELETE FROM dashboard_provisioning WHERE dashboard_id in (select id from dashboard where folder_id = ?)") + deletes = append(deletes, "DELETE FROM dashboard WHERE folder_id = ?") + + dashIds := []struct { + Id int64 + }{} + err := sess.SQL("select id from dashboard where folder_id = ?", dashboard.Id).Find(&dashIds) if err != nil { return err } + + for _, id := range dashIds { + if err := deleteAlertDefinition(id.Id, sess); err != nil { + return nil + } + } } if err := deleteAlertDefinition(dashboard.Id, sess); err != nil { return nil } + for _, sql := range deletes { + _, err := sess.Exec(sql, dashboard.Id) + + if err != nil { + return err + } + } + return nil }) } @@ -347,12 +368,7 @@ func GetDashboards(query *m.GetDashboardsQuery) error { err := x.In("id", query.DashboardIds).Find(&dashboards) query.Result = dashboards - - if err != nil { - return err - } - - return nil + return err } // GetDashboardPermissionsForUser returns the maximum permission the specified user has for a dashboard(s) @@ -416,7 +432,7 @@ func GetDashboardPermissionsForUser(query *m.GetDashboardPermissionsForUserQuery params = append(params, query.UserId) params = append(params, dialect.BooleanStr(false)) - err := x.Sql(sql, params...).Find(&query.Result) + err := x.SQL(sql, params...).Find(&query.Result) for _, p := range query.Result { p.PermissionName = p.Permission.String() @@ -431,12 +447,7 @@ func GetDashboardsByPluginId(query *m.GetDashboardsByPluginIdQuery) error { err := x.Where(whereExpr, query.OrgId, query.PluginId).Find(&dashboards) query.Result = dashboards - - if err != nil { - return err - } - - return nil + return err } type DashboardSlugDTO struct { @@ -451,7 +462,7 @@ func GetDashboardSlugById(query *m.GetDashboardSlugByIdQuery) error { if err != nil { return err - } else if exists == false { + } else if !exists { return m.ErrDashboardNotFound } @@ -479,7 +490,7 @@ func GetDashboardUIDById(query *m.GetDashboardRefByIdQuery) error { if err != nil { return err - } else if exists == false { + } else if !exists { return m.ErrDashboardNotFound } @@ -544,6 +555,10 @@ func getExistingDashboardByIdOrUidForUpdate(sess *DBSession, cmd *m.ValidateDash dash.SetId(existingByUid.Id) dash.SetUid(existingByUid.Uid) existing = existingByUid + + if !dash.IsFolder { + cmd.Result.IsParentFolderChanged = true + } } if (existing.IsFolder && !dash.IsFolder) || @@ -551,6 +566,10 @@ func getExistingDashboardByIdOrUidForUpdate(sess *DBSession, cmd *m.ValidateDash return m.ErrDashboardTypeMismatch } + if !dash.IsFolder && dash.FolderId != existing.FolderId { + cmd.Result.IsParentFolderChanged = true + } + // check for is someone else has written in between if dash.Version != existing.Version { if cmd.Overwrite { @@ -561,7 +580,7 @@ func getExistingDashboardByIdOrUidForUpdate(sess *DBSession, cmd *m.ValidateDash } // do not allow plugin dashboard updates without overwrite flag - if existing.PluginId != "" && cmd.Overwrite == false { + if existing.PluginId != "" && !cmd.Overwrite { return m.UpdatePluginDashboardError{PluginId: existing.PluginId} } @@ -586,6 +605,10 @@ func getExistingDashboardByTitleAndFolder(sess *DBSession, cmd *m.ValidateDashbo return m.ErrDashboardFolderWithSameNameAsDashboard } + if !dash.IsFolder && (dash.FolderId != existing.FolderId || dash.Id == 0) { + cmd.Result.IsParentFolderChanged = true + } + if cmd.Overwrite { dash.SetId(existing.Id) dash.SetUid(existing.Uid) @@ -599,6 +622,7 @@ func getExistingDashboardByTitleAndFolder(sess *DBSession, cmd *m.ValidateDashbo } func ValidateDashboardBeforeSave(cmd *m.ValidateDashboardBeforeSaveCommand) (err error) { + cmd.Result = &m.ValidateDashboardBeforeSaveResult{} return inTransaction(func(sess *DBSession) error { if err = getExistingDashboardByIdOrUidForUpdate(sess, cmd); err != nil { return err @@ -611,3 +635,27 @@ func ValidateDashboardBeforeSave(cmd *m.ValidateDashboardBeforeSaveCommand) (err return nil }) } + +func HasEditPermissionInFolders(query *m.HasEditPermissionInFoldersQuery) error { + if query.SignedInUser.HasRole(m.ROLE_EDITOR) { + query.Result = true + return nil + } + + builder := &SqlBuilder{} + builder.Write("SELECT COUNT(dashboard.id) AS count FROM dashboard WHERE dashboard.org_id = ? AND dashboard.is_folder = ?", query.SignedInUser.OrgId, dialect.BooleanStr(true)) + builder.writeDashboardPermissionFilter(query.SignedInUser, m.PERMISSION_EDIT) + + type folderCount struct { + Count int64 + } + + resp := make([]*folderCount, 0) + if err := x.SQL(builder.GetSqlString(), builder.params...).Find(&resp); err != nil { + return err + } + + query.Result = len(resp) > 0 && resp[0].Count > 0 + + return nil +} diff --git a/pkg/services/sqlstore/dashboard_acl.go b/pkg/services/sqlstore/dashboard_acl.go index ae91d1d41f3..0b195c4562b 100644 --- a/pkg/services/sqlstore/dashboard_acl.go +++ b/pkg/services/sqlstore/dashboard_acl.go @@ -35,10 +35,8 @@ func UpdateDashboardAcl(cmd *m.UpdateDashboardAclCommand) error { // Update dashboard HasAcl flag dashboard := m.Dashboard{HasAcl: true} - if _, err := sess.Cols("has_acl").Where("id=?", cmd.DashboardId).Update(&dashboard); err != nil { - return err - } - return nil + _, err = sess.Cols("has_acl").Where("id=?", cmd.DashboardId).Update(&dashboard) + return err }) } @@ -69,7 +67,8 @@ func GetDashboardAclInfoList(query *m.GetDashboardAclInfoListQuery) error { '' as title, '' as slug, '' as uid,` + - falseStr + ` AS is_folder + falseStr + ` AS is_folder,` + + falseStr + ` AS inherited FROM dashboard_acl as da WHERE da.dashboard_id = -1` query.Result = make([]*m.DashboardAclInfoDTO, 0) @@ -92,10 +91,12 @@ func GetDashboardAclInfoList(query *m.GetDashboardAclInfoListQuery) error { u.login AS user_login, u.email AS user_email, ug.name AS team, + ug.email AS team_email, d.title, d.slug, d.uid, - d.is_folder + d.is_folder, + CASE WHEN (da.dashboard_id = -1 AND d.folder_id > 0) OR da.dashboard_id = d.folder_id THEN ` + dialect.BooleanStr(true) + ` ELSE ` + falseStr + ` END AS inherited FROM dashboard as d LEFT JOIN dashboard folder on folder.id = d.folder_id LEFT JOIN dashboard_acl AS da ON diff --git a/pkg/services/sqlstore/dashboard_acl_test.go b/pkg/services/sqlstore/dashboard_acl_test.go index 8fbb9c0d813..a034a0565a3 100644 --- a/pkg/services/sqlstore/dashboard_acl_test.go +++ b/pkg/services/sqlstore/dashboard_acl_test.go @@ -26,6 +26,22 @@ func TestDashboardAclDataAccess(t *testing.T) { }) Convey("Given dashboard folder with default permissions", func() { + Convey("When reading folder acl should include default acl", func() { + query := m.GetDashboardAclInfoListQuery{DashboardId: savedFolder.Id, OrgId: 1} + + err := GetDashboardAclInfoList(&query) + So(err, ShouldBeNil) + + So(len(query.Result), ShouldEqual, 2) + defaultPermissionsId := -1 + So(query.Result[0].DashboardId, ShouldEqual, defaultPermissionsId) + So(*query.Result[0].Role, ShouldEqual, m.ROLE_VIEWER) + So(query.Result[0].Inherited, ShouldBeFalse) + So(query.Result[1].DashboardId, ShouldEqual, defaultPermissionsId) + So(*query.Result[1].Role, ShouldEqual, m.ROLE_EDITOR) + So(query.Result[1].Inherited, ShouldBeFalse) + }) + Convey("When reading dashboard acl should include acl for parent folder", func() { query := m.GetDashboardAclInfoListQuery{DashboardId: childDash.Id, OrgId: 1} @@ -36,8 +52,10 @@ func TestDashboardAclDataAccess(t *testing.T) { defaultPermissionsId := -1 So(query.Result[0].DashboardId, ShouldEqual, defaultPermissionsId) So(*query.Result[0].Role, ShouldEqual, m.ROLE_VIEWER) + So(query.Result[0].Inherited, ShouldBeTrue) So(query.Result[1].DashboardId, ShouldEqual, defaultPermissionsId) So(*query.Result[1].Role, ShouldEqual, m.ROLE_EDITOR) + So(query.Result[1].Inherited, ShouldBeTrue) }) }) @@ -94,7 +112,9 @@ func TestDashboardAclDataAccess(t *testing.T) { So(len(query.Result), ShouldEqual, 2) So(query.Result[0].DashboardId, ShouldEqual, savedFolder.Id) + So(query.Result[0].Inherited, ShouldBeTrue) So(query.Result[1].DashboardId, ShouldEqual, childDash.Id) + So(query.Result[1].Inherited, ShouldBeFalse) }) }) }) @@ -118,9 +138,12 @@ func TestDashboardAclDataAccess(t *testing.T) { So(len(query.Result), ShouldEqual, 3) So(query.Result[0].DashboardId, ShouldEqual, defaultPermissionsId) So(*query.Result[0].Role, ShouldEqual, m.ROLE_VIEWER) + So(query.Result[0].Inherited, ShouldBeTrue) So(query.Result[1].DashboardId, ShouldEqual, defaultPermissionsId) So(*query.Result[1].Role, ShouldEqual, m.ROLE_EDITOR) + So(query.Result[1].Inherited, ShouldBeTrue) So(query.Result[2].DashboardId, ShouldEqual, childDash.Id) + So(query.Result[2].Inherited, ShouldBeFalse) }) }) @@ -131,6 +154,7 @@ func TestDashboardAclDataAccess(t *testing.T) { DashboardId: savedFolder.Id, Permission: m.PERMISSION_EDIT, }) + So(err, ShouldBeNil) q1 := &m.GetDashboardAclInfoListQuery{DashboardId: savedFolder.Id, OrgId: 1} err = GetDashboardAclInfoList(q1) @@ -209,8 +233,10 @@ func TestDashboardAclDataAccess(t *testing.T) { defaultPermissionsId := -1 So(query.Result[0].DashboardId, ShouldEqual, defaultPermissionsId) So(*query.Result[0].Role, ShouldEqual, m.ROLE_VIEWER) + So(query.Result[0].Inherited, ShouldBeFalse) So(query.Result[1].DashboardId, ShouldEqual, defaultPermissionsId) So(*query.Result[1].Role, ShouldEqual, m.ROLE_EDITOR) + So(query.Result[1].Inherited, ShouldBeFalse) }) }) }) diff --git a/pkg/services/sqlstore/dashboard_folder_test.go b/pkg/services/sqlstore/dashboard_folder_test.go index ea8f1216706..cdd107c3e90 100644 --- a/pkg/services/sqlstore/dashboard_folder_test.go +++ b/pkg/services/sqlstore/dashboard_folder_test.go @@ -46,6 +46,7 @@ func TestDashboardFolderDataAccess(t *testing.T) { OrgId: 1, DashboardIds: []int64{folder.Id, dashInRoot.Id}, } err := SearchDashboards(query) + So(err, ShouldBeNil) So(len(query.Result), ShouldEqual, 1) So(query.Result[0].Id, ShouldEqual, dashInRoot.Id) @@ -220,7 +221,6 @@ func TestDashboardFolderDataAccess(t *testing.T) { }) Convey("Given two dashboard folders", func() { - folder1 := insertTestDashboard("1 test dash folder", 1, 0, true, "prod") folder2 := insertTestDashboard("2 test dash folder", 1, 0, true, "prod") insertTestDashboard("folder in another org", 2, 0, true, "prod") @@ -263,6 +263,15 @@ func TestDashboardFolderDataAccess(t *testing.T) { So(query.Result[1].DashboardId, ShouldEqual, folder2.Id) So(query.Result[1].Permission, ShouldEqual, m.PERMISSION_ADMIN) }) + + Convey("should have edit permission in folders", func() { + query := &m.HasEditPermissionInFoldersQuery{ + SignedInUser: &m.SignedInUser{UserId: adminUser.Id, OrgId: 1, OrgRole: m.ROLE_ADMIN}, + } + err := HasEditPermissionInFolders(query) + So(err, ShouldBeNil) + So(query.Result, ShouldBeTrue) + }) }) Convey("Editor users", func() { @@ -309,6 +318,14 @@ func TestDashboardFolderDataAccess(t *testing.T) { So(query.Result[0].Id, ShouldEqual, folder2.Id) }) + Convey("should have edit permission in folders", func() { + query := &m.HasEditPermissionInFoldersQuery{ + SignedInUser: &m.SignedInUser{UserId: editorUser.Id, OrgId: 1, OrgRole: m.ROLE_EDITOR}, + } + err := HasEditPermissionInFolders(query) + So(err, ShouldBeNil) + So(query.Result, ShouldBeTrue) + }) }) Convey("Viewer users", func() { @@ -352,6 +369,41 @@ func TestDashboardFolderDataAccess(t *testing.T) { So(len(query.Result), ShouldEqual, 1) So(query.Result[0].Id, ShouldEqual, folder1.Id) }) + + Convey("should not have edit permission in folders", func() { + query := &m.HasEditPermissionInFoldersQuery{ + SignedInUser: &m.SignedInUser{UserId: viewerUser.Id, OrgId: 1, OrgRole: m.ROLE_VIEWER}, + } + err := HasEditPermissionInFolders(query) + So(err, ShouldBeNil) + So(query.Result, ShouldBeFalse) + }) + + Convey("and admin permission is given for user with org role viewer in one dashboard folder", func() { + testHelperUpdateDashboardAcl(folder1.Id, m.DashboardAcl{DashboardId: folder1.Id, OrgId: 1, UserId: viewerUser.Id, Permission: m.PERMISSION_ADMIN}) + + Convey("should have edit permission in folders", func() { + query := &m.HasEditPermissionInFoldersQuery{ + SignedInUser: &m.SignedInUser{UserId: viewerUser.Id, OrgId: 1, OrgRole: m.ROLE_VIEWER}, + } + err := HasEditPermissionInFolders(query) + So(err, ShouldBeNil) + So(query.Result, ShouldBeTrue) + }) + }) + + Convey("and edit permission is given for user with org role viewer in one dashboard folder", func() { + testHelperUpdateDashboardAcl(folder1.Id, m.DashboardAcl{DashboardId: folder1.Id, OrgId: 1, UserId: viewerUser.Id, Permission: m.PERMISSION_EDIT}) + + Convey("should have edit permission in folders", func() { + query := &m.HasEditPermissionInFoldersQuery{ + SignedInUser: &m.SignedInUser{UserId: viewerUser.Id, OrgId: 1, OrgRole: m.ROLE_VIEWER}, + } + err := HasEditPermissionInFolders(query) + So(err, ShouldBeNil) + So(query.Result, ShouldBeTrue) + }) + }) }) }) }) diff --git a/pkg/services/sqlstore/dashboard_provisioning.go b/pkg/services/sqlstore/dashboard_provisioning.go index 69409c3b873..33fbb01c5b7 100644 --- a/pkg/services/sqlstore/dashboard_provisioning.go +++ b/pkg/services/sqlstore/dashboard_provisioning.go @@ -8,6 +8,7 @@ import ( func init() { bus.AddHandler("sql", GetProvisionedDashboardDataQuery) bus.AddHandler("sql", SaveProvisionedDashboard) + bus.AddHandler("sql", GetProvisionedDataByDashboardId) } type DashboardExtras struct { @@ -17,6 +18,19 @@ type DashboardExtras struct { Value string } +func GetProvisionedDataByDashboardId(cmd *models.IsDashboardProvisionedQuery) error { + result := &models.DashboardProvisioning{} + + exist, err := x.Where("dashboard_id = ?", cmd.DashboardId).Get(result) + if err != nil { + return err + } + + cmd.Result = exist + + return nil +} + func SaveProvisionedDashboard(cmd *models.SaveProvisionedDashboardCommand) error { return inTransaction(func(sess *DBSession) error { err := saveDashboard(sess, cmd.DashboardCmd) diff --git a/pkg/services/sqlstore/dashboard_provisioning_test.go b/pkg/services/sqlstore/dashboard_provisioning_test.go index b752173b67d..1b7a3976727 100644 --- a/pkg/services/sqlstore/dashboard_provisioning_test.go +++ b/pkg/services/sqlstore/dashboard_provisioning_test.go @@ -13,17 +13,30 @@ func TestDashboardProvisioningTest(t *testing.T) { Convey("Testing Dashboard provisioning", t, func() { InitTestDB(t) - saveDashboardCmd := &models.SaveDashboardCommand{ + folderCmd := &models.SaveDashboardCommand{ OrgId: 1, FolderId: 0, - IsFolder: false, + IsFolder: true, Dashboard: simplejson.NewFromAny(map[string]interface{}{ "id": nil, "title": "test dashboard", }), } - Convey("Saving dashboards with extras", func() { + err := SaveDashboard(folderCmd) + So(err, ShouldBeNil) + + saveDashboardCmd := &models.SaveDashboardCommand{ + OrgId: 1, + IsFolder: false, + FolderId: folderCmd.Result.Id, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "id": nil, + "title": "test dashboard", + }), + } + + Convey("Saving dashboards with provisioning meta data", func() { now := time.Now() cmd := &models.SaveProvisionedDashboardCommand{ @@ -50,6 +63,38 @@ func TestDashboardProvisioningTest(t *testing.T) { So(query.Result[0].DashboardId, ShouldEqual, dashId) So(query.Result[0].Updated, ShouldEqual, now.Unix()) }) + + Convey("Can query for one provisioned dashboard", func() { + query := &models.IsDashboardProvisionedQuery{DashboardId: cmd.Result.Id} + + err := GetProvisionedDataByDashboardId(query) + So(err, ShouldBeNil) + + So(query.Result, ShouldBeTrue) + }) + + Convey("Can query for none provisioned dashboard", func() { + query := &models.IsDashboardProvisionedQuery{DashboardId: 3000} + + err := GetProvisionedDataByDashboardId(query) + So(err, ShouldBeNil) + So(query.Result, ShouldBeFalse) + }) + + Convey("Deleteing folder should delete provision meta data", func() { + deleteCmd := &models.DeleteDashboardCommand{ + Id: folderCmd.Result.Id, + OrgId: 1, + } + + So(DeleteDashboard(deleteCmd), ShouldBeNil) + + query := &models.IsDashboardProvisionedQuery{DashboardId: cmd.Result.Id} + + err = GetProvisionedDataByDashboardId(query) + So(err, ShouldBeNil) + So(query.Result, ShouldBeFalse) + }) }) }) } diff --git a/pkg/services/sqlstore/dashboard_service_integration_test.go b/pkg/services/sqlstore/dashboard_service_integration_test.go index d005270c33c..a4e76aca340 100644 --- a/pkg/services/sqlstore/dashboard_service_integration_test.go +++ b/pkg/services/sqlstore/dashboard_service_integration_test.go @@ -19,7 +19,6 @@ func TestIntegratedDashboardService(t *testing.T) { var testOrgId int64 = 1 Convey("Given saved folders and dashboards in organization A", func() { - bus.AddHandler("test", func(cmd *models.ValidateDashboardAlertsCommand) error { return nil }) @@ -28,6 +27,11 @@ func TestIntegratedDashboardService(t *testing.T) { return nil }) + bus.AddHandler("test", func(cmd *models.IsDashboardProvisionedQuery) error { + cmd.Result = false + return nil + }) + savedFolder := saveTestFolder("Saved folder", testOrgId) savedDashInFolder := saveTestDashboard("Saved dash in folder", testOrgId, savedFolder.Id) saveTestDashboard("Other saved dash in folder", testOrgId, savedFolder.Id) @@ -74,7 +78,7 @@ func TestIntegratedDashboardService(t *testing.T) { Convey("Given organization B", func() { var otherOrgId int64 = 2 - Convey("When saving a dashboard with id that are saved in organization A", func() { + Convey("When creating a dashboard with same id as dashboard in organization A", func() { cmd := models.SaveDashboardCommand{ OrgId: otherOrgId, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -93,7 +97,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) permissionScenario("Given user has permission to save", true, func(sc *dashboardPermissionScenarioContext) { - Convey("When saving a dashboard with uid that are saved in organization A", func() { + Convey("When creating a dashboard with same uid as dashboard in organization A", func() { var otherOrgId int64 = 2 cmd := models.SaveDashboardCommand{ OrgId: otherOrgId, @@ -106,7 +110,7 @@ func TestIntegratedDashboardService(t *testing.T) { res := callSaveWithResult(cmd) - Convey("It should create dashboard in other organization", func() { + Convey("It should create a new dashboard in organization B", func() { So(res, ShouldNotBeNil) query := models.GetDashboardQuery{OrgId: otherOrgId, Uid: savedDashInFolder.Uid} @@ -126,7 +130,7 @@ func TestIntegratedDashboardService(t *testing.T) { permissionScenario("Given user has no permission to save", false, func(sc *dashboardPermissionScenarioContext) { - Convey("When trying to create a new dashboard in the General folder", func() { + Convey("When creating a new dashboard in the General folder", func() { cmd := models.SaveDashboardCommand{ OrgId: testOrgId, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -138,7 +142,7 @@ func TestIntegratedDashboardService(t *testing.T) { err := callSaveWithError(cmd) - Convey("It should call dashboard guardian with correct arguments and result in access denied error", func() { + Convey("It should create dashboard guardian for General Folder with correct arguments and result in access denied error", func() { So(err, ShouldNotBeNil) So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) @@ -148,7 +152,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to create a new dashboard in other folder", func() { + Convey("When creating a new dashboard in other folder", func() { cmd := models.SaveDashboardCommand{ OrgId: testOrgId, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -161,7 +165,7 @@ func TestIntegratedDashboardService(t *testing.T) { err := callSaveWithError(cmd) - Convey("It should call dashboard guardian with correct arguments and rsult in access denied error", func() { + Convey("It should create dashboard guardian for other folder with correct arguments and rsult in access denied error", func() { So(err, ShouldNotBeNil) So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) @@ -171,7 +175,54 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update a dashboard by existing id in the General folder", func() { + Convey("When creating a new dashboard by existing title in folder", func() { + cmd := models.SaveDashboardCommand{ + OrgId: testOrgId, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "title": savedDashInFolder.Title, + }), + FolderId: savedFolder.Id, + UserId: 10000, + Overwrite: true, + } + + err := callSaveWithError(cmd) + + Convey("It should create dashboard guardian for folder with correct arguments and result in access denied error", func() { + So(err, ShouldNotBeNil) + So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) + + So(sc.dashboardGuardianMock.DashId, ShouldEqual, savedFolder.Id) + So(sc.dashboardGuardianMock.OrgId, ShouldEqual, cmd.OrgId) + So(sc.dashboardGuardianMock.User.UserId, ShouldEqual, cmd.UserId) + }) + }) + + Convey("When creating a new dashboard by existing uid in folder", func() { + cmd := models.SaveDashboardCommand{ + OrgId: testOrgId, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "uid": savedDashInFolder.Uid, + "title": "New dash", + }), + FolderId: savedFolder.Id, + UserId: 10000, + Overwrite: true, + } + + err := callSaveWithError(cmd) + + Convey("It should create dashboard guardian for folder with correct arguments and result in access denied error", func() { + So(err, ShouldNotBeNil) + So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) + + So(sc.dashboardGuardianMock.DashId, ShouldEqual, savedFolder.Id) + So(sc.dashboardGuardianMock.OrgId, ShouldEqual, cmd.OrgId) + So(sc.dashboardGuardianMock.User.UserId, ShouldEqual, cmd.UserId) + }) + }) + + Convey("When updating a dashboard by existing id in the General folder", func() { cmd := models.SaveDashboardCommand{ OrgId: testOrgId, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -185,7 +236,7 @@ func TestIntegratedDashboardService(t *testing.T) { err := callSaveWithError(cmd) - Convey("It should call dashboard guardian with correct arguments and result in access denied error", func() { + Convey("It should create dashboard guardian for dashboard with correct arguments and result in access denied error", func() { So(err, ShouldNotBeNil) So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) @@ -195,7 +246,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update a dashboard by existing id in other folder", func() { + Convey("When updating a dashboard by existing id in other folder", func() { cmd := models.SaveDashboardCommand{ OrgId: testOrgId, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -209,7 +260,7 @@ func TestIntegratedDashboardService(t *testing.T) { err := callSaveWithError(cmd) - Convey("It should call dashboard guardian with correct arguments and result in access denied error", func() { + Convey("It should create dashboard guardian for dashboard with correct arguments and result in access denied error", func() { So(err, ShouldNotBeNil) So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) @@ -218,6 +269,102 @@ func TestIntegratedDashboardService(t *testing.T) { So(sc.dashboardGuardianMock.User.UserId, ShouldEqual, cmd.UserId) }) }) + + Convey("When moving a dashboard by existing id to other folder from General folder", func() { + cmd := models.SaveDashboardCommand{ + OrgId: testOrgId, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "id": savedDashInGeneralFolder.Id, + "title": "Dash", + }), + FolderId: otherSavedFolder.Id, + UserId: 10000, + Overwrite: true, + } + + err := callSaveWithError(cmd) + + Convey("It should create dashboard guardian for other folder with correct arguments and result in access denied error", func() { + So(err, ShouldNotBeNil) + So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) + + So(sc.dashboardGuardianMock.DashId, ShouldEqual, otherSavedFolder.Id) + So(sc.dashboardGuardianMock.OrgId, ShouldEqual, cmd.OrgId) + So(sc.dashboardGuardianMock.User.UserId, ShouldEqual, cmd.UserId) + }) + }) + + Convey("When moving a dashboard by existing id to the General folder from other folder", func() { + cmd := models.SaveDashboardCommand{ + OrgId: testOrgId, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "id": savedDashInFolder.Id, + "title": "Dash", + }), + FolderId: 0, + UserId: 10000, + Overwrite: true, + } + + err := callSaveWithError(cmd) + + Convey("It should create dashboard guardian for General folder with correct arguments and result in access denied error", func() { + So(err, ShouldNotBeNil) + So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) + + So(sc.dashboardGuardianMock.DashId, ShouldEqual, 0) + So(sc.dashboardGuardianMock.OrgId, ShouldEqual, cmd.OrgId) + So(sc.dashboardGuardianMock.User.UserId, ShouldEqual, cmd.UserId) + }) + }) + + Convey("When moving a dashboard by existing uid to other folder from General folder", func() { + cmd := models.SaveDashboardCommand{ + OrgId: testOrgId, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "uid": savedDashInGeneralFolder.Uid, + "title": "Dash", + }), + FolderId: otherSavedFolder.Id, + UserId: 10000, + Overwrite: true, + } + + err := callSaveWithError(cmd) + + Convey("It should create dashboard guardian for other folder with correct arguments and result in access denied error", func() { + So(err, ShouldNotBeNil) + So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) + + So(sc.dashboardGuardianMock.DashId, ShouldEqual, otherSavedFolder.Id) + So(sc.dashboardGuardianMock.OrgId, ShouldEqual, cmd.OrgId) + So(sc.dashboardGuardianMock.User.UserId, ShouldEqual, cmd.UserId) + }) + }) + + Convey("When moving a dashboard by existing uid to the General folder from other folder", func() { + cmd := models.SaveDashboardCommand{ + OrgId: testOrgId, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "uid": savedDashInFolder.Uid, + "title": "Dash", + }), + FolderId: 0, + UserId: 10000, + Overwrite: true, + } + + err := callSaveWithError(cmd) + + Convey("It should create dashboard guardian for General folder with correct arguments and result in access denied error", func() { + So(err, ShouldNotBeNil) + So(err, ShouldEqual, models.ErrDashboardUpdateAccessDenied) + + So(sc.dashboardGuardianMock.DashId, ShouldEqual, 0) + So(sc.dashboardGuardianMock.OrgId, ShouldEqual, cmd.OrgId) + So(sc.dashboardGuardianMock.User.UserId, ShouldEqual, cmd.UserId) + }) + }) }) // Given user has permission to save @@ -668,7 +815,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update existing folder to a dashboard using id", func() { + Convey("When updating existing folder to a dashboard using id", func() { cmd := models.SaveDashboardCommand{ OrgId: 1, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -687,7 +834,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update existing dashboard to a folder using id", func() { + Convey("When updating existing dashboard to a folder using id", func() { cmd := models.SaveDashboardCommand{ OrgId: 1, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -706,7 +853,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update existing folder to a dashboard using uid", func() { + Convey("When updating existing folder to a dashboard using uid", func() { cmd := models.SaveDashboardCommand{ OrgId: 1, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -725,7 +872,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update existing dashboard to a folder using uid", func() { + Convey("When updating existing dashboard to a folder using uid", func() { cmd := models.SaveDashboardCommand{ OrgId: 1, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -744,7 +891,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update existing folder to a dashboard using title", func() { + Convey("When updating existing folder to a dashboard using title", func() { cmd := models.SaveDashboardCommand{ OrgId: 1, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -762,7 +909,7 @@ func TestIntegratedDashboardService(t *testing.T) { }) }) - Convey("When trying to update existing dashboard to a folder using title", func() { + Convey("When updating existing dashboard to a folder using title", func() { cmd := models.SaveDashboardCommand{ OrgId: 1, Dashboard: simplejson.NewFromAny(map[string]interface{}{ @@ -785,29 +932,6 @@ func TestIntegratedDashboardService(t *testing.T) { }) } -type scenarioContext struct { - dashboardGuardianMock *guardian.FakeDashboardGuardian -} - -type scenarioFunc func(c *scenarioContext) - -func dashboardGuardianScenario(desc string, mock *guardian.FakeDashboardGuardian, fn scenarioFunc) { - Convey(desc, func() { - origNewDashboardGuardian := guardian.New - guardian.MockDashboardGuardian(mock) - - sc := &scenarioContext{ - dashboardGuardianMock: mock, - } - - defer func() { - guardian.New = origNewDashboardGuardian - }() - - fn(sc) - }) -} - type dashboardPermissionScenarioContext struct { dashboardGuardianMock *guardian.FakeDashboardGuardian } @@ -850,23 +974,6 @@ func callSaveWithError(cmd models.SaveDashboardCommand) error { return err } -func dashboardServiceScenario(desc string, mock *guardian.FakeDashboardGuardian, fn scenarioFunc) { - Convey(desc, func() { - origNewDashboardGuardian := guardian.New - guardian.MockDashboardGuardian(mock) - - sc := &scenarioContext{ - dashboardGuardianMock: mock, - } - - defer func() { - guardian.New = origNewDashboardGuardian - }() - - fn(sc) - }) -} - func saveTestDashboard(title string, orgId int64, folderId int64) *models.Dashboard { cmd := models.SaveDashboardCommand{ OrgId: orgId, diff --git a/pkg/services/sqlstore/dashboard_snapshot.go b/pkg/services/sqlstore/dashboard_snapshot.go index 9e82bbb2c83..2e2ea8a4783 100644 --- a/pkg/services/sqlstore/dashboard_snapshot.go +++ b/pkg/services/sqlstore/dashboard_snapshot.go @@ -80,7 +80,7 @@ func GetDashboardSnapshot(query *m.GetDashboardSnapshotQuery) error { if err != nil { return err - } else if has == false { + } else if !has { return m.ErrDashboardSnapshotNotFound } diff --git a/pkg/services/sqlstore/dashboard_snapshot_test.go b/pkg/services/sqlstore/dashboard_snapshot_test.go index 2081cbf6194..09b3fb7607f 100644 --- a/pkg/services/sqlstore/dashboard_snapshot_test.go +++ b/pkg/services/sqlstore/dashboard_snapshot_test.go @@ -4,7 +4,6 @@ import ( "testing" "time" - "github.com/go-xorm/xorm" . "github.com/smartystreets/goconvey/convey" "github.com/grafana/grafana/pkg/components/simplejson" @@ -110,46 +109,43 @@ func TestDashboardSnapshotDBAccess(t *testing.T) { } func TestDeleteExpiredSnapshots(t *testing.T) { - Convey("Testing dashboard snapshots clean up", t, func() { - x := InitTestDB(t) + sqlstore := InitTestDB(t) + Convey("Testing dashboard snapshots clean up", t, func() { setting.SnapShotRemoveExpired = true - notExpiredsnapshot := createTestSnapshot(x, "key1", 1000) - createTestSnapshot(x, "key2", -1000) - createTestSnapshot(x, "key3", -1000) + notExpiredsnapshot := createTestSnapshot(sqlstore, "key1", 48000) + createTestSnapshot(sqlstore, "key2", -1200) + createTestSnapshot(sqlstore, "key3", -1200) - Convey("Clean up old dashboard snapshots", func() { - err := DeleteExpiredSnapshots(&m.DeleteExpiredSnapshotsCommand{}) - So(err, ShouldBeNil) + err := DeleteExpiredSnapshots(&m.DeleteExpiredSnapshotsCommand{}) + So(err, ShouldBeNil) - query := m.GetDashboardSnapshotsQuery{ - OrgId: 1, - SignedInUser: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}, - } - err = SearchDashboardSnapshots(&query) - So(err, ShouldBeNil) + query := m.GetDashboardSnapshotsQuery{ + OrgId: 1, + SignedInUser: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}, + } + err = SearchDashboardSnapshots(&query) + So(err, ShouldBeNil) - So(len(query.Result), ShouldEqual, 1) - So(query.Result[0].Key, ShouldEqual, notExpiredsnapshot.Key) - }) + So(len(query.Result), ShouldEqual, 1) + So(query.Result[0].Key, ShouldEqual, notExpiredsnapshot.Key) - Convey("Don't delete anything if there are no expired snapshots", func() { - err := DeleteExpiredSnapshots(&m.DeleteExpiredSnapshotsCommand{}) - So(err, ShouldBeNil) + err = DeleteExpiredSnapshots(&m.DeleteExpiredSnapshotsCommand{}) + So(err, ShouldBeNil) - query := m.GetDashboardSnapshotsQuery{ - OrgId: 1, - SignedInUser: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}, - } - SearchDashboardSnapshots(&query) + query = m.GetDashboardSnapshotsQuery{ + OrgId: 1, + SignedInUser: &m.SignedInUser{OrgRole: m.ROLE_ADMIN}, + } + SearchDashboardSnapshots(&query) - So(len(query.Result), ShouldEqual, 1) - }) + So(len(query.Result), ShouldEqual, 1) + So(query.Result[0].Key, ShouldEqual, notExpiredsnapshot.Key) }) } -func createTestSnapshot(x *xorm.Engine, key string, expires int64) *m.DashboardSnapshot { +func createTestSnapshot(sqlstore *SqlStore, key string, expires int64) *m.DashboardSnapshot { cmd := m.CreateDashboardSnapshotCommand{ Key: key, DeleteKey: "delete" + key, @@ -164,9 +160,11 @@ func createTestSnapshot(x *xorm.Engine, key string, expires int64) *m.DashboardS So(err, ShouldBeNil) // Set expiry date manually - to be able to create expired snapshots - expireDate := time.Now().Add(time.Second * time.Duration(expires)) - _, err = x.Exec("update dashboard_snapshot set expires = ? where "+dialect.Quote("key")+" = ?", expireDate, key) - So(err, ShouldBeNil) + if expires < 0 { + expireDate := time.Now().Add(time.Second * time.Duration(expires)) + _, err = sqlstore.engine.Exec("UPDATE dashboard_snapshot SET expires = ? WHERE id = ?", expireDate, cmd.Result.Id) + So(err, ShouldBeNil) + } return cmd.Result } diff --git a/pkg/services/sqlstore/dashboard_test.go b/pkg/services/sqlstore/dashboard_test.go index 9124a686236..8ff78c4a0ff 100644 --- a/pkg/services/sqlstore/dashboard_test.go +++ b/pkg/services/sqlstore/dashboard_test.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "fmt" "testing" "time" @@ -104,9 +105,8 @@ func TestDashboardDataAccess(t *testing.T) { timesCalled += 1 if timesCalled <= 2 { return savedDash.Uid - } else { - return util.GenerateShortUid() } + return util.GenerateShortUid() } cmd := m.SaveDashboardCommand{ OrgId: 1, @@ -181,7 +181,7 @@ func TestDashboardDataAccess(t *testing.T) { So(err, ShouldBeNil) So(query.Result.FolderId, ShouldEqual, 0) So(query.Result.CreatedBy, ShouldEqual, savedDash.CreatedBy) - So(query.Result.Created, ShouldEqual, savedDash.Created.Truncate(time.Second)) + So(query.Result.Created, ShouldHappenWithin, 3*time.Second, savedDash.Created) So(query.Result.UpdatedBy, ShouldEqual, 100) So(query.Result.Updated.IsZero(), ShouldBeFalse) }) @@ -387,10 +387,11 @@ func insertTestDashboardForPlugin(title string, orgId int64, folderId int64, isF func createUser(name string, role string, isAdmin bool) m.User { setting.AutoAssignOrg = true + setting.AutoAssignOrgId = 1 setting.AutoAssignOrgRole = role currentUserCmd := m.CreateUserCommand{Login: name, Email: name + "@test.com", Name: "a " + name, IsAdmin: isAdmin} - err := CreateUser(¤tUserCmd) + err := CreateUser(context.Background(), ¤tUserCmd) So(err, ShouldBeNil) q1 := m.GetUserOrgListQuery{UserId: currentUserCmd.Result.Id} diff --git a/pkg/services/sqlstore/dashboard_version.go b/pkg/services/sqlstore/dashboard_version.go index 547f62628f3..1f2850b2021 100644 --- a/pkg/services/sqlstore/dashboard_version.go +++ b/pkg/services/sqlstore/dashboard_version.go @@ -67,30 +67,39 @@ func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error { return nil } +const MAX_VERSIONS_TO_DELETE = 100 + func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error { return inTransaction(func(sess *DBSession) error { - versions := []DashboardVersionExp{} versionsToKeep := setting.DashboardVersionsToKeep - if versionsToKeep < 1 { versionsToKeep = 1 } - err := sess.Table("dashboard_version"). - Select("dashboard_version.id, dashboard_version.version, dashboard_version.dashboard_id"). - Where(`dashboard_id IN ( - SELECT dashboard_id FROM dashboard_version - GROUP BY dashboard_id HAVING COUNT(dashboard_version.id) > ? - )`, versionsToKeep). - Desc("dashboard_version.dashboard_id", "dashboard_version.version"). - Find(&versions) + // Idea of this query is finding version IDs to delete based on formula: + // min_version_to_keep = min_version + (versions_count - versions_to_keep) + // where version stats is processed for each dashboard. This guarantees that we keep at least versions_to_keep + // versions, but in some cases (when versions are sparse) this number may be more. + versionIdsToDeleteQuery := `SELECT id + FROM dashboard_version, ( + SELECT dashboard_id, count(version) as count, min(version) as min + FROM dashboard_version + GROUP BY dashboard_id + ) AS vtd + WHERE dashboard_version.dashboard_id=vtd.dashboard_id + AND version < vtd.min + vtd.count - ?` + var versionIdsToDelete []interface{} + err := sess.SQL(versionIdsToDeleteQuery, versionsToKeep).Find(&versionIdsToDelete) if err != nil { return err } - // Keep last versionsToKeep versions and delete other - versionIdsToDelete := getVersionIDsToDelete(versions, versionsToKeep) + // Don't delete more than MAX_VERSIONS_TO_DELETE version per time + if len(versionIdsToDelete) > MAX_VERSIONS_TO_DELETE { + versionIdsToDelete = versionIdsToDelete[:MAX_VERSIONS_TO_DELETE] + } + if len(versionIdsToDelete) > 0 { deleteExpiredSql := `DELETE FROM dashboard_version WHERE id IN (?` + strings.Repeat(",?", len(versionIdsToDelete)-1) + `)` expiredResponse, err := sess.Exec(deleteExpiredSql, versionIdsToDelete...) @@ -103,34 +112,3 @@ func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error { return nil }) } - -// Short version of DashboardVersion for getting expired versions -type DashboardVersionExp struct { - Id int64 `json:"id"` - DashboardId int64 `json:"dashboardId"` - Version int `json:"version"` -} - -func getVersionIDsToDelete(versions []DashboardVersionExp, versionsToKeep int) []interface{} { - versionIds := make([]interface{}, 0) - - if len(versions) == 0 { - return versionIds - } - - currentDashboard := versions[0].DashboardId - count := 0 - for _, v := range versions { - if v.DashboardId == currentDashboard { - count++ - } else { - count = 1 - currentDashboard = v.DashboardId - } - if count > versionsToKeep { - versionIds = append(versionIds, v.Id) - } - } - - return versionIds -} diff --git a/pkg/services/sqlstore/dashboard_version_test.go b/pkg/services/sqlstore/dashboard_version_test.go index 1b74e7847c4..a6403755d05 100644 --- a/pkg/services/sqlstore/dashboard_version_test.go +++ b/pkg/services/sqlstore/dashboard_version_test.go @@ -136,10 +136,30 @@ func TestDeleteExpiredVersions(t *testing.T) { err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) So(err, ShouldBeNil) - query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1} + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1, Limit: versionsToWrite} GetDashboardVersions(&query) So(len(query.Result), ShouldEqual, versionsToWrite) }) + + Convey("Don't delete more than MAX_VERSIONS_TO_DELETE per iteration", func() { + versionsToWriteBigNumber := MAX_VERSIONS_TO_DELETE + versionsToWrite + for i := 0; i < versionsToWriteBigNumber-versionsToWrite; i++ { + updateTestDashboard(savedDash, map[string]interface{}{ + "tags": "different-tag", + }) + } + + err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) + So(err, ShouldBeNil) + + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1, Limit: versionsToWriteBigNumber} + GetDashboardVersions(&query) + + // Ensure we have at least versionsToKeep versions + So(len(query.Result), ShouldBeGreaterThanOrEqualTo, versionsToKeep) + // Ensure we haven't deleted more than MAX_VERSIONS_TO_DELETE rows + So(versionsToWriteBigNumber-len(query.Result), ShouldBeLessThanOrEqualTo, MAX_VERSIONS_TO_DELETE) + }) }) } diff --git a/pkg/services/sqlstore/datasource.go b/pkg/services/sqlstore/datasource.go index 00d520bcfc6..7f70e5c25fc 100644 --- a/pkg/services/sqlstore/datasource.go +++ b/pkg/services/sqlstore/datasource.go @@ -27,6 +27,7 @@ func GetDataSourceById(query *m.GetDataSourceByIdQuery) error { datasource := m.DataSource{OrgId: query.OrgId, Id: query.Id} has, err := x.Get(&datasource) + if err != nil { return err } diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 2a364d5f464..198a47b50ff 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -65,6 +65,16 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("Add column is_default", NewAddColumnMigration(alert_notification, &Column{ Name: "is_default", Type: DB_Bool, Nullable: false, Default: "0", })) + mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{ + Name: "frequency", Type: DB_BigInt, Nullable: true, + })) + mg.AddMigration("Add column send_reminder", NewAddColumnMigration(alert_notification, &Column{ + Name: "send_reminder", Type: DB_Bool, Nullable: true, Default: "0", + })) + mg.AddMigration("Add column disable_resolve_message", NewAddColumnMigration(alert_notification, &Column{ + Name: "disable_resolve_message", Type: DB_Bool, Nullable: false, Default: "0", + })) + mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0])) mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{ @@ -82,4 +92,45 @@ func addAlertMigrations(mg *Migrator) { {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "settings", Type: DB_Text, Nullable: false}, })) + + notification_journal := Table{ + Name: "alert_notification_journal", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "alert_id", Type: DB_BigInt, Nullable: false}, + {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, + {Name: "sent_at", Type: DB_BigInt, Nullable: false}, + {Name: "success", Type: DB_Bool, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType}, + }, + } + + mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal)) + mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0])) + + mg.AddMigration("drop alert_notification_journal", NewDropTableMigration("alert_notification_journal")) + + alert_notification_state := Table{ + Name: "alert_notification_state", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "alert_id", Type: DB_BigInt, Nullable: false}, + {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, + {Name: "state", Type: DB_NVarchar, Length: 50, Nullable: false}, + {Name: "version", Type: DB_BigInt, Nullable: false}, + {Name: "updated_at", Type: DB_BigInt, Nullable: false}, + {Name: "alert_rule_state_updated_version", Type: DB_BigInt, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: UniqueIndex}, + }, + } + + mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state)) + mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id", + NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0])) } diff --git a/pkg/services/sqlstore/migrations/annotation_mig.go b/pkg/services/sqlstore/migrations/annotation_mig.go index 8d2bf94bc42..49920dee490 100644 --- a/pkg/services/sqlstore/migrations/annotation_mig.go +++ b/pkg/services/sqlstore/migrations/annotation_mig.go @@ -86,8 +86,27 @@ func addAnnotationMig(mg *Migrator) { // clear alert text // updateTextFieldSql := "UPDATE annotation SET TEXT = '' WHERE alert_id > 0" - mg.AddMigration("Update alert annotations and set TEXT to empty", new(RawSqlMigration). - Sqlite(updateTextFieldSql). - Postgres(updateTextFieldSql). - Mysql(updateTextFieldSql)) + mg.AddMigration("Update alert annotations and set TEXT to empty", NewRawSqlMigration(updateTextFieldSql)) + + // + // Add a 'created' & 'updated' column + // + mg.AddMigration("Add created time to annotation table", NewAddColumnMigration(table, &Column{ + Name: "created", Type: DB_BigInt, Nullable: true, Default: "0", + })) + mg.AddMigration("Add updated time to annotation table", NewAddColumnMigration(table, &Column{ + Name: "updated", Type: DB_BigInt, Nullable: true, Default: "0", + })) + mg.AddMigration("Add index for created in annotation table", NewAddIndexMigration(table, &Index{ + Cols: []string{"org_id", "created"}, Type: IndexType, + })) + mg.AddMigration("Add index for updated in annotation table", NewAddIndexMigration(table, &Index{ + Cols: []string{"org_id", "updated"}, Type: IndexType, + })) + + // + // Convert epoch saved as seconds to milliseconds + // + updateEpochSql := "UPDATE annotation SET epoch = (epoch*1000) where epoch < 9999999999" + mg.AddMigration("Convert existing annotations from seconds to milliseconds", NewRawSqlMigration(updateEpochSql)) } diff --git a/pkg/services/sqlstore/migrations/dashboard_acl.go b/pkg/services/sqlstore/migrations/dashboard_acl.go index cc3b813c12f..32e4aec6d4a 100644 --- a/pkg/services/sqlstore/migrations/dashboard_acl.go +++ b/pkg/services/sqlstore/migrations/dashboard_acl.go @@ -45,8 +45,5 @@ INSERT INTO dashboard_acl (-1,-1, 2,'Editor','2017-06-20','2017-06-20') ` - mg.AddMigration("save default acl rules in dashboard_acl table", new(RawSqlMigration). - Sqlite(rawSQL). - Postgres(rawSQL). - Mysql(rawSQL)) + mg.AddMigration("save default acl rules in dashboard_acl table", NewRawSqlMigration(rawSQL)) } diff --git a/pkg/services/sqlstore/migrations/dashboard_mig.go b/pkg/services/sqlstore/migrations/dashboard_mig.go index 296950ee497..b770afb1b4e 100644 --- a/pkg/services/sqlstore/migrations/dashboard_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_mig.go @@ -90,9 +90,7 @@ func addDashboardMigration(mg *Migrator) { mg.AddMigration("drop table dashboard_v1", NewDropTableMigration("dashboard_v1")) // change column type of dashboard.data - mg.AddMigration("alter dashboard.data to mediumtext v1", new(RawSqlMigration). - Sqlite("SELECT 0 WHERE 0;"). - Postgres("SELECT 0;"). + mg.AddMigration("alter dashboard.data to mediumtext v1", NewRawSqlMigration(""). Mysql("ALTER TABLE dashboard MODIFY data MEDIUMTEXT;")) // add column to store updater of a dashboard @@ -157,7 +155,7 @@ func addDashboardMigration(mg *Migrator) { Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: true, })) - mg.AddMigration("Update uid column values in dashboard", new(RawSqlMigration). + mg.AddMigration("Update uid column values in dashboard", NewRawSqlMigration(""). Sqlite("UPDATE dashboard SET uid=printf('%09d',id) WHERE uid IS NULL;"). Postgres("UPDATE dashboard SET uid=lpad('' || id,9,'0') WHERE uid IS NULL;"). Mysql("UPDATE dashboard SET uid=lpad(id,9,'0') WHERE uid IS NULL;")) @@ -213,4 +211,8 @@ func addDashboardMigration(mg *Migrator) { "name": "name", "external_id": "external_id", }) + + mg.AddMigration("Add check_sum column", NewAddColumnMigration(dashboardExtrasTableV2, &Column{ + Name: "check_sum", Type: DB_NVarchar, Length: 32, Nullable: true, + })) } diff --git a/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go b/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go index 1a72bcba929..b880497cd23 100644 --- a/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go @@ -50,9 +50,7 @@ func addDashboardSnapshotMigrations(mg *Migrator) { addTableIndicesMigrations(mg, "v5", snapshotV5) // change column type of dashboard - mg.AddMigration("alter dashboard_snapshot to mediumtext v2", new(RawSqlMigration). - Sqlite("SELECT 0 WHERE 0;"). - Postgres("SELECT 0;"). + mg.AddMigration("alter dashboard_snapshot to mediumtext v2", NewRawSqlMigration(""). Mysql("ALTER TABLE dashboard_snapshot MODIFY dashboard MEDIUMTEXT;")) mg.AddMigration("Update dashboard_snapshot table charset", NewTableCharsetMigration("dashboard_snapshot", []*Column{ diff --git a/pkg/services/sqlstore/migrations/dashboard_version_mig.go b/pkg/services/sqlstore/migrations/dashboard_version_mig.go index 601161a6c89..8a38f885de3 100644 --- a/pkg/services/sqlstore/migrations/dashboard_version_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_version_mig.go @@ -28,10 +28,7 @@ func addDashboardVersionMigration(mg *Migrator) { // before new dashboards where created with version 0, now they are always inserted with version 1 const setVersionTo1WhereZeroSQL = `UPDATE dashboard SET version = 1 WHERE version = 0` - mg.AddMigration("Set dashboard version to 1 where 0", new(RawSqlMigration). - Sqlite(setVersionTo1WhereZeroSQL). - Postgres(setVersionTo1WhereZeroSQL). - Mysql(setVersionTo1WhereZeroSQL)) + mg.AddMigration("Set dashboard version to 1 where 0", NewRawSqlMigration(setVersionTo1WhereZeroSQL)) const rawSQL = `INSERT INTO dashboard_version ( @@ -54,14 +51,9 @@ SELECT '', dashboard.data FROM dashboard;` - mg.AddMigration("save existing dashboard data in dashboard_version table v1", new(RawSqlMigration). - Sqlite(rawSQL). - Postgres(rawSQL). - Mysql(rawSQL)) + mg.AddMigration("save existing dashboard data in dashboard_version table v1", NewRawSqlMigration(rawSQL)) // change column type of dashboard_version.data - mg.AddMigration("alter dashboard_version.data to mediumtext v1", new(RawSqlMigration). - Sqlite("SELECT 0 WHERE 0;"). - Postgres("SELECT 0;"). + mg.AddMigration("alter dashboard_version.data to mediumtext v1", NewRawSqlMigration(""). Mysql("ALTER TABLE dashboard_version MODIFY data MEDIUMTEXT;")) } diff --git a/pkg/services/sqlstore/migrations/datasource_mig.go b/pkg/services/sqlstore/migrations/datasource_mig.go index 919881adaba..9011429e398 100644 --- a/pkg/services/sqlstore/migrations/datasource_mig.go +++ b/pkg/services/sqlstore/migrations/datasource_mig.go @@ -122,10 +122,7 @@ func addDataSourceMigration(mg *Migrator) { })) const setVersionToOneWhereZero = `UPDATE data_source SET version = 1 WHERE version = 0` - mg.AddMigration("Update initial version to 1", new(RawSqlMigration). - Sqlite(setVersionToOneWhereZero). - Postgres(setVersionToOneWhereZero). - Mysql(setVersionToOneWhereZero)) + mg.AddMigration("Update initial version to 1", NewRawSqlMigration(setVersionToOneWhereZero)) mg.AddMigration("Add read_only data column", NewAddColumnMigration(tableV2, &Column{ Name: "read_only", Type: DB_Bool, Nullable: true, diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 282f98e7318..58ac6256f41 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -30,6 +30,7 @@ func AddMigrations(mg *Migrator) { addDashboardAclMigrations(mg) addTagMigration(mg) addLoginAttemptMigrations(mg) + addUserAuthMigrations(mg) } func addMigrationLogMigrations(mg *Migrator) { diff --git a/pkg/services/sqlstore/migrations/migrations_test.go b/pkg/services/sqlstore/migrations/migrations_test.go index 51aea0bbdef..ec4cb5fbce1 100644 --- a/pkg/services/sqlstore/migrations/migrations_test.go +++ b/pkg/services/sqlstore/migrations/migrations_test.go @@ -8,11 +8,8 @@ import ( "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" . "github.com/smartystreets/goconvey/convey" - //"github.com/grafana/grafana/pkg/log" ) -var indexTypes = []string{"Unknown", "INDEX", "UNIQUE INDEX"} - func TestMigrations(t *testing.T) { testDBs := []sqlutil.TestDB{ sqlutil.TestDB_Sqlite3, @@ -28,9 +25,9 @@ func TestMigrations(t *testing.T) { x, err := xorm.NewEngine(testDB.DriverName, testDB.ConnStr) So(err, ShouldBeNil) - sqlutil.CleanDB(x) + NewDialect(x).CleanDB() - has, err := x.SQL(sql).Get(&r) + _, err = x.SQL(sql).Get(&r) So(err, ShouldNotBeNil) mg := NewMigrator(x) @@ -39,10 +36,10 @@ func TestMigrations(t *testing.T) { err = mg.Start() So(err, ShouldBeNil) - has, err = x.SQL(sql).Get(&r) + has, err := x.SQL(sql).Get(&r) So(err, ShouldBeNil) So(has, ShouldBeTrue) - expectedMigrations := mg.MigrationsCount() - 2 //we currently skip to migrations. We should rewrite skipped migrations to write in the log as well. until then we have to keep this + expectedMigrations := mg.MigrationsCount() //we currently skip to migrations. We should rewrite skipped migrations to write in the log as well. until then we have to keep this So(r.Count, ShouldEqual, expectedMigrations) mg = NewMigrator(x) diff --git a/pkg/services/sqlstore/migrations/org_mig.go b/pkg/services/sqlstore/migrations/org_mig.go index cf9b19f6f5b..4e9f4295017 100644 --- a/pkg/services/sqlstore/migrations/org_mig.go +++ b/pkg/services/sqlstore/migrations/org_mig.go @@ -48,27 +48,6 @@ func addOrgMigrations(mg *Migrator) { mg.AddMigration("create org_user table v1", NewAddTableMigration(orgUserV1)) addTableIndicesMigrations(mg, "v1", orgUserV1) - //------- copy data from old table------------------- - mg.AddMigration("copy data account to org", NewCopyTableDataMigration("org", "account", map[string]string{ - "id": "id", - "version": "version", - "name": "name", - "created": "created", - "updated": "updated", - }).IfTableExists("account")) - - mg.AddMigration("copy data account_user to org_user", NewCopyTableDataMigration("org_user", "account_user", map[string]string{ - "id": "id", - "org_id": "account_id", - "user_id": "user_id", - "role": "role", - "created": "created", - "updated": "updated", - }).IfTableExists("account_user")) - - mg.AddMigration("Drop old table account", NewDropTableMigration("account")) - mg.AddMigration("Drop old table account_user", NewDropTableMigration("account_user")) - mg.AddMigration("Update org table charset", NewTableCharsetMigration("org", []*Column{ {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "address1", Type: DB_NVarchar, Length: 255, Nullable: true}, @@ -85,8 +64,5 @@ func addOrgMigrations(mg *Migrator) { })) const migrateReadOnlyViewersToViewers = `UPDATE org_user SET role = 'Viewer' WHERE role = 'Read Only Editor'` - mg.AddMigration("Migrate all Read Only Viewers to Viewers", new(RawSqlMigration). - Sqlite(migrateReadOnlyViewersToViewers). - Postgres(migrateReadOnlyViewersToViewers). - Mysql(migrateReadOnlyViewersToViewers)) + mg.AddMigration("Migrate all Read Only Viewers to Viewers", NewRawSqlMigration(migrateReadOnlyViewersToViewers)) } diff --git a/pkg/services/sqlstore/migrations/preferences_mig.go b/pkg/services/sqlstore/migrations/preferences_mig.go index b3822fe5239..d8134f11f79 100644 --- a/pkg/services/sqlstore/migrations/preferences_mig.go +++ b/pkg/services/sqlstore/migrations/preferences_mig.go @@ -34,4 +34,13 @@ func addPreferencesMigrations(mg *Migrator) { {Name: "timezone", Type: DB_NVarchar, Length: 50, Nullable: false}, {Name: "theme", Type: DB_NVarchar, Length: 20, Nullable: false}, })) + + mg.AddMigration("Add column team_id in preferences", NewAddColumnMigration(preferencesV2, &Column{ + Name: "team_id", Type: DB_BigInt, Nullable: true, + })) + + mg.AddMigration("Update team_id column values in preferences", NewRawSqlMigration(""). + Sqlite("UPDATE preferences SET team_id=0 WHERE team_id IS NULL;"). + Postgres("UPDATE preferences SET team_id=0 WHERE team_id IS NULL;"). + Mysql("UPDATE preferences SET team_id=0 WHERE team_id IS NULL;")) } diff --git a/pkg/services/sqlstore/migrations/stats_mig.go b/pkg/services/sqlstore/migrations/stats_mig.go index 7e10eeb9f90..c47b8202c53 100644 --- a/pkg/services/sqlstore/migrations/stats_mig.go +++ b/pkg/services/sqlstore/migrations/stats_mig.go @@ -2,37 +2,38 @@ package migrations import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" -func addStatsMigrations(mg *Migrator) { - statTable := Table{ - Name: "stat", - Columns: []*Column{ - {Name: "id", Type: DB_Int, IsPrimaryKey: true, IsAutoIncrement: true}, - {Name: "metric", Type: DB_Varchar, Length: 20, Nullable: false}, - {Name: "type", Type: DB_Int, Nullable: false}, - }, - Indices: []*Index{ - {Cols: []string{"metric"}, Type: UniqueIndex}, - }, - } - - // create table - mg.AddMigration("create stat table", NewAddTableMigration(statTable)) - - // create indices - mg.AddMigration("add index stat.metric", NewAddIndexMigration(statTable, statTable.Indices[0])) - - statValue := Table{ - Name: "stat_value", - Columns: []*Column{ - {Name: "id", Type: DB_Int, IsPrimaryKey: true, IsAutoIncrement: true}, - {Name: "value", Type: DB_Double, Nullable: false}, - {Name: "time", Type: DB_DateTime, Nullable: false}, - }, - } - - // create table - mg.AddMigration("create stat_value table", NewAddTableMigration(statValue)) -} +// commented out because of the deadcode CI check +//func addStatsMigrations(mg *Migrator) { +// statTable := Table{ +// Name: "stat", +// Columns: []*Column{ +// {Name: "id", Type: DB_Int, IsPrimaryKey: true, IsAutoIncrement: true}, +// {Name: "metric", Type: DB_Varchar, Length: 20, Nullable: false}, +// {Name: "type", Type: DB_Int, Nullable: false}, +// }, +// Indices: []*Index{ +// {Cols: []string{"metric"}, Type: UniqueIndex}, +// }, +// } +// +// // create table +// mg.AddMigration("create stat table", NewAddTableMigration(statTable)) +// +// // create indices +// mg.AddMigration("add index stat.metric", NewAddIndexMigration(statTable, statTable.Indices[0])) +// +// statValue := Table{ +// Name: "stat_value", +// Columns: []*Column{ +// {Name: "id", Type: DB_Int, IsPrimaryKey: true, IsAutoIncrement: true}, +// {Name: "value", Type: DB_Double, Nullable: false}, +// {Name: "time", Type: DB_DateTime, Nullable: false}, +// }, +// } +// +// // create table +// mg.AddMigration("create stat_value table", NewAddTableMigration(statValue)) +//} func addTestDataMigrations(mg *Migrator) { testData := Table{ diff --git a/pkg/services/sqlstore/migrations/team_mig.go b/pkg/services/sqlstore/migrations/team_mig.go index eb0641fbc32..34c46ad13cf 100644 --- a/pkg/services/sqlstore/migrations/team_mig.go +++ b/pkg/services/sqlstore/migrations/team_mig.go @@ -50,4 +50,8 @@ func addTeamMigrations(mg *Migrator) { mg.AddMigration("Add column email to team table", NewAddColumnMigration(teamV1, &Column{ Name: "email", Type: DB_NVarchar, Nullable: true, Length: 190, })) + + mg.AddMigration("Add column external to team_member table", NewAddColumnMigration(teamMemberV1, &Column{ + Name: "external", Type: DB_Bool, Nullable: true, + })) } diff --git a/pkg/services/sqlstore/migrations/user_auth_mig.go b/pkg/services/sqlstore/migrations/user_auth_mig.go new file mode 100644 index 00000000000..2771035b47f --- /dev/null +++ b/pkg/services/sqlstore/migrations/user_auth_mig.go @@ -0,0 +1,28 @@ +package migrations + +import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + +func addUserAuthMigrations(mg *Migrator) { + userAuthV1 := Table{ + Name: "user_auth", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "user_id", Type: DB_BigInt, Nullable: false}, + {Name: "auth_module", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "auth_id", Type: DB_NVarchar, Length: 100, Nullable: false}, + {Name: "created", Type: DB_DateTime, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"auth_module", "auth_id"}}, + }, + } + + // create table + mg.AddMigration("create user auth table", NewAddTableMigration(userAuthV1)) + // add indices + addTableIndicesMigrations(mg, "v1", userAuthV1) + + mg.AddMigration("alter user_auth.auth_id to length 190", NewRawSqlMigration(""). + Postgres("ALTER TABLE user_auth ALTER COLUMN auth_id TYPE VARCHAR(190);"). + Mysql("ALTER TABLE user_auth MODIFY auth_id VARCHAR(190);")) +} diff --git a/pkg/services/sqlstore/migrations/user_mig.go b/pkg/services/sqlstore/migrations/user_mig.go index edcfbb7b889..e273cb7d542 100644 --- a/pkg/services/sqlstore/migrations/user_mig.go +++ b/pkg/services/sqlstore/migrations/user_mig.go @@ -1,6 +1,12 @@ package migrations -import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +import ( + "fmt" + + "github.com/go-xorm/xorm" + . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/util" +) func addUserMigrations(mg *Migrator) { userV1 := Table{ @@ -107,4 +113,37 @@ func addUserMigrations(mg *Migrator) { mg.AddMigration("Add last_seen_at column to user", NewAddColumnMigration(userV2, &Column{ Name: "last_seen_at", Type: DB_DateTime, Nullable: true, })) + + // Adds salt & rands for old users who used ldap or oauth + mg.AddMigration("Add missing user data", &AddMissingUserSaltAndRandsMigration{}) +} + +type AddMissingUserSaltAndRandsMigration struct { + MigrationBase +} + +func (m *AddMissingUserSaltAndRandsMigration) Sql(dialect Dialect) string { + return "code migration" +} + +type TempUserDTO struct { + Id int64 + Login string +} + +func (m *AddMissingUserSaltAndRandsMigration) Exec(sess *xorm.Session, mg *Migrator) error { + users := make([]*TempUserDTO, 0) + + err := sess.SQL(fmt.Sprintf("SELECT id, login from %s WHERE rands = ''", mg.Dialect.Quote("user"))).Find(&users) + if err != nil { + return err + } + + for _, user := range users { + _, err := sess.Exec("UPDATE "+mg.Dialect.Quote("user")+" SET salt = ?, rands = ? WHERE id = ?", util.GetRandomString(10), util.GetRandomString(10), user.Id) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/services/sqlstore/migrator/column.go b/pkg/services/sqlstore/migrator/column.go index 11d6a1b8b08..28cef60a94d 100644 --- a/pkg/services/sqlstore/migrator/column.go +++ b/pkg/services/sqlstore/migrator/column.go @@ -15,48 +15,9 @@ type Column struct { } func (col *Column) String(d Dialect) string { - sql := d.QuoteStr() + col.Name + d.QuoteStr() + " " - - sql += d.SqlType(col) + " " - - if col.IsPrimaryKey { - sql += "PRIMARY KEY " - if col.IsAutoIncrement { - sql += d.AutoIncrStr() + " " - } - } - - if d.ShowCreateNull() { - if col.Nullable { - sql += "NULL " - } else { - sql += "NOT NULL " - } - } - - if col.Default != "" { - sql += "DEFAULT " + col.Default + " " - } - - return sql + return d.ColString(col) } func (col *Column) StringNoPk(d Dialect) string { - sql := d.QuoteStr() + col.Name + d.QuoteStr() + " " - - sql += d.SqlType(col) + " " - - if d.ShowCreateNull() { - if col.Nullable { - sql += "NULL " - } else { - sql += "NOT NULL " - } - } - - if col.Default != "" { - sql += "DEFAULT " + d.Default(col) + " " - } - - return sql + return d.ColStringNoPk(col) } diff --git a/pkg/services/sqlstore/migrator/dialect.go b/pkg/services/sqlstore/migrator/dialect.go index 064b5981063..506a01c3ed8 100644 --- a/pkg/services/sqlstore/migrator/dialect.go +++ b/pkg/services/sqlstore/migrator/dialect.go @@ -3,11 +3,12 @@ package migrator import ( "fmt" "strings" + + "github.com/go-xorm/xorm" ) type Dialect interface { DriverName() string - QuoteStr() string Quote(string) string AndStr() string AutoIncrStr() string @@ -31,16 +32,31 @@ type Dialect interface { TableCheckSql(tableName string) (string, []interface{}) RenameTable(oldName string, newName string) string UpdateTableSql(tableName string, columns []*Column) string + + ColString(*Column) string + ColStringNoPk(*Column) string + + Limit(limit int64) string + LimitOffset(limit int64, offset int64) string + + PreInsertId(table string, sess *xorm.Session) error + PostInsertId(table string, sess *xorm.Session) error + + CleanDB() error + NoOpSql() string + + IsUniqueConstraintViolation(err error) bool } -func NewDialect(name string) Dialect { +func NewDialect(engine *xorm.Engine) Dialect { + name := engine.DriverName() switch name { case MYSQL: - return NewMysqlDialect() + return NewMysqlDialect(engine) case SQLITE: - return NewSqlite3Dialect() + return NewSqlite3Dialect(engine) case POSTGRES: - return NewPostgresDialect() + return NewPostgresDialect(engine) } panic("Unsupported database type: " + name) @@ -48,6 +64,7 @@ func NewDialect(name string) Dialect { type BaseDialect struct { dialect Dialect + engine *xorm.Engine driverName string } @@ -84,8 +101,7 @@ func (db *BaseDialect) DateTimeFunc(value string) string { } func (b *BaseDialect) CreateTableSql(table *Table) string { - var sql string - sql = "CREATE TABLE IF NOT EXISTS " + sql := "CREATE TABLE IF NOT EXISTS " sql += b.dialect.Quote(table.Name) + " (\n" pkList := table.PrimaryKeys @@ -101,9 +117,12 @@ func (b *BaseDialect) CreateTableSql(table *Table) string { } if len(pkList) > 1 { - sql += "PRIMARY KEY ( " - sql += b.dialect.Quote(strings.Join(pkList, b.dialect.Quote(","))) - sql += " ), " + quotedCols := []string{} + for _, col := range pkList { + quotedCols = append(quotedCols, b.dialect.Quote(col)) + } + + sql += "PRIMARY KEY ( " + strings.Join(quotedCols, ",") + " ), " } sql = sql[:len(sql)-2] + ")" @@ -128,9 +147,12 @@ func (db *BaseDialect) CreateIndexSql(tableName string, index *Index) string { idxName := index.XName(tableName) - return fmt.Sprintf("CREATE%s INDEX %v ON %v (%v);", unique, - quote(idxName), quote(tableName), - quote(strings.Join(index.Cols, quote(",")))) + quotedCols := []string{} + for _, col := range index.Cols { + quotedCols = append(quotedCols, db.dialect.Quote(col)) + } + + return fmt.Sprintf("CREATE%s INDEX %v ON %v (%v);", unique, quote(idxName), quote(tableName), strings.Join(quotedCols, ",")) } func (db *BaseDialect) QuoteColList(cols []string) string { @@ -162,11 +184,81 @@ func (db *BaseDialect) RenameTable(oldName string, newName string) string { func (db *BaseDialect) DropIndexSql(tableName string, index *Index) string { quote := db.dialect.Quote - var name string - name = index.XName(tableName) + name := index.XName(tableName) return fmt.Sprintf("DROP INDEX %v ON %s", quote(name), quote(tableName)) } func (db *BaseDialect) UpdateTableSql(tableName string, columns []*Column) string { return "-- NOT REQUIRED" } + +func (db *BaseDialect) ColString(col *Column) string { + sql := db.dialect.Quote(col.Name) + " " + + sql += db.dialect.SqlType(col) + " " + + if col.IsPrimaryKey { + sql += "PRIMARY KEY " + if col.IsAutoIncrement { + sql += db.dialect.AutoIncrStr() + " " + } + } + + if db.dialect.ShowCreateNull() { + if col.Nullable { + sql += "NULL " + } else { + sql += "NOT NULL " + } + } + + if col.Default != "" { + sql += "DEFAULT " + db.dialect.Default(col) + " " + } + + return sql +} + +func (db *BaseDialect) ColStringNoPk(col *Column) string { + sql := db.dialect.Quote(col.Name) + " " + + sql += db.dialect.SqlType(col) + " " + + if db.dialect.ShowCreateNull() { + if col.Nullable { + sql += "NULL " + } else { + sql += "NOT NULL " + } + } + + if col.Default != "" { + sql += "DEFAULT " + db.dialect.Default(col) + " " + } + + return sql +} + +func (db *BaseDialect) Limit(limit int64) string { + return fmt.Sprintf(" LIMIT %d", limit) +} + +func (db *BaseDialect) LimitOffset(limit int64, offset int64) string { + return fmt.Sprintf(" LIMIT %d OFFSET %d", limit, offset) +} + +func (db *BaseDialect) PreInsertId(table string, sess *xorm.Session) error { + return nil +} + +func (db *BaseDialect) PostInsertId(table string, sess *xorm.Session) error { + return nil +} + +func (db *BaseDialect) CleanDB() error { + return nil +} + +func (db *BaseDialect) NoOpSql() string { + return "SELECT 0;" +} diff --git a/pkg/services/sqlstore/migrator/migrations.go b/pkg/services/sqlstore/migrator/migrations.go index 2fec8825fa4..fd71cc3d290 100644 --- a/pkg/services/sqlstore/migrator/migrations.go +++ b/pkg/services/sqlstore/migrator/migrations.go @@ -1,7 +1,6 @@ package migrator import ( - "fmt" "strings" ) @@ -25,37 +24,58 @@ func (m *MigrationBase) GetCondition() MigrationCondition { type RawSqlMigration struct { MigrationBase - sqlite string - mysql string - postgres string + sql map[string]string +} + +func NewRawSqlMigration(sql string) *RawSqlMigration { + m := &RawSqlMigration{} + if sql != "" { + m.Default(sql) + } + return m } func (m *RawSqlMigration) Sql(dialect Dialect) string { - switch dialect.DriverName() { - case MYSQL: - return m.mysql - case SQLITE: - return m.sqlite - case POSTGRES: - return m.postgres + if m.sql != nil { + if val := m.sql[dialect.DriverName()]; val != "" { + return val + } + + if val := m.sql["default"]; val != "" { + return val + } } - panic("db type not supported") + return dialect.NoOpSql() +} + +func (m *RawSqlMigration) Set(dialect string, sql string) *RawSqlMigration { + if m.sql == nil { + m.sql = make(map[string]string) + } + + m.sql[dialect] = sql + return m +} + +func (m *RawSqlMigration) Default(sql string) *RawSqlMigration { + return m.Set("default", sql) } func (m *RawSqlMigration) Sqlite(sql string) *RawSqlMigration { - m.sqlite = sql - return m + return m.Set(SQLITE, sql) } func (m *RawSqlMigration) Mysql(sql string) *RawSqlMigration { - m.mysql = sql - return m + return m.Set(MYSQL, sql) } func (m *RawSqlMigration) Postgres(sql string) *RawSqlMigration { - m.postgres = sql - return m + return m.Set(POSTGRES, sql) +} + +func (m *RawSqlMigration) Mssql(sql string) *RawSqlMigration { + return m.Set(MSSQL, sql) } type AddColumnMigration struct { @@ -113,7 +133,7 @@ func NewDropIndexMigration(table Table, index *Index) *DropIndexMigration { func (m *DropIndexMigration) Sql(dialect Dialect) string { if m.index.Name == "" { - m.index.Name = fmt.Sprintf("%s", strings.Join(m.index.Cols, "_")) + m.index.Name = strings.Join(m.index.Cols, "_") } return dialect.DropIndexSql(m.tableName, m.index) } @@ -180,7 +200,7 @@ type CopyTableDataMigration struct { targetTable string sourceCols []string targetCols []string - colMap map[string]string + //colMap map[string]string } func NewCopyTableDataMigration(targetTable string, sourceTable string, colMap map[string]string) *CopyTableDataMigration { diff --git a/pkg/services/sqlstore/migrator/migrator.go b/pkg/services/sqlstore/migrator/migrator.go index a8bd36ac8a3..dead6f2b416 100644 --- a/pkg/services/sqlstore/migrator/migrator.go +++ b/pkg/services/sqlstore/migrator/migrator.go @@ -12,7 +12,7 @@ import ( type Migrator struct { x *xorm.Engine - dialect Dialect + Dialect Dialect migrations []Migration Logger log.Logger } @@ -31,7 +31,7 @@ func NewMigrator(engine *xorm.Engine) *Migrator { mg.x = engine mg.Logger = log.New("migrator") mg.migrations = make([]Migration, 0) - mg.dialect = NewDialect(mg.x.DriverName()) + mg.Dialect = NewDialect(mg.x) return mg } @@ -86,7 +86,7 @@ func (mg *Migrator) Start() error { continue } - sql := m.Sql(mg.dialect) + sql := m.Sql(mg.Dialect) record := MigrationLog{ MigrationId: m.Id(), @@ -97,17 +97,15 @@ func (mg *Migrator) Start() error { mg.Logger.Debug("Executing", "sql", sql) err := mg.inTransaction(func(sess *xorm.Session) error { - - if err := mg.exec(m, sess); err != nil { + err := mg.exec(m, sess) + if err != nil { mg.Logger.Error("Exec failed", "error", err, "sql", sql) record.Error = err.Error() sess.Insert(&record) return err - } else { - record.Success = true - sess.Insert(&record) } - + record.Success = true + sess.Insert(&record) return nil }) @@ -124,15 +122,21 @@ func (mg *Migrator) exec(m Migration, sess *xorm.Session) error { condition := m.GetCondition() if condition != nil { - sql, args := condition.Sql(mg.dialect) - results, err := sess.Query(sql, args...) + sql, args := condition.Sql(mg.Dialect) + results, err := sess.SQL(sql).Query(args...) if err != nil || len(results) == 0 { - mg.Logger.Info("Skipping migration condition not fulfilled", "id", m.Id()) + mg.Logger.Debug("Skipping migration condition not fulfilled", "id", m.Id()) return sess.Rollback() } } - _, err := sess.Exec(m.Sql(mg.dialect)) + var err error + if codeMigration, ok := m.(CodeMigration); ok { + err = codeMigration.Exec(sess, mg) + } else { + _, err = sess.Exec(m.Sql(mg.Dialect)) + } + if err != nil { mg.Logger.Error("Executing migration failed", "id", m.Id(), "error", err) return err diff --git a/pkg/services/sqlstore/migrator/mysql_dialect.go b/pkg/services/sqlstore/migrator/mysql_dialect.go index 1968558dbb8..7daa4597430 100644 --- a/pkg/services/sqlstore/migrator/mysql_dialect.go +++ b/pkg/services/sqlstore/migrator/mysql_dialect.go @@ -1,17 +1,23 @@ package migrator import ( + "fmt" "strconv" "strings" + + "github.com/VividCortex/mysqlerr" + "github.com/go-sql-driver/mysql" + "github.com/go-xorm/xorm" ) type Mysql struct { BaseDialect } -func NewMysqlDialect() *Mysql { +func NewMysqlDialect(engine *xorm.Engine) *Mysql { d := Mysql{} d.BaseDialect.dialect = &d + d.BaseDialect.engine = engine d.BaseDialect.driverName = MYSQL return &d } @@ -24,10 +30,6 @@ func (db *Mysql) Quote(name string) string { return "`" + name + "`" } -func (db *Mysql) QuoteStr() string { - return "`" -} - func (db *Mysql) AutoIncrStr() string { return "AUTO_INCREMENT" } @@ -66,8 +68,8 @@ func (db *Mysql) SqlType(c *Column) string { res = c.Type } - var hasLen1 bool = (c.Length > 0) - var hasLen2 bool = (c.Length2 > 0) + var hasLen1 = (c.Length > 0) + var hasLen2 = (c.Length2 > 0) if res == DB_BigInt && !hasLen1 && !hasLen2 { c.Length = 20 @@ -105,3 +107,33 @@ func (db *Mysql) UpdateTableSql(tableName string, columns []*Column) string { return "ALTER TABLE " + db.Quote(tableName) + " " + strings.Join(statements, ", ") + ";" } + +func (db *Mysql) CleanDB() error { + tables, _ := db.engine.DBMetas() + sess := db.engine.NewSession() + defer sess.Close() + + for _, table := range tables { + if _, err := sess.Exec("set foreign_key_checks = 0"); err != nil { + return fmt.Errorf("failed to disable foreign key checks") + } + if _, err := sess.Exec("drop table " + table.Name + " ;"); err != nil { + return fmt.Errorf("failed to delete table: %v, err: %v", table.Name, err) + } + if _, err := sess.Exec("set foreign_key_checks = 1"); err != nil { + return fmt.Errorf("failed to disable foreign key checks") + } + } + + return nil +} + +func (db *Mysql) IsUniqueConstraintViolation(err error) bool { + if driverErr, ok := err.(*mysql.MySQLError); ok { + if driverErr.Number == mysqlerr.ER_DUP_ENTRY { + return true + } + } + + return false +} diff --git a/pkg/services/sqlstore/migrator/postgres_dialect.go b/pkg/services/sqlstore/migrator/postgres_dialect.go index 8de26194411..ab8812a1e26 100644 --- a/pkg/services/sqlstore/migrator/postgres_dialect.go +++ b/pkg/services/sqlstore/migrator/postgres_dialect.go @@ -4,15 +4,19 @@ import ( "fmt" "strconv" "strings" + + "github.com/go-xorm/xorm" + "github.com/lib/pq" ) type Postgres struct { BaseDialect } -func NewPostgresDialect() *Postgres { +func NewPostgresDialect(engine *xorm.Engine) *Postgres { d := Postgres{} d.BaseDialect.dialect = &d + d.BaseDialect.engine = engine d.BaseDialect.driverName = POSTGRES return &d } @@ -25,10 +29,6 @@ func (db *Postgres) Quote(name string) string { return "\"" + name + "\"" } -func (db *Postgres) QuoteStr() string { - return "\"" -} - func (b *Postgres) LikeStr() string { return "ILIKE" } @@ -45,9 +45,8 @@ func (b *Postgres) Default(col *Column) string { if col.Type == DB_Bool { if col.Default == "0" { return "FALSE" - } else { - return "TRUE" } + return "TRUE" } return col.Default } @@ -92,8 +91,8 @@ func (db *Postgres) SqlType(c *Column) string { res = t } - var hasLen1 bool = (c.Length > 0) - var hasLen2 bool = (c.Length2 > 0) + var hasLen1 = (c.Length > 0) + var hasLen2 = (c.Length2 > 0) if hasLen2 { res += "(" + strconv.Itoa(c.Length) + "," + strconv.Itoa(c.Length2) + ")" } else if hasLen1 { @@ -118,8 +117,33 @@ func (db *Postgres) UpdateTableSql(tableName string, columns []*Column) string { var statements = []string{} for _, col := range columns { - statements = append(statements, "ALTER "+db.QuoteStr()+col.Name+db.QuoteStr()+" TYPE "+db.SqlType(col)) + statements = append(statements, "ALTER "+db.Quote(col.Name)+" TYPE "+db.SqlType(col)) } return "ALTER TABLE " + db.Quote(tableName) + " " + strings.Join(statements, ", ") + ";" } + +func (db *Postgres) CleanDB() error { + sess := db.engine.NewSession() + defer sess.Close() + + if _, err := sess.Exec("DROP SCHEMA public CASCADE;"); err != nil { + return fmt.Errorf("Failed to drop schema public") + } + + if _, err := sess.Exec("CREATE SCHEMA public;"); err != nil { + return fmt.Errorf("Failed to create schema public") + } + + return nil +} + +func (db *Postgres) IsUniqueConstraintViolation(err error) bool { + if driverErr, ok := err.(*pq.Error); ok { + if driverErr.Code == "23505" { + return true + } + } + + return false +} diff --git a/pkg/services/sqlstore/migrator/sqlite_dialect.go b/pkg/services/sqlstore/migrator/sqlite_dialect.go index 1a31cee4f5e..446e3fcef12 100644 --- a/pkg/services/sqlstore/migrator/sqlite_dialect.go +++ b/pkg/services/sqlstore/migrator/sqlite_dialect.go @@ -1,14 +1,20 @@ package migrator -import "fmt" +import ( + "fmt" + + "github.com/go-xorm/xorm" + sqlite3 "github.com/mattn/go-sqlite3" +) type Sqlite3 struct { BaseDialect } -func NewSqlite3Dialect() *Sqlite3 { +func NewSqlite3Dialect(engine *xorm.Engine) *Sqlite3 { d := Sqlite3{} d.BaseDialect.dialect = &d + d.BaseDialect.engine = engine d.BaseDialect.driverName = SQLITE return &d } @@ -21,10 +27,6 @@ func (db *Sqlite3) Quote(name string) string { return "`" + name + "`" } -func (db *Sqlite3) QuoteStr() string { - return "`" -} - func (db *Sqlite3) AutoIncrStr() string { return "AUTOINCREMENT" } @@ -77,3 +79,17 @@ func (db *Sqlite3) DropIndexSql(tableName string, index *Index) string { idxName := index.XName(tableName) return fmt.Sprintf("DROP INDEX %v", quote(idxName)) } + +func (db *Sqlite3) CleanDB() error { + return nil +} + +func (db *Sqlite3) IsUniqueConstraintViolation(err error) bool { + if driverErr, ok := err.(sqlite3.Error); ok { + if driverErr.ExtendedCode == sqlite3.ErrConstraintUnique { + return true + } + } + + return false +} diff --git a/pkg/services/sqlstore/migrator/types.go b/pkg/services/sqlstore/migrator/types.go index d42eba0f58a..48354998d8d 100644 --- a/pkg/services/sqlstore/migrator/types.go +++ b/pkg/services/sqlstore/migrator/types.go @@ -3,12 +3,15 @@ package migrator import ( "fmt" "strings" + + "github.com/go-xorm/xorm" ) const ( POSTGRES = "postgres" SQLITE = "sqlite3" MYSQL = "mysql" + MSSQL = "mssql" ) type Migration interface { @@ -18,6 +21,11 @@ type Migration interface { GetCondition() MigrationCondition } +type CodeMigration interface { + Migration + Exec(sess *xorm.Session, migrator *Migrator) error +} + type SQLType string type ColumnType string @@ -46,7 +54,7 @@ type Index struct { func (index *Index) XName(tableName string) string { if index.Name == "" { - index.Name = fmt.Sprintf("%s", strings.Join(index.Cols, "_")) + index.Name = strings.Join(index.Cols, "_") } if !strings.HasPrefix(index.Name, "UQE_") && diff --git a/pkg/services/sqlstore/org.go b/pkg/services/sqlstore/org.go index 8931f1cf0f5..e36a80322d8 100644 --- a/pkg/services/sqlstore/org.go +++ b/pkg/services/sqlstore/org.go @@ -133,7 +133,7 @@ func UpdateOrg(cmd *m.UpdateOrgCommand) error { Updated: time.Now(), } - affectedRows, err := sess.Id(cmd.OrgId).Update(&org) + affectedRows, err := sess.ID(cmd.OrgId).Update(&org) if err != nil { return err @@ -166,7 +166,7 @@ func UpdateOrgAddress(cmd *m.UpdateOrgAddressCommand) error { Updated: time.Now(), } - if _, err := sess.Id(cmd.OrgId).Update(&org); err != nil { + if _, err := sess.ID(cmd.OrgId).Update(&org); err != nil { return err } diff --git a/pkg/services/sqlstore/org_test.go b/pkg/services/sqlstore/org_test.go index c57d15a48d5..c02686c24ba 100644 --- a/pkg/services/sqlstore/org_test.go +++ b/pkg/services/sqlstore/org_test.go @@ -1,7 +1,9 @@ package sqlstore import ( + "context" "testing" + "time" . "github.com/smartystreets/goconvey/convey" @@ -15,15 +17,16 @@ func TestAccountDataAccess(t *testing.T) { Convey("Given single org mode", func() { setting.AutoAssignOrg = true + setting.AutoAssignOrgId = 1 setting.AutoAssignOrgRole = "Viewer" Convey("Users should be added to default organization", func() { ac1cmd := m.CreateUserCommand{Login: "ac1", Email: "ac1@test.com", Name: "ac1 name"} ac2cmd := m.CreateUserCommand{Login: "ac2", Email: "ac2@test.com", Name: "ac2 name"} - err := CreateUser(&ac1cmd) + err := CreateUser(context.Background(), &ac1cmd) So(err, ShouldBeNil) - err = CreateUser(&ac2cmd) + err = CreateUser(context.Background(), &ac2cmd) So(err, ShouldBeNil) q1 := m.GetUserOrgListQuery{UserId: ac1cmd.Result.Id} @@ -42,8 +45,8 @@ func TestAccountDataAccess(t *testing.T) { ac1cmd := m.CreateUserCommand{Login: "ac1", Email: "ac1@test.com", Name: "ac1 name"} ac2cmd := m.CreateUserCommand{Login: "ac2", Email: "ac2@test.com", Name: "ac2 name", IsAdmin: true} - err := CreateUser(&ac1cmd) - err = CreateUser(&ac2cmd) + err := CreateUser(context.Background(), &ac1cmd) + err = CreateUser(context.Background(), &ac2cmd) So(err, ShouldBeNil) ac1 := ac1cmd.Result @@ -149,7 +152,7 @@ func TestAccountDataAccess(t *testing.T) { }) Convey("Can set using org", func() { - cmd := m.SetUsingOrgCommand{UserId: ac2.Id, OrgId: ac1.Id} + cmd := m.SetUsingOrgCommand{UserId: ac2.Id, OrgId: ac1.OrgId} err := SetUsingOrg(&cmd) So(err, ShouldBeNil) @@ -158,13 +161,40 @@ func TestAccountDataAccess(t *testing.T) { err := GetSignedInUser(&query) So(err, ShouldBeNil) - So(query.Result.OrgId, ShouldEqual, ac1.Id) + So(query.Result.OrgId, ShouldEqual, ac1.OrgId) So(query.Result.Email, ShouldEqual, "ac2@test.com") So(query.Result.Name, ShouldEqual, "ac2 name") So(query.Result.Login, ShouldEqual, "ac2") So(query.Result.OrgName, ShouldEqual, "ac1@test.com") So(query.Result.OrgRole, ShouldEqual, "Viewer") }) + + Convey("Should set last org as current when removing user from current", func() { + remCmd := m.RemoveOrgUserCommand{OrgId: ac1.OrgId, UserId: ac2.Id} + err := RemoveOrgUser(&remCmd) + So(err, ShouldBeNil) + + query := m.GetSignedInUserQuery{UserId: ac2.Id} + err = GetSignedInUser(&query) + + So(err, ShouldBeNil) + So(query.Result.OrgId, ShouldEqual, ac2.OrgId) + }) + }) + + Convey("Removing user from org should delete user completely if in no other org", func() { + // make sure ac2 has no org + err := DeleteOrg(&m.DeleteOrgCommand{Id: ac2.OrgId}) + So(err, ShouldBeNil) + + // remove frome ac2 from ac1 org + remCmd := m.RemoveOrgUserCommand{OrgId: ac1.OrgId, UserId: ac2.Id, ShouldDeleteOrphanedUser: true} + err = RemoveOrgUser(&remCmd) + So(err, ShouldBeNil) + So(remCmd.UserWasDeleted, ShouldBeTrue) + + err = GetSignedInUser(&m.GetSignedInUserQuery{UserId: ac2.Id}) + So(err, ShouldEqual, m.ErrUserNotFound) }) Convey("Cannot delete last admin org user", func() { @@ -181,7 +211,7 @@ func TestAccountDataAccess(t *testing.T) { Convey("Given an org user with dashboard permissions", func() { ac3cmd := m.CreateUserCommand{Login: "ac3", Email: "ac3@test.com", Name: "ac3 name", IsAdmin: false} - err := CreateUser(&ac3cmd) + err := CreateUser(context.Background(), &ac3cmd) So(err, ShouldBeNil) ac3 := ac3cmd.Result @@ -241,6 +271,8 @@ func TestAccountDataAccess(t *testing.T) { func testHelperUpdateDashboardAcl(dashboardId int64, items ...m.DashboardAcl) error { cmd := m.UpdateDashboardAclCommand{DashboardId: dashboardId} for _, item := range items { + item.Created = time.Now() + item.Updated = time.Now() cmd.Items = append(cmd.Items, &item) } return UpdateDashboardAcl(&cmd) diff --git a/pkg/services/sqlstore/org_users.go b/pkg/services/sqlstore/org_users.go index 0b991c73c55..abbc320020e 100644 --- a/pkg/services/sqlstore/org_users.go +++ b/pkg/services/sqlstore/org_users.go @@ -20,7 +20,14 @@ func init() { func AddOrgUser(cmd *m.AddOrgUserCommand) error { return inTransaction(func(sess *DBSession) error { // check if user exists - if res, err := sess.Query("SELECT 1 from org_user WHERE org_id=? and user_id=?", cmd.OrgId, cmd.UserId); err != nil { + var user m.User + if exists, err := sess.ID(cmd.UserId).Get(&user); err != nil { + return err + } else if !exists { + return m.ErrUserNotFound + } + + if res, err := sess.Query("SELECT 1 from org_user WHERE org_id=? and user_id=?", cmd.OrgId, user.Id); err != nil { return err } else if len(res) == 1 { return m.ErrOrgUserAlreadyAdded @@ -41,7 +48,26 @@ func AddOrgUser(cmd *m.AddOrgUserCommand) error { } _, err := sess.Insert(&entity) - return err + if err != nil { + return err + } + + var userOrgs []*m.UserOrgDTO + sess.Table("org_user") + sess.Join("INNER", "org", "org_user.org_id=org.id") + sess.Where("org_user.user_id=? AND org_user.org_id=?", user.Id, user.OrgId) + sess.Cols("org.name", "org_user.role", "org_user.org_id") + err = sess.Find(&userOrgs) + + if err != nil { + return err + } + + if len(userOrgs) == 0 { + return setUsingOrgInTransaction(sess, user.Id, cmd.OrgId) + } + + return nil }) } @@ -59,7 +85,7 @@ func UpdateOrgUser(cmd *m.UpdateOrgUserCommand) error { orgUser.Role = cmd.Role orgUser.Updated = time.Now() - _, err = sess.Id(orgUser.Id).Update(&orgUser) + _, err = sess.ID(orgUser.Id).Update(&orgUser) if err != nil { return err } @@ -110,6 +136,14 @@ func GetOrgUsers(query *m.GetOrgUsersQuery) error { func RemoveOrgUser(cmd *m.RemoveOrgUserCommand) error { return inTransaction(func(sess *DBSession) error { + // check if user exists + var user m.User + if exists, err := sess.ID(cmd.UserId).Get(&user); err != nil { + return err + } else if !exists { + return m.ErrUserNotFound + } + deletes := []string{ "DELETE FROM org_user WHERE org_id=? and user_id=?", "DELETE FROM dashboard_acl WHERE org_id=? and user_id = ?", @@ -123,7 +157,48 @@ func RemoveOrgUser(cmd *m.RemoveOrgUserCommand) error { } } - return validateOneAdminLeftInOrg(cmd.OrgId, sess) + // validate that after delete there is at least one user with admin role in org + if err := validateOneAdminLeftInOrg(cmd.OrgId, sess); err != nil { + return err + } + + // check user other orgs and update user current org + var userOrgs []*m.UserOrgDTO + sess.Table("org_user") + sess.Join("INNER", "org", "org_user.org_id=org.id") + sess.Where("org_user.user_id=?", user.Id) + sess.Cols("org.name", "org_user.role", "org_user.org_id") + err := sess.Find(&userOrgs) + + if err != nil { + return err + } + + if len(userOrgs) > 0 { + hasCurrentOrgSet := false + for _, userOrg := range userOrgs { + if user.OrgId == userOrg.OrgId { + hasCurrentOrgSet = true + break + } + } + + if !hasCurrentOrgSet { + err = setUsingOrgInTransaction(sess, user.Id, userOrgs[0].OrgId) + if err != nil { + return err + } + } + } else if cmd.ShouldDeleteOrphanedUser { + // no other orgs, delete the full user + if err := deleteUserInTransaction(sess, &m.DeleteUserCommand{UserId: user.Id}); err != nil { + return err + } + + cmd.UserWasDeleted = true + } + + return nil }) } diff --git a/pkg/services/sqlstore/playlist.go b/pkg/services/sqlstore/playlist.go index 67720cbadb8..99519fe762f 100644 --- a/pkg/services/sqlstore/playlist.go +++ b/pkg/services/sqlstore/playlist.go @@ -22,6 +22,9 @@ func CreatePlaylist(cmd *m.CreatePlaylistCommand) error { } _, err := x.Insert(&playlist) + if err != nil { + return err + } playlistItems := make([]m.PlaylistItem, 0) for _, item := range cmd.Items { @@ -61,7 +64,7 @@ func UpdatePlaylist(cmd *m.UpdatePlaylistCommand) error { Interval: playlist.Interval, } - _, err := x.ID(cmd.Id).Cols("id", "name", "interval").Update(&playlist) + _, err := x.ID(cmd.Id).Cols("name", "interval").Update(&playlist) if err != nil { return err diff --git a/pkg/services/sqlstore/plugin_setting.go b/pkg/services/sqlstore/plugin_setting.go index 172995872eb..8fbf1b6be1c 100644 --- a/pkg/services/sqlstore/plugin_setting.go +++ b/pkg/services/sqlstore/plugin_setting.go @@ -26,7 +26,7 @@ func GetPluginSettings(query *m.GetPluginSettingsQuery) error { params = append(params, query.OrgId) } - sess := x.Sql(sql, params...) + sess := x.SQL(sql, params...) query.Result = make([]*m.PluginSettingInfoDTO, 0) return sess.Find(&query.Result) } @@ -36,7 +36,7 @@ func GetPluginSettingById(query *m.GetPluginSettingByIdQuery) error { has, err := x.Get(&pluginSetting) if err != nil { return err - } else if has == false { + } else if !has { return m.ErrPluginSettingNotFound } query.Result = &pluginSetting @@ -48,6 +48,9 @@ func UpdatePluginSetting(cmd *m.UpdatePluginSettingCmd) error { var pluginSetting m.PluginSetting exists, err := sess.Where("org_id=? and plugin_id=?", cmd.OrgId, cmd.PluginId).Get(&pluginSetting) + if err != nil { + return err + } sess.UseBool("enabled") sess.UseBool("pinned") if !exists { @@ -72,34 +75,33 @@ func UpdatePluginSetting(cmd *m.UpdatePluginSettingCmd) error { _, err = sess.Insert(&pluginSetting) return err - } else { - for key, data := range cmd.SecureJsonData { - encryptedData, err := util.Encrypt([]byte(data), setting.SecretKey) - if err != nil { - return err - } - - pluginSetting.SecureJsonData[key] = encryptedData - } - - // add state change event on commit success - if pluginSetting.Enabled != cmd.Enabled { - sess.events = append(sess.events, &m.PluginStateChangedEvent{ - PluginId: cmd.PluginId, - OrgId: cmd.OrgId, - Enabled: cmd.Enabled, - }) - } - - pluginSetting.Updated = time.Now() - pluginSetting.Enabled = cmd.Enabled - pluginSetting.JsonData = cmd.JsonData - pluginSetting.Pinned = cmd.Pinned - pluginSetting.PluginVersion = cmd.PluginVersion - - _, err = sess.Id(pluginSetting.Id).Update(&pluginSetting) - return err } + for key, data := range cmd.SecureJsonData { + encryptedData, err := util.Encrypt([]byte(data), setting.SecretKey) + if err != nil { + return err + } + + pluginSetting.SecureJsonData[key] = encryptedData + } + + // add state change event on commit success + if pluginSetting.Enabled != cmd.Enabled { + sess.events = append(sess.events, &m.PluginStateChangedEvent{ + PluginId: cmd.PluginId, + OrgId: cmd.OrgId, + Enabled: cmd.Enabled, + }) + } + + pluginSetting.Updated = time.Now() + pluginSetting.Enabled = cmd.Enabled + pluginSetting.JsonData = cmd.JsonData + pluginSetting.Pinned = cmd.Pinned + pluginSetting.PluginVersion = cmd.PluginVersion + + _, err = sess.ID(pluginSetting.Id).Update(&pluginSetting) + return err }) } diff --git a/pkg/services/sqlstore/preferences.go b/pkg/services/sqlstore/preferences.go index 399b23f3ffa..858a2c77075 100644 --- a/pkg/services/sqlstore/preferences.go +++ b/pkg/services/sqlstore/preferences.go @@ -1,6 +1,7 @@ package sqlstore import ( + "strings" "time" "github.com/grafana/grafana/pkg/bus" @@ -16,11 +17,22 @@ func init() { } func GetPreferencesWithDefaults(query *m.GetPreferencesWithDefaultsQuery) error { - + params := make([]interface{}, 0) + filter := "" + if len(query.User.Teams) > 0 { + filter = "(org_id=? AND team_id IN (?" + strings.Repeat(",?", len(query.User.Teams)-1) + ")) OR " + params = append(params, query.User.OrgId) + for _, v := range query.User.Teams { + params = append(params, v) + } + } + filter += "(org_id=? AND user_id=? AND team_id=0) OR (org_id=? AND team_id=0 AND user_id=0)" + params = append(params, query.User.OrgId) + params = append(params, query.User.UserId) + params = append(params, query.User.OrgId) prefs := make([]*m.Preferences, 0) - filter := "(org_id=? AND user_id=?) OR (org_id=? AND user_id=0)" - err := x.Where(filter, query.OrgId, query.UserId, query.OrgId). - OrderBy("user_id ASC"). + err := x.Where(filter, params...). + OrderBy("user_id ASC, team_id ASC"). Find(&prefs) if err != nil { @@ -50,9 +62,8 @@ func GetPreferencesWithDefaults(query *m.GetPreferencesWithDefaultsQuery) error } func GetPreferences(query *m.GetPreferencesQuery) error { - var prefs m.Preferences - exists, err := x.Where("org_id=? AND user_id=?", query.OrgId, query.UserId).Get(&prefs) + exists, err := x.Where("org_id=? AND user_id=? AND team_id=?", query.OrgId, query.UserId, query.TeamId).Get(&prefs) if err != nil { return err @@ -71,12 +82,16 @@ func SavePreferences(cmd *m.SavePreferencesCommand) error { return inTransaction(func(sess *DBSession) error { var prefs m.Preferences - exists, err := sess.Where("org_id=? AND user_id=?", cmd.OrgId, cmd.UserId).Get(&prefs) + exists, err := sess.Where("org_id=? AND user_id=? AND team_id=?", cmd.OrgId, cmd.UserId, cmd.TeamId).Get(&prefs) + if err != nil { + return err + } if !exists { prefs = m.Preferences{ UserId: cmd.UserId, OrgId: cmd.OrgId, + TeamId: cmd.TeamId, HomeDashboardId: cmd.HomeDashboardId, Timezone: cmd.Timezone, Theme: cmd.Theme, @@ -85,14 +100,13 @@ func SavePreferences(cmd *m.SavePreferencesCommand) error { } _, err = sess.Insert(&prefs) return err - } else { - prefs.HomeDashboardId = cmd.HomeDashboardId - prefs.Timezone = cmd.Timezone - prefs.Theme = cmd.Theme - prefs.Updated = time.Now() - prefs.Version += 1 - _, err := sess.Id(prefs.Id).AllCols().Update(&prefs) - return err } + prefs.HomeDashboardId = cmd.HomeDashboardId + prefs.Timezone = cmd.Timezone + prefs.Theme = cmd.Theme + prefs.Updated = time.Now() + prefs.Version += 1 + _, err = sess.ID(prefs.Id).AllCols().Update(&prefs) + return err }) } diff --git a/pkg/services/sqlstore/preferences_test.go b/pkg/services/sqlstore/preferences_test.go new file mode 100644 index 00000000000..f9a839bf5a7 --- /dev/null +++ b/pkg/services/sqlstore/preferences_test.go @@ -0,0 +1,91 @@ +package sqlstore + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" +) + +func TestPreferencesDataAccess(t *testing.T) { + Convey("Testing preferences data access", t, func() { + InitTestDB(t) + + Convey("GetPreferencesWithDefaults with no saved preferences should return defaults", func() { + query := &models.GetPreferencesWithDefaultsQuery{User: &models.SignedInUser{}} + err := GetPreferencesWithDefaults(query) + So(err, ShouldBeNil) + So(query.Result.Theme, ShouldEqual, setting.DefaultTheme) + So(query.Result.Timezone, ShouldEqual, "browser") + So(query.Result.HomeDashboardId, ShouldEqual, 0) + }) + + Convey("GetPreferencesWithDefaults with saved org and user home dashboard should return user home dashboard", func() { + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, HomeDashboardId: 1}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, UserId: 1, HomeDashboardId: 4}) + + query := &models.GetPreferencesWithDefaultsQuery{User: &models.SignedInUser{OrgId: 1, UserId: 1}} + err := GetPreferencesWithDefaults(query) + So(err, ShouldBeNil) + So(query.Result.HomeDashboardId, ShouldEqual, 4) + }) + + Convey("GetPreferencesWithDefaults with saved org and other user home dashboard should return org home dashboard", func() { + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, HomeDashboardId: 1}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, UserId: 1, HomeDashboardId: 4}) + + query := &models.GetPreferencesWithDefaultsQuery{User: &models.SignedInUser{OrgId: 1, UserId: 2}} + err := GetPreferencesWithDefaults(query) + So(err, ShouldBeNil) + So(query.Result.HomeDashboardId, ShouldEqual, 1) + }) + + Convey("GetPreferencesWithDefaults with saved org and teams home dashboard should return last team home dashboard", func() { + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, HomeDashboardId: 1}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 2, HomeDashboardId: 2}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 3, HomeDashboardId: 3}) + + query := &models.GetPreferencesWithDefaultsQuery{User: &models.SignedInUser{OrgId: 1, Teams: []int64{2, 3}}} + err := GetPreferencesWithDefaults(query) + So(err, ShouldBeNil) + So(query.Result.HomeDashboardId, ShouldEqual, 3) + }) + + Convey("GetPreferencesWithDefaults with saved org and other teams home dashboard should return org home dashboard", func() { + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, HomeDashboardId: 1}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 2, HomeDashboardId: 2}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 3, HomeDashboardId: 3}) + + query := &models.GetPreferencesWithDefaultsQuery{User: &models.SignedInUser{OrgId: 1}} + err := GetPreferencesWithDefaults(query) + So(err, ShouldBeNil) + So(query.Result.HomeDashboardId, ShouldEqual, 1) + }) + + Convey("GetPreferencesWithDefaults with saved org, teams and user home dashboard should return user home dashboard", func() { + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, HomeDashboardId: 1}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 2, HomeDashboardId: 2}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 3, HomeDashboardId: 3}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, UserId: 1, HomeDashboardId: 4}) + + query := &models.GetPreferencesWithDefaultsQuery{User: &models.SignedInUser{OrgId: 1, UserId: 1, Teams: []int64{2, 3}}} + err := GetPreferencesWithDefaults(query) + So(err, ShouldBeNil) + So(query.Result.HomeDashboardId, ShouldEqual, 4) + }) + + Convey("GetPreferencesWithDefaults with saved org, other teams and user home dashboard should return org home dashboard", func() { + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, HomeDashboardId: 1}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 2, HomeDashboardId: 2}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, TeamId: 3, HomeDashboardId: 3}) + SavePreferences(&models.SavePreferencesCommand{OrgId: 1, UserId: 1, HomeDashboardId: 4}) + + query := &models.GetPreferencesWithDefaultsQuery{User: &models.SignedInUser{OrgId: 1, UserId: 2}} + err := GetPreferencesWithDefaults(query) + So(err, ShouldBeNil) + So(query.Result.HomeDashboardId, ShouldEqual, 1) + }) + }) +} diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index 0a857efce40..7005b341268 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -2,6 +2,7 @@ package sqlstore import ( "fmt" + "time" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" @@ -30,14 +31,14 @@ func GetOrgQuotaByTarget(query *m.GetOrgQuotaByTargetQuery) error { has, err := x.Get("a) if err != nil { return err - } else if has == false { + } else if !has { quota.Limit = query.Default } //get quota used. rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(query.Target)) resp := make([]*targetCount, 0) - if err := x.Sql(rawSql, query.OrgId).Find(&resp); err != nil { + if err := x.SQL(rawSql, query.OrgId).Find(&resp); err != nil { return err } @@ -80,7 +81,7 @@ func GetOrgQuotas(query *m.GetOrgQuotasQuery) error { //get quota used. rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(q.Target)) resp := make([]*targetCount, 0) - if err := x.Sql(rawSql, q.OrgId).Find(&resp); err != nil { + if err := x.SQL(rawSql, q.OrgId).Find(&resp); err != nil { return err } result[i] = &m.OrgQuotaDTO{ @@ -98,22 +99,24 @@ func UpdateOrgQuota(cmd *m.UpdateOrgQuotaCmd) error { return inTransaction(func(sess *DBSession) error { //Check if quota is already defined in the DB quota := m.Quota{ - Target: cmd.Target, - OrgId: cmd.OrgId, + Target: cmd.Target, + OrgId: cmd.OrgId, + Updated: time.Now(), } has, err := sess.Get("a) if err != nil { return err } quota.Limit = cmd.Limit - if has == false { + if !has { + quota.Created = time.Now() //No quota in the DB for this target, so create a new one. if _, err := sess.Insert("a); err != nil { return err } } else { //update existing quota entry in the DB. - if _, err := sess.Id(quota.Id).Update("a); err != nil { + if _, err := sess.ID(quota.Id).Update("a); err != nil { return err } } @@ -130,14 +133,14 @@ func GetUserQuotaByTarget(query *m.GetUserQuotaByTargetQuery) error { has, err := x.Get("a) if err != nil { return err - } else if has == false { + } else if !has { quota.Limit = query.Default } //get quota used. rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where user_id=?", dialect.Quote(query.Target)) resp := make([]*targetCount, 0) - if err := x.Sql(rawSql, query.UserId).Find(&resp); err != nil { + if err := x.SQL(rawSql, query.UserId).Find(&resp); err != nil { return err } @@ -180,7 +183,7 @@ func GetUserQuotas(query *m.GetUserQuotasQuery) error { //get quota used. rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where user_id=?", dialect.Quote(q.Target)) resp := make([]*targetCount, 0) - if err := x.Sql(rawSql, q.UserId).Find(&resp); err != nil { + if err := x.SQL(rawSql, q.UserId).Find(&resp); err != nil { return err } result[i] = &m.UserQuotaDTO{ @@ -198,22 +201,24 @@ func UpdateUserQuota(cmd *m.UpdateUserQuotaCmd) error { return inTransaction(func(sess *DBSession) error { //Check if quota is already defined in the DB quota := m.Quota{ - Target: cmd.Target, - UserId: cmd.UserId, + Target: cmd.Target, + UserId: cmd.UserId, + Updated: time.Now(), } has, err := sess.Get("a) if err != nil { return err } quota.Limit = cmd.Limit - if has == false { + if !has { + quota.Created = time.Now() //No quota in the DB for this target, so create a new one. if _, err := sess.Insert("a); err != nil { return err } } else { //update existing quota entry in the DB. - if _, err := sess.Id(quota.Id).Update("a); err != nil { + if _, err := sess.ID(quota.Id).Update("a); err != nil { return err } } @@ -226,7 +231,7 @@ func GetGlobalQuotaByTarget(query *m.GetGlobalQuotaByTargetQuery) error { //get quota used. rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s", dialect.Quote(query.Target)) resp := make([]*targetCount, 0) - if err := x.Sql(rawSql).Find(&resp); err != nil { + if err := x.SQL(rawSql).Find(&resp); err != nil { return err } diff --git a/pkg/services/sqlstore/quota_test.go b/pkg/services/sqlstore/quota_test.go index 5ef618e166d..49e028e9cd3 100644 --- a/pkg/services/sqlstore/quota_test.go +++ b/pkg/services/sqlstore/quota_test.go @@ -43,6 +43,7 @@ func TestQuotaCommandsAndQueries(t *testing.T) { Name: "TestOrg", UserId: 1, } + err := CreateOrg(&userCmd) So(err, ShouldBeNil) orgId = userCmd.Result.Id @@ -104,12 +105,12 @@ func TestQuotaCommandsAndQueries(t *testing.T) { }) }) Convey("Given saved user quota for org", func() { - userQoutaCmd := m.UpdateUserQuotaCmd{ + userQuotaCmd := m.UpdateUserQuotaCmd{ UserId: userId, Target: "org_user", Limit: 10, } - err := UpdateUserQuota(&userQoutaCmd) + err := UpdateUserQuota(&userQuotaCmd) So(err, ShouldBeNil) Convey("Should be able to get saved quota by user id and target", func() { diff --git a/pkg/services/sqlstore/search_builder.go b/pkg/services/sqlstore/search_builder.go index ddfbfbfc551..7817c4635c9 100644 --- a/pkg/services/sqlstore/search_builder.go +++ b/pkg/services/sqlstore/search_builder.go @@ -92,7 +92,7 @@ func (sb *SearchBuilder) ToSql() (string, []interface{}) { LEFT OUTER JOIN dashboard folder on folder.id = dashboard.folder_id LEFT OUTER JOIN dashboard_tag on dashboard.id = dashboard_tag.dashboard_id`) - sb.sql.WriteString(" ORDER BY dashboard.title ASC LIMIT 5000") + sb.sql.WriteString(" ORDER BY dashboard.title ASC" + dialect.Limit(5000)) return sb.sql.String(), sb.params } @@ -135,12 +135,11 @@ func (sb *SearchBuilder) buildTagQuery() { // this ends the inner select (tag filtered part) sb.sql.WriteString(` GROUP BY dashboard.id HAVING COUNT(dashboard.id) >= ? - LIMIT ?) as ids + ORDER BY dashboard.id` + dialect.Limit(int64(sb.limit)) + `) as ids INNER JOIN dashboard on ids.id = dashboard.id `) sb.params = append(sb.params, len(sb.tags)) - sb.params = append(sb.params, sb.limit) } func (sb *SearchBuilder) buildMainQuery() { @@ -153,8 +152,7 @@ func (sb *SearchBuilder) buildMainQuery() { sb.sql.WriteString(` WHERE `) sb.buildSearchWhereClause() - sb.sql.WriteString(` LIMIT ?) as ids INNER JOIN dashboard on ids.id = dashboard.id `) - sb.params = append(sb.params, sb.limit) + sb.sql.WriteString(` ORDER BY dashboard.title` + dialect.Limit(int64(sb.limit)) + `) as ids INNER JOIN dashboard on ids.id = dashboard.id `) } func (sb *SearchBuilder) buildSearchWhereClause() { diff --git a/pkg/services/sqlstore/search_builder_test.go b/pkg/services/sqlstore/search_builder_test.go index e8b02c445ec..e7ec3eedac6 100644 --- a/pkg/services/sqlstore/search_builder_test.go +++ b/pkg/services/sqlstore/search_builder_test.go @@ -4,13 +4,10 @@ import ( "testing" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" . "github.com/smartystreets/goconvey/convey" ) func TestSearchBuilder(t *testing.T) { - dialect = migrator.NewDialect("sqlite3") - Convey("Testing building a search", t, func() { signedInUser := &m.SignedInUser{ OrgId: 1, @@ -23,7 +20,7 @@ func TestSearchBuilder(t *testing.T) { sql, params := sb.IsStarred().WithTitle("test").ToSql() So(sql, ShouldStartWith, "SELECT") So(sql, ShouldContainSubstring, "INNER JOIN dashboard on ids.id = dashboard.id") - So(sql, ShouldEndWith, "ORDER BY dashboard.title ASC LIMIT 5000") + So(sql, ShouldContainSubstring, "ORDER BY dashboard.title ASC") So(len(params), ShouldBeGreaterThan, 0) }) @@ -31,7 +28,7 @@ func TestSearchBuilder(t *testing.T) { sql, params := sb.WithTags([]string{"tag1", "tag2"}).ToSql() So(sql, ShouldStartWith, "SELECT") So(sql, ShouldContainSubstring, "LEFT OUTER JOIN dashboard_tag") - So(sql, ShouldEndWith, "ORDER BY dashboard.title ASC LIMIT 5000") + So(sql, ShouldContainSubstring, "ORDER BY dashboard.title ASC") So(len(params), ShouldBeGreaterThan, 0) }) }) diff --git a/pkg/services/sqlstore/session.go b/pkg/services/sqlstore/session.go new file mode 100644 index 00000000000..29d7392678f --- /dev/null +++ b/pkg/services/sqlstore/session.go @@ -0,0 +1,71 @@ +package sqlstore + +import ( + "context" + "reflect" + + "github.com/go-xorm/xorm" +) + +type DBSession struct { + *xorm.Session + events []interface{} +} + +type dbTransactionFunc func(sess *DBSession) error + +func (sess *DBSession) publishAfterCommit(msg interface{}) { + sess.events = append(sess.events, msg) +} + +func newSession() *DBSession { + return &DBSession{Session: x.NewSession()} +} + +func startSession(ctx context.Context, engine *xorm.Engine, beginTran bool) (*DBSession, error) { + value := ctx.Value(ContextSessionName) + var sess *DBSession + sess, ok := value.(*DBSession) + + if ok { + return sess, nil + } + + newSess := &DBSession{Session: engine.NewSession()} + if beginTran { + err := newSess.Begin() + if err != nil { + return nil, err + } + } + return newSess, nil +} + +func withDbSession(ctx context.Context, callback dbTransactionFunc) error { + sess, err := startSession(ctx, x, false) + if err != nil { + return err + } + + return callback(sess) +} + +func (sess *DBSession) InsertId(bean interface{}) (int64, error) { + table := sess.DB().Mapper.Obj2Table(getTypeName(bean)) + + dialect.PreInsertId(table, sess.Session) + + id, err := sess.Session.InsertOne(bean) + + dialect.PostInsertId(table, sess.Session) + + return id, err +} + +func getTypeName(bean interface{}) (res string) { + t := reflect.TypeOf(bean) + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t.Name() +} diff --git a/pkg/services/sqlstore/shared.go b/pkg/services/sqlstore/shared.go deleted file mode 100644 index 0f4aeb969c6..00000000000 --- a/pkg/services/sqlstore/shared.go +++ /dev/null @@ -1,69 +0,0 @@ -package sqlstore - -import ( - "time" - - "github.com/go-xorm/xorm" - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/log" - sqlite3 "github.com/mattn/go-sqlite3" -) - -type DBSession struct { - *xorm.Session - events []interface{} -} - -type dbTransactionFunc func(sess *DBSession) error - -func (sess *DBSession) publishAfterCommit(msg interface{}) { - sess.events = append(sess.events, msg) -} - -func newSession() *DBSession { - return &DBSession{Session: x.NewSession()} -} - -func inTransaction(callback dbTransactionFunc) error { - return inTransactionWithRetry(callback, 0) -} - -func inTransactionWithRetry(callback dbTransactionFunc, retry int) error { - var err error - - sess := newSession() - defer sess.Close() - - if err = sess.Begin(); err != nil { - return err - } - - err = callback(sess) - - // special handling of database locked errors for sqlite, then we can retry 3 times - if sqlError, ok := err.(sqlite3.Error); ok && retry < 5 { - if sqlError.Code == sqlite3.ErrLocked { - sess.Rollback() - time.Sleep(time.Millisecond * time.Duration(10)) - sqlog.Info("Database table locked, sleeping then retrying", "retry", retry) - return inTransactionWithRetry(callback, retry+1) - } - } - - if err != nil { - sess.Rollback() - return err - } else if err = sess.Commit(); err != nil { - return err - } - - if len(sess.events) > 0 { - for _, e := range sess.events { - if err = bus.Publish(e); err != nil { - log.Error(3, "Failed to publish event after commit", err) - } - } - } - - return nil -} diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index c00a55667d1..95b53be9d4a 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "fmt" "net/url" "os" @@ -8,256 +9,395 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/annotations" + "github.com/grafana/grafana/pkg/services/cache" "github.com/grafana/grafana/pkg/services/sqlstore/migrations" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" "github.com/grafana/grafana/pkg/setting" "github.com/go-sql-driver/mysql" - _ "github.com/go-sql-driver/mysql" "github.com/go-xorm/xorm" - _ "github.com/lib/pq" - _ "github.com/mattn/go-sqlite3" _ "github.com/grafana/grafana/pkg/tsdb/mssql" + _ "github.com/lib/pq" + sqlite3 "github.com/mattn/go-sqlite3" ) +var ( + x *xorm.Engine + dialect migrator.Dialect + + sqlog log.Logger = log.New("sqlstore") +) + +const ContextSessionName = "db-session" + +func init() { + registry.Register(®istry.Descriptor{ + Name: "SqlStore", + Instance: &SqlStore{}, + InitPriority: registry.High, + }) +} + +type SqlStore struct { + Cfg *setting.Cfg `inject:""` + Bus bus.Bus `inject:""` + CacheService *cache.CacheService `inject:""` + + dbCfg DatabaseConfig + engine *xorm.Engine + log log.Logger + Dialect migrator.Dialect + skipEnsureAdmin bool +} + +// NewSession returns a new DBSession +func (ss *SqlStore) NewSession() *DBSession { + return &DBSession{Session: ss.engine.NewSession()} +} + +// WithDbSession calls the callback with an session attached to the context. +func (ss *SqlStore) WithDbSession(ctx context.Context, callback dbTransactionFunc) error { + sess, err := startSession(ctx, ss.engine, false) + if err != nil { + return err + } + + return callback(sess) +} + +// WithTransactionalDbSession calls the callback with an session within a transaction +func (ss *SqlStore) WithTransactionalDbSession(ctx context.Context, callback dbTransactionFunc) error { + return ss.inTransactionWithRetryCtx(ctx, callback, 0) +} + +func (ss *SqlStore) inTransactionWithRetryCtx(ctx context.Context, callback dbTransactionFunc, retry int) error { + sess, err := startSession(ctx, ss.engine, true) + if err != nil { + return err + } + + defer sess.Close() + + err = callback(sess) + + // special handling of database locked errors for sqlite, then we can retry 3 times + if sqlError, ok := err.(sqlite3.Error); ok && retry < 5 { + if sqlError.Code == sqlite3.ErrLocked { + sess.Rollback() + time.Sleep(time.Millisecond * time.Duration(10)) + sqlog.Info("Database table locked, sleeping then retrying", "retry", retry) + return ss.inTransactionWithRetryCtx(ctx, callback, retry+1) + } + } + + if err != nil { + sess.Rollback() + return err + } else if err = sess.Commit(); err != nil { + return err + } + + if len(sess.events) > 0 { + for _, e := range sess.events { + if err = bus.Publish(e); err != nil { + log.Error(3, "Failed to publish event after commit. error: %v", err) + } + } + } + + return nil +} + +func (ss *SqlStore) Init() error { + ss.log = log.New("sqlstore") + ss.readConfig() + + engine, err := ss.getEngine() + + if err != nil { + return fmt.Errorf("Fail to connect to database: %v", err) + } + + ss.engine = engine + ss.Dialect = migrator.NewDialect(ss.engine) + + // temporarily still set global var + x = engine + dialect = ss.Dialect + + migrator := migrator.NewMigrator(x) + migrations.AddMigrations(migrator) + + for _, descriptor := range registry.GetServices() { + sc, ok := descriptor.Instance.(registry.DatabaseMigrator) + if ok { + sc.AddMigration(migrator) + } + } + + if err := migrator.Start(); err != nil { + return fmt.Errorf("Migration failed err: %v", err) + } + + // Init repo instances + annotations.SetRepository(&SqlAnnotationRepo{}) + ss.Bus.SetTransactionManager(ss) + + // Register handlers + ss.addUserQueryAndCommandHandlers() + + // ensure admin user + if ss.skipEnsureAdmin { + return nil + } + + return ss.ensureAdminUser() +} + +func (ss *SqlStore) ensureAdminUser() error { + systemUserCountQuery := m.GetSystemUserCountStatsQuery{} + + err := ss.InTransaction(context.Background(), func(ctx context.Context) error { + + err := bus.DispatchCtx(ctx, &systemUserCountQuery) + if err != nil { + return fmt.Errorf("Could not determine if admin user exists: %v", err) + } + + if systemUserCountQuery.Result.Count > 0 { + return nil + } + + cmd := m.CreateUserCommand{} + cmd.Login = setting.AdminUser + cmd.Email = setting.AdminUser + "@localhost" + cmd.Password = setting.AdminPassword + cmd.IsAdmin = true + + if err := bus.DispatchCtx(ctx, &cmd); err != nil { + return fmt.Errorf("Failed to create admin user: %v", err) + } + + ss.log.Info("Created default admin", "user", setting.AdminUser) + + return nil + }) + + return err +} + +func (ss *SqlStore) buildConnectionString() (string, error) { + cnnstr := ss.dbCfg.ConnectionString + + // special case used by integration tests + if cnnstr != "" { + return cnnstr, nil + } + + switch ss.dbCfg.Type { + case migrator.MYSQL: + protocol := "tcp" + if strings.HasPrefix(ss.dbCfg.Host, "/") { + protocol = "unix" + } + + cnnstr = fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&allowNativePasswords=true", + ss.dbCfg.User, ss.dbCfg.Pwd, protocol, ss.dbCfg.Host, ss.dbCfg.Name) + + if ss.dbCfg.SslMode == "true" || ss.dbCfg.SslMode == "skip-verify" { + tlsCert, err := makeCert("custom", ss.dbCfg) + if err != nil { + return "", err + } + mysql.RegisterTLSConfig("custom", tlsCert) + cnnstr += "&tls=custom" + } + case migrator.POSTGRES: + var host, port = "127.0.0.1", "5432" + fields := strings.Split(ss.dbCfg.Host, ":") + if len(fields) > 0 && len(strings.TrimSpace(fields[0])) > 0 { + host = fields[0] + } + if len(fields) > 1 && len(strings.TrimSpace(fields[1])) > 0 { + port = fields[1] + } + if ss.dbCfg.Pwd == "" { + ss.dbCfg.Pwd = "''" + } + if ss.dbCfg.User == "" { + ss.dbCfg.User = "''" + } + cnnstr = fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", ss.dbCfg.User, ss.dbCfg.Pwd, host, port, ss.dbCfg.Name, ss.dbCfg.SslMode, ss.dbCfg.ClientCertPath, ss.dbCfg.ClientKeyPath, ss.dbCfg.CaCertPath) + case migrator.SQLITE: + // special case for tests + if !filepath.IsAbs(ss.dbCfg.Path) { + ss.dbCfg.Path = filepath.Join(ss.Cfg.DataPath, ss.dbCfg.Path) + } + os.MkdirAll(path.Dir(ss.dbCfg.Path), os.ModePerm) + cnnstr = "file:" + ss.dbCfg.Path + "?cache=shared&mode=rwc" + default: + return "", fmt.Errorf("Unknown database type: %s", ss.dbCfg.Type) + } + + return cnnstr, nil +} + +func (ss *SqlStore) getEngine() (*xorm.Engine, error) { + connectionString, err := ss.buildConnectionString() + + if err != nil { + return nil, err + } + + sqlog.Info("Connecting to DB", "dbtype", ss.dbCfg.Type) + engine, err := xorm.NewEngine(ss.dbCfg.Type, connectionString) + if err != nil { + return nil, err + } + + engine.SetMaxOpenConns(ss.dbCfg.MaxOpenConn) + engine.SetMaxIdleConns(ss.dbCfg.MaxIdleConn) + engine.SetConnMaxLifetime(time.Second * time.Duration(ss.dbCfg.ConnMaxLifetime)) + + // configure sql logging + debugSql := ss.Cfg.Raw.Section("database").Key("log_queries").MustBool(false) + if !debugSql { + engine.SetLogger(&xorm.DiscardLogger{}) + } else { + engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm"))) + engine.ShowSQL(true) + engine.ShowExecTime(true) + } + + return engine, nil +} + +func (ss *SqlStore) readConfig() { + sec := ss.Cfg.Raw.Section("database") + + cfgURL := sec.Key("url").String() + if len(cfgURL) != 0 { + dbURL, _ := url.Parse(cfgURL) + ss.dbCfg.Type = dbURL.Scheme + ss.dbCfg.Host = dbURL.Host + + pathSplit := strings.Split(dbURL.Path, "/") + if len(pathSplit) > 1 { + ss.dbCfg.Name = pathSplit[1] + } + + userInfo := dbURL.User + if userInfo != nil { + ss.dbCfg.User = userInfo.Username() + ss.dbCfg.Pwd, _ = userInfo.Password() + } + } else { + ss.dbCfg.Type = sec.Key("type").String() + ss.dbCfg.Host = sec.Key("host").String() + ss.dbCfg.Name = sec.Key("name").String() + ss.dbCfg.User = sec.Key("user").String() + ss.dbCfg.ConnectionString = sec.Key("connection_string").String() + ss.dbCfg.Pwd = sec.Key("password").String() + } + + ss.dbCfg.MaxOpenConn = sec.Key("max_open_conn").MustInt(0) + ss.dbCfg.MaxIdleConn = sec.Key("max_idle_conn").MustInt(2) + ss.dbCfg.ConnMaxLifetime = sec.Key("conn_max_lifetime").MustInt(14400) + + ss.dbCfg.SslMode = sec.Key("ssl_mode").String() + ss.dbCfg.CaCertPath = sec.Key("ca_cert_path").String() + ss.dbCfg.ClientKeyPath = sec.Key("client_key_path").String() + ss.dbCfg.ClientCertPath = sec.Key("client_cert_path").String() + ss.dbCfg.ServerCertName = sec.Key("server_cert_name").String() + ss.dbCfg.Path = sec.Key("path").MustString("data/grafana.db") +} + +func InitTestDB(t *testing.T) *SqlStore { + t.Helper() + sqlstore := &SqlStore{} + sqlstore.skipEnsureAdmin = true + sqlstore.Bus = bus.New() + sqlstore.CacheService = cache.New(5*time.Minute, 10*time.Minute) + + dbType := migrator.SQLITE + + // environment variable present for test db? + if db, present := os.LookupEnv("GRAFANA_TEST_DB"); present { + dbType = db + } + + // set test db config + sqlstore.Cfg = setting.NewCfg() + sec, _ := sqlstore.Cfg.Raw.NewSection("database") + sec.NewKey("type", dbType) + + switch dbType { + case "mysql": + sec.NewKey("connection_string", sqlutil.TestDB_Mysql.ConnStr) + case "postgres": + sec.NewKey("connection_string", sqlutil.TestDB_Postgres.ConnStr) + default: + sec.NewKey("connection_string", sqlutil.TestDB_Sqlite3.ConnStr) + } + + // need to get engine to clean db before we init + engine, err := xorm.NewEngine(dbType, sec.Key("connection_string").String()) + if err != nil { + t.Fatalf("Failed to init test database: %v", err) + } + + sqlstore.Dialect = migrator.NewDialect(engine) + + // temp global var until we get rid of global vars + dialect = sqlstore.Dialect + + if err := dialect.CleanDB(); err != nil { + t.Fatalf("Failed to clean test db %v", err) + } + + if err := sqlstore.Init(); err != nil { + t.Fatalf("Failed to init test database: %v", err) + } + + sqlstore.engine.DatabaseTZ = time.UTC + sqlstore.engine.TZLocation = time.UTC + + return sqlstore +} + +func IsTestDbMySql() bool { + if db, present := os.LookupEnv("GRAFANA_TEST_DB"); present { + return db == migrator.MYSQL + } + + return false +} + +func IsTestDbPostgres() bool { + if db, present := os.LookupEnv("GRAFANA_TEST_DB"); present { + return db == migrator.POSTGRES + } + + return false +} + type DatabaseConfig struct { Type, Host, Name, User, Pwd, Path, SslMode string CaCertPath string ClientKeyPath string ClientCertPath string ServerCertName string + ConnectionString string MaxOpenConn int MaxIdleConn int -} - -var ( - x *xorm.Engine - dialect migrator.Dialect - - HasEngine bool - - DbCfg DatabaseConfig - - UseSQLite3 bool - sqlog log.Logger = log.New("sqlstore") -) - -func EnsureAdminUser() { - statsQuery := m.GetSystemStatsQuery{} - - if err := bus.Dispatch(&statsQuery); err != nil { - log.Fatal(3, "Could not determine if admin user exists: %v", err) - return - } - - if statsQuery.Result.Users > 0 { - return - } - - cmd := m.CreateUserCommand{} - cmd.Login = setting.AdminUser - cmd.Email = setting.AdminUser + "@localhost" - cmd.Password = setting.AdminPassword - cmd.IsAdmin = true - - if err := bus.Dispatch(&cmd); err != nil { - log.Error(3, "Failed to create default admin user", err) - return - } - - log.Info("Created default admin user: %v", setting.AdminUser) -} - -func NewEngine() { - x, err := getEngine() - - if err != nil { - sqlog.Crit("Fail to connect to database", "error", err) - os.Exit(1) - } - - err = SetEngine(x) - - if err != nil { - sqlog.Error("Fail to initialize orm engine", "error", err) - os.Exit(1) - } -} - -func SetEngine(engine *xorm.Engine) (err error) { - x = engine - dialect = migrator.NewDialect(x.DriverName()) - - migrator := migrator.NewMigrator(x) - migrations.AddMigrations(migrator) - - if err := migrator.Start(); err != nil { - return fmt.Errorf("Sqlstore::Migration failed err: %v\n", err) - } - - // Init repo instances - annotations.SetRepository(&SqlAnnotationRepo{}) - return nil -} - -func getEngine() (*xorm.Engine, error) { - LoadConfig() - - cnnstr := "" - switch DbCfg.Type { - case "mysql": - protocol := "tcp" - if strings.HasPrefix(DbCfg.Host, "/") { - protocol = "unix" - } - - cnnstr = fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&allowNativePasswords=true", - DbCfg.User, DbCfg.Pwd, protocol, DbCfg.Host, DbCfg.Name) - - if DbCfg.SslMode == "true" || DbCfg.SslMode == "skip-verify" { - tlsCert, err := makeCert("custom", DbCfg) - if err != nil { - return nil, err - } - mysql.RegisterTLSConfig("custom", tlsCert) - cnnstr += "&tls=custom" - } - case "postgres": - var host, port = "127.0.0.1", "5432" - fields := strings.Split(DbCfg.Host, ":") - if len(fields) > 0 && len(strings.TrimSpace(fields[0])) > 0 { - host = fields[0] - } - if len(fields) > 1 && len(strings.TrimSpace(fields[1])) > 0 { - port = fields[1] - } - if DbCfg.Pwd == "" { - DbCfg.Pwd = "''" - } - if DbCfg.User == "" { - DbCfg.User = "''" - } - cnnstr = fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", DbCfg.User, DbCfg.Pwd, host, port, DbCfg.Name, DbCfg.SslMode, DbCfg.ClientCertPath, DbCfg.ClientKeyPath, DbCfg.CaCertPath) - case "sqlite3": - if !filepath.IsAbs(DbCfg.Path) { - DbCfg.Path = filepath.Join(setting.DataPath, DbCfg.Path) - } - os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm) - cnnstr = "file:" + DbCfg.Path + "?cache=shared&mode=rwc" - default: - return nil, fmt.Errorf("Unknown database type: %s", DbCfg.Type) - } - - sqlog.Info("Initializing DB", "dbtype", DbCfg.Type) - engine, err := xorm.NewEngine(DbCfg.Type, cnnstr) - if err != nil { - return nil, err - } else { - engine.SetMaxOpenConns(DbCfg.MaxOpenConn) - engine.SetMaxIdleConns(DbCfg.MaxIdleConn) - debugSql := setting.Cfg.Section("database").Key("log_queries").MustBool(false) - if !debugSql { - engine.SetLogger(&xorm.DiscardLogger{}) - } else { - engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm"))) - engine.ShowSQL(true) - engine.ShowExecTime(true) - } - } - return engine, nil -} - -func LoadConfig() { - sec := setting.Cfg.Section("database") - - cfgURL := sec.Key("url").String() - if len(cfgURL) != 0 { - dbURL, _ := url.Parse(cfgURL) - DbCfg.Type = dbURL.Scheme - DbCfg.Host = dbURL.Host - - pathSplit := strings.Split(dbURL.Path, "/") - if len(pathSplit) > 1 { - DbCfg.Name = pathSplit[1] - } - - userInfo := dbURL.User - if userInfo != nil { - DbCfg.User = userInfo.Username() - DbCfg.Pwd, _ = userInfo.Password() - } - } else { - DbCfg.Type = sec.Key("type").String() - DbCfg.Host = sec.Key("host").String() - DbCfg.Name = sec.Key("name").String() - DbCfg.User = sec.Key("user").String() - if len(DbCfg.Pwd) == 0 { - DbCfg.Pwd = sec.Key("password").String() - } - } - DbCfg.MaxOpenConn = sec.Key("max_open_conn").MustInt(0) - DbCfg.MaxIdleConn = sec.Key("max_idle_conn").MustInt(0) - - if DbCfg.Type == "sqlite3" { - UseSQLite3 = true - // only allow one connection as sqlite3 has multi threading issues that cause table locks - // DbCfg.MaxIdleConn = 1 - // DbCfg.MaxOpenConn = 1 - } - DbCfg.SslMode = sec.Key("ssl_mode").String() - DbCfg.CaCertPath = sec.Key("ca_cert_path").String() - DbCfg.ClientKeyPath = sec.Key("client_key_path").String() - DbCfg.ClientCertPath = sec.Key("client_cert_path").String() - DbCfg.ServerCertName = sec.Key("server_cert_name").String() - DbCfg.Path = sec.Key("path").MustString("data/grafana.db") -} - -var ( - dbSqlite = "sqlite" - dbMySql = "mysql" - dbPostgres = "postgres" -) - -func InitTestDB(t *testing.T) *xorm.Engine { - selectedDb := dbSqlite - //selectedDb := dbMySql - //selectedDb := dbPostgres - - var x *xorm.Engine - var err error - - // environment variable present for test db? - if db, present := os.LookupEnv("GRAFANA_TEST_DB"); present { - selectedDb = db - } - - switch strings.ToLower(selectedDb) { - case dbMySql: - x, err = xorm.NewEngine(sqlutil.TestDB_Mysql.DriverName, sqlutil.TestDB_Mysql.ConnStr) - case dbPostgres: - x, err = xorm.NewEngine(sqlutil.TestDB_Postgres.DriverName, sqlutil.TestDB_Postgres.ConnStr) - default: - x, err = xorm.NewEngine(sqlutil.TestDB_Sqlite3.DriverName, sqlutil.TestDB_Sqlite3.ConnStr) - } - - // x.ShowSQL() - - if err != nil { - t.Fatalf("Failed to init in memory sqllite3 db %v", err) - } - - sqlutil.CleanDB(x) - - if err := SetEngine(x); err != nil { - t.Fatal(err) - } - - return x + ConnMaxLifetime int } diff --git a/pkg/services/sqlstore/sqlutil/sqlutil.go b/pkg/services/sqlstore/sqlutil/sqlutil.go index 46306ac8e0d..f73985fb1ba 100644 --- a/pkg/services/sqlstore/sqlutil/sqlutil.go +++ b/pkg/services/sqlstore/sqlutil/sqlutil.go @@ -1,11 +1,5 @@ package sqlutil -import ( - "fmt" - - "github.com/go-xorm/xorm" -) - type TestDB struct { DriverName string ConnStr string @@ -15,34 +9,3 @@ var TestDB_Sqlite3 = TestDB{DriverName: "sqlite3", ConnStr: ":memory:"} var TestDB_Mysql = TestDB{DriverName: "mysql", ConnStr: "grafana:password@tcp(localhost:3306)/grafana_tests?collation=utf8mb4_unicode_ci"} var TestDB_Postgres = TestDB{DriverName: "postgres", ConnStr: "user=grafanatest password=grafanatest host=localhost port=5432 dbname=grafanatest sslmode=disable"} var TestDB_Mssql = TestDB{DriverName: "mssql", ConnStr: "server=localhost;port=1433;database=grafanatest;user id=grafana;password=Password!"} - -func CleanDB(x *xorm.Engine) { - if x.DriverName() == "postgres" { - sess := x.NewSession() - defer sess.Close() - - if _, err := sess.Exec("DROP SCHEMA public CASCADE;"); err != nil { - panic("Failed to drop schema public") - } - - if _, err := sess.Exec("CREATE SCHEMA public;"); err != nil { - panic("Failed to create schema public") - } - } else if x.DriverName() == "mysql" { - tables, _ := x.DBMetas() - sess := x.NewSession() - defer sess.Close() - - for _, table := range tables { - if _, err := sess.Exec("set foreign_key_checks = 0"); err != nil { - panic("failed to disable foreign key checks") - } - if _, err := sess.Exec("drop table " + table.Name + " ;"); err != nil { - panic(fmt.Sprintf("failed to delete table: %v, err: %v", table.Name, err)) - } - if _, err := sess.Exec("set foreign_key_checks = 1"); err != nil { - panic("failed to disable foreign key checks") - } - } - } -} diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go index cfe2d88c82c..2cec86e7239 100644 --- a/pkg/services/sqlstore/stats.go +++ b/pkg/services/sqlstore/stats.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "time" "github.com/grafana/grafana/pkg/bus" @@ -10,64 +11,79 @@ import ( func init() { bus.AddHandler("sql", GetSystemStats) bus.AddHandler("sql", GetDataSourceStats) + bus.AddHandler("sql", GetDataSourceAccessStats) bus.AddHandler("sql", GetAdminStats) + bus.AddHandlerCtx("sql", GetAlertNotifiersUsageStats) + bus.AddHandlerCtx("sql", GetSystemUserCountStats) } -var activeUserTimeLimit time.Duration = time.Hour * 24 * 30 +var activeUserTimeLimit = time.Hour * 24 * 30 + +func GetAlertNotifiersUsageStats(ctx context.Context, query *m.GetAlertNotifierUsageStatsQuery) error { + var rawSql = `SELECT COUNT(*) as count, type FROM alert_notification GROUP BY type` + query.Result = make([]*m.NotifierUsageStats, 0) + err := x.SQL(rawSql).Find(&query.Result) + return err +} func GetDataSourceStats(query *m.GetDataSourceStatsQuery) error { var rawSql = `SELECT COUNT(*) as count, type FROM data_source GROUP BY type` query.Result = make([]*m.DataSourceStats, 0) err := x.SQL(rawSql).Find(&query.Result) - if err != nil { - return err - } + return err +} +func GetDataSourceAccessStats(query *m.GetDataSourceAccessStatsQuery) error { + var rawSql = `SELECT COUNT(*) as count, type, access FROM data_source GROUP BY type, access` + query.Result = make([]*m.DataSourceAccessStats, 0) + err := x.SQL(rawSql).Find(&query.Result) return err } func GetSystemStats(query *m.GetSystemStatsQuery) error { - var rawSql = `SELECT - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("user") + ` - ) AS users, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("org") + ` - ) AS orgs, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("dashboard") + ` - ) AS dashboards, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("data_source") + ` - ) AS datasources, - ( - SELECT COUNT(*) FROM ` + dialect.Quote("star") + ` - ) AS stars, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("playlist") + ` - ) AS playlists, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("alert") + ` - ) AS alerts, - ( - SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` where last_seen_at > ? - ) as active_users - ` + sb := &SqlBuilder{} + sb.Write("SELECT ") + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + `) AS users,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("dashboard") + `) AS dashboards,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("alert") + `) AS alerts,`) activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit) + sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` where last_seen_at > ?) AS active_users,`, activeUserDeadlineDate) + + sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` where is_folder = ?) AS folders,`, dialect.BooleanStr(true)) + + sb.Write(`( + SELECT COUNT(acl.id) + FROM `+dialect.Quote("dashboard_acl")+` as acl + inner join `+dialect.Quote("dashboard")+` as d + on d.id = acl.dashboard_id + WHERE d.is_folder = ? + ) AS dashboard_permissions,`, dialect.BooleanStr(false)) + + sb.Write(`( + SELECT COUNT(acl.id) + FROM `+dialect.Quote("dashboard_acl")+` as acl + inner join `+dialect.Quote("dashboard")+` as d + on d.id = acl.dashboard_id + WHERE d.is_folder = ? + ) AS folder_permissions,`, dialect.BooleanStr(true)) + + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams`) + var stats m.SystemStats - _, err := x.SQL(rawSql, activeUserDeadlineDate).Get(&stats) + _, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats) if err != nil { return err } query.Result = &stats + return err } @@ -125,3 +141,19 @@ func GetAdminStats(query *m.GetAdminStatsQuery) error { query.Result = &stats return err } + +func GetSystemUserCountStats(ctx context.Context, query *m.GetSystemUserCountStatsQuery) error { + return withDbSession(ctx, func(sess *DBSession) error { + + var rawSql = `SELECT COUNT(id) AS Count FROM ` + dialect.Quote("user") + var stats m.SystemUserCountStats + _, err := sess.SQL(rawSql).Get(&stats) + if err != nil { + return err + } + + query.Result = &stats + + return err + }) +} diff --git a/pkg/services/sqlstore/stats_test.go b/pkg/services/sqlstore/stats_test.go new file mode 100644 index 00000000000..6949a0dbda2 --- /dev/null +++ b/pkg/services/sqlstore/stats_test.go @@ -0,0 +1,46 @@ +package sqlstore + +import ( + "context" + "testing" + + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestStatsDataAccess(t *testing.T) { + + Convey("Testing Stats Data Access", t, func() { + InitTestDB(t) + + Convey("Get system stats should not results in error", func() { + query := m.GetSystemStatsQuery{} + err := GetSystemStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get system user count stats should not results in error", func() { + query := m.GetSystemUserCountStatsQuery{} + err := GetSystemUserCountStats(context.Background(), &query) + So(err, ShouldBeNil) + }) + + Convey("Get datasource stats should not results in error", func() { + query := m.GetDataSourceStatsQuery{} + err := GetDataSourceStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get datasource access stats should not results in error", func() { + query := m.GetDataSourceAccessStatsQuery{} + err := GetDataSourceAccessStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get alert notifier stats should not results in error", func() { + query := m.GetAlertNotifierUsageStatsQuery{} + err := GetAlertNotifiersUsageStats(context.Background(), &query) + So(err, ShouldBeNil) + }) + }) +} diff --git a/pkg/services/sqlstore/team.go b/pkg/services/sqlstore/team.go index d238301c7ce..a3010a086e5 100644 --- a/pkg/services/sqlstore/team.go +++ b/pkg/services/sqlstore/team.go @@ -22,6 +22,16 @@ func init() { bus.AddHandler("sql", GetTeamMembers) } +func getTeamSelectSqlBase() string { + return `SELECT + team.id as id, + team.org_id, + team.name as name, + team.email as email, + (SELECT COUNT(*) from team_member where team_member.team_id = team.id) as member_count + FROM team as team ` +} + func CreateTeam(cmd *m.CreateTeamCommand) error { return inTransaction(func(sess *DBSession) error { @@ -64,7 +74,7 @@ func UpdateTeam(cmd *m.UpdateTeamCommand) error { sess.MustCols("email") - affectedRows, err := sess.Id(cmd.Id).Update(&team) + affectedRows, err := sess.ID(cmd.Id).Update(&team) if err != nil { return err @@ -130,21 +140,15 @@ func isTeamNameTaken(orgId int64, name string, existingId int64, sess *DBSession func SearchTeams(query *m.SearchTeamsQuery) error { query.Result = m.SearchTeamQueryResult{ - Teams: make([]*m.SearchTeamDto, 0), + Teams: make([]*m.TeamDTO, 0), } queryWithWildcards := "%" + query.Query + "%" var sql bytes.Buffer params := make([]interface{}, 0) - sql.WriteString(`select - team.id as id, - team.org_id, - team.name as name, - team.email as email, - (select count(*) from team_member where team_member.team_id = team.id) as member_count - from team as team - where team.org_id = ?`) + sql.WriteString(getTeamSelectSqlBase()) + sql.WriteString(` WHERE team.org_id = ?`) params = append(params, query.OrgId) @@ -161,12 +165,11 @@ func SearchTeams(query *m.SearchTeamsQuery) error { sql.WriteString(` order by team.name asc`) if query.Limit != 0 { - sql.WriteString(` limit ? offset ?`) offset := query.Limit * (query.Page - 1) - params = append(params, query.Limit, offset) + sql.WriteString(dialect.LimitOffset(int64(query.Limit), int64(offset))) } - if err := x.Sql(sql.String(), params...).Find(&query.Result.Teams); err != nil { + if err := x.SQL(sql.String(), params...).Find(&query.Result.Teams); err != nil { return err } @@ -187,8 +190,14 @@ func SearchTeams(query *m.SearchTeamsQuery) error { } func GetTeamById(query *m.GetTeamByIdQuery) error { - var team m.Team - exists, err := x.Where("org_id=? and id=?", query.OrgId, query.Id).Get(&team) + var sql bytes.Buffer + + sql.WriteString(getTeamSelectSqlBase()) + sql.WriteString(` WHERE team.org_id = ? and team.id = ?`) + + var team m.TeamDTO + exists, err := x.SQL(sql.String(), query.OrgId, query.Id).Get(&team) + if err != nil { return err } @@ -203,18 +212,16 @@ func GetTeamById(query *m.GetTeamByIdQuery) error { // GetTeamsByUser is used by the Guardian when checking a users' permissions func GetTeamsByUser(query *m.GetTeamsByUserQuery) error { - query.Result = make([]*m.Team, 0) + query.Result = make([]*m.TeamDTO, 0) - sess := x.Table("team") - sess.Join("INNER", "team_member", "team.id=team_member.team_id") - sess.Where("team.org_id=? and team_member.user_id=?", query.OrgId, query.UserId) + var sql bytes.Buffer - err := sess.Find(&query.Result) - if err != nil { - return err - } + sql.WriteString(getTeamSelectSqlBase()) + sql.WriteString(` INNER JOIN team_member on team.id = team_member.team_id`) + sql.WriteString(` WHERE team.org_id = ? and team_member.user_id = ?`) - return nil + err := x.SQL(sql.String(), query.OrgId, query.UserId).Find(&query.Result) + return err } // AddTeamMember adds a user to a team @@ -233,11 +240,12 @@ func AddTeamMember(cmd *m.AddTeamMemberCommand) error { } entity := m.TeamMember{ - OrgId: cmd.OrgId, - TeamId: cmd.TeamId, - UserId: cmd.UserId, - Created: time.Now(), - Updated: time.Now(), + OrgId: cmd.OrgId, + TeamId: cmd.TeamId, + UserId: cmd.UserId, + External: cmd.External, + Created: time.Now(), + Updated: time.Now(), } _, err := sess.Insert(&entity) @@ -273,8 +281,19 @@ func GetTeamMembers(query *m.GetTeamMembersQuery) error { query.Result = make([]*m.TeamMemberDTO, 0) sess := x.Table("team_member") sess.Join("INNER", "user", fmt.Sprintf("team_member.user_id=%s.id", x.Dialect().Quote("user"))) - sess.Where("team_member.org_id=? and team_member.team_id=?", query.OrgId, query.TeamId) - sess.Cols("user.org_id", "team_member.team_id", "team_member.user_id", "user.email", "user.login") + if query.OrgId != 0 { + sess.Where("team_member.org_id=?", query.OrgId) + } + if query.TeamId != 0 { + sess.Where("team_member.team_id=?", query.TeamId) + } + if query.UserId != 0 { + sess.Where("team_member.user_id=?", query.UserId) + } + if query.External { + sess.Where("team_member.external=?", dialect.BooleanStr(true)) + } + sess.Cols("team_member.org_id", "team_member.team_id", "team_member.user_id", "user.email", "user.login", "team_member.external") sess.Asc("user.login", "user.email") err := sess.Find(&query.Result) diff --git a/pkg/services/sqlstore/team_test.go b/pkg/services/sqlstore/team_test.go index f136411eeba..8f243617262 100644 --- a/pkg/services/sqlstore/team_test.go +++ b/pkg/services/sqlstore/team_test.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "fmt" "testing" @@ -22,7 +23,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { Name: fmt.Sprint("user", i), Login: fmt.Sprint("loginuser", i), } - err := CreateUser(userCmd) + err := CreateUser(context.Background(), userCmd) So(err, ShouldBeNil) userIds = append(userIds, userCmd.Result.Id) } @@ -49,13 +50,29 @@ func TestTeamCommandsAndQueries(t *testing.T) { err = AddTeamMember(&m.AddTeamMemberCommand{OrgId: testOrgId, TeamId: team1.Id, UserId: userIds[0]}) So(err, ShouldBeNil) + err = AddTeamMember(&m.AddTeamMemberCommand{OrgId: testOrgId, TeamId: team1.Id, UserId: userIds[1], External: true}) + So(err, ShouldBeNil) q1 := &m.GetTeamMembersQuery{OrgId: testOrgId, TeamId: team1.Id} err = GetTeamMembers(q1) So(err, ShouldBeNil) + So(q1.Result, ShouldHaveLength, 2) So(q1.Result[0].TeamId, ShouldEqual, team1.Id) So(q1.Result[0].Login, ShouldEqual, "loginuser0") So(q1.Result[0].OrgId, ShouldEqual, testOrgId) + So(q1.Result[1].TeamId, ShouldEqual, team1.Id) + So(q1.Result[1].Login, ShouldEqual, "loginuser1") + So(q1.Result[1].OrgId, ShouldEqual, testOrgId) + So(q1.Result[1].External, ShouldEqual, true) + + q2 := &m.GetTeamMembersQuery{OrgId: testOrgId, TeamId: team1.Id, External: true} + err = GetTeamMembers(q2) + So(err, ShouldBeNil) + So(q2.Result, ShouldHaveLength, 1) + So(q2.Result[0].TeamId, ShouldEqual, team1.Id) + So(q2.Result[0].Login, ShouldEqual, "loginuser1") + So(q2.Result[0].OrgId, ShouldEqual, testOrgId) + So(q2.Result[0].External, ShouldEqual, true) }) Convey("Should be able to search for teams", func() { @@ -74,6 +91,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { Convey("Should be able to return all teams a user is member of", func() { groupId := group2.Result.Id err := AddTeamMember(&m.AddTeamMemberCommand{OrgId: testOrgId, TeamId: groupId, UserId: userIds[0]}) + So(err, ShouldBeNil) query := &m.GetTeamsByUserQuery{OrgId: testOrgId, UserId: userIds[0]} err = GetTeamsByUser(query) @@ -103,7 +121,7 @@ func TestTeamCommandsAndQueries(t *testing.T) { err = AddTeamMember(&m.AddTeamMemberCommand{OrgId: testOrgId, TeamId: groupId, UserId: userIds[2]}) So(err, ShouldBeNil) err = testHelperUpdateDashboardAcl(1, m.DashboardAcl{DashboardId: 1, OrgId: testOrgId, Permission: m.PERMISSION_EDIT, TeamId: groupId}) - + So(err, ShouldBeNil) err = DeleteTeam(&m.DeleteTeamCommand{OrgId: testOrgId, Id: groupId}) So(err, ShouldBeNil) diff --git a/pkg/services/sqlstore/temp_user.go b/pkg/services/sqlstore/temp_user.go index 43e1f027057..f13752f8038 100644 --- a/pkg/services/sqlstore/temp_user.go +++ b/pkg/services/sqlstore/temp_user.go @@ -96,7 +96,7 @@ func GetTempUsersQuery(query *m.GetTempUsersQuery) error { rawSql += " ORDER BY tu.created desc" query.Result = make([]*m.TempUserDTO, 0) - sess := x.Sql(rawSql, params...) + sess := x.SQL(rawSql, params...) err := sess.Find(&query.Result) return err } @@ -121,12 +121,12 @@ func GetTempUserByCode(query *m.GetTempUserByCodeQuery) error { WHERE tu.code=?` var tempUser m.TempUserDTO - sess := x.Sql(rawSql, query.Code) + sess := x.SQL(rawSql, query.Code) has, err := sess.Get(&tempUser) if err != nil { return err - } else if has == false { + } else if !has { return m.ErrTempUserNotFound } diff --git a/pkg/services/sqlstore/transactions.go b/pkg/services/sqlstore/transactions.go new file mode 100644 index 00000000000..edf29fffb8f --- /dev/null +++ b/pkg/services/sqlstore/transactions.go @@ -0,0 +1,106 @@ +package sqlstore + +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" + sqlite3 "github.com/mattn/go-sqlite3" +) + +func (ss *SqlStore) InTransaction(ctx context.Context, fn func(ctx context.Context) error) error { + return ss.inTransactionWithRetry(ctx, fn, 0) +} + +func (ss *SqlStore) inTransactionWithRetry(ctx context.Context, fn func(ctx context.Context) error, retry int) error { + sess, err := startSession(ctx, ss.engine, true) + if err != nil { + return err + } + + defer sess.Close() + + withValue := context.WithValue(ctx, ContextSessionName, sess) + + err = fn(withValue) + + // special handling of database locked errors for sqlite, then we can retry 3 times + if sqlError, ok := err.(sqlite3.Error); ok && retry < 5 { + if sqlError.Code == sqlite3.ErrLocked { + sess.Rollback() + time.Sleep(time.Millisecond * time.Duration(10)) + ss.log.Info("Database table locked, sleeping then retrying", "retry", retry) + return ss.inTransactionWithRetry(ctx, fn, retry+1) + } + } + + if err != nil { + sess.Rollback() + return err + } + + if err = sess.Commit(); err != nil { + return err + } + + if len(sess.events) > 0 { + for _, e := range sess.events { + if err = bus.Publish(e); err != nil { + ss.log.Error("Failed to publish event after commit", err) + } + } + } + + return nil +} + +func inTransactionWithRetry(callback dbTransactionFunc, retry int) error { + return inTransactionWithRetryCtx(context.Background(), callback, retry) +} + +func inTransactionWithRetryCtx(ctx context.Context, callback dbTransactionFunc, retry int) error { + sess, err := startSession(ctx, x, true) + if err != nil { + return err + } + + defer sess.Close() + + err = callback(sess) + + // special handling of database locked errors for sqlite, then we can retry 3 times + if sqlError, ok := err.(sqlite3.Error); ok && retry < 5 { + if sqlError.Code == sqlite3.ErrLocked { + sess.Rollback() + time.Sleep(time.Millisecond * time.Duration(10)) + sqlog.Info("Database table locked, sleeping then retrying", "retry", retry) + return inTransactionWithRetry(callback, retry+1) + } + } + + if err != nil { + sess.Rollback() + return err + } else if err = sess.Commit(); err != nil { + return err + } + + if len(sess.events) > 0 { + for _, e := range sess.events { + if err = bus.Publish(e); err != nil { + log.Error(3, "Failed to publish event after commit. error: %v", err) + } + } + } + + return nil +} + +func inTransaction(callback dbTransactionFunc) error { + return inTransactionWithRetry(callback, 0) +} + +func inTransactionCtx(ctx context.Context, callback dbTransactionFunc) error { + return inTransactionWithRetryCtx(ctx, callback, 0) +} diff --git a/pkg/services/sqlstore/transactions_test.go b/pkg/services/sqlstore/transactions_test.go new file mode 100644 index 00000000000..041359cf1d3 --- /dev/null +++ b/pkg/services/sqlstore/transactions_test.go @@ -0,0 +1,56 @@ +package sqlstore + +import ( + "context" + "errors" + "testing" + + "github.com/grafana/grafana/pkg/models" + + . "github.com/smartystreets/goconvey/convey" +) + +var ProvokedError = errors.New("testing error.") + +func TestTransaction(t *testing.T) { + ss := InitTestDB(t) + + Convey("InTransaction asdf asdf", t, func() { + cmd := &models.AddApiKeyCommand{Key: "secret-key", Name: "key", OrgId: 1} + + err := AddApiKey(cmd) + So(err, ShouldBeNil) + + deleteApiKeyCmd := &models.DeleteApiKeyCommand{Id: cmd.Result.Id, OrgId: 1} + + Convey("can update key", func() { + err := ss.InTransaction(context.Background(), func(ctx context.Context) error { + return DeleteApiKeyCtx(ctx, deleteApiKeyCmd) + }) + + So(err, ShouldBeNil) + + query := &models.GetApiKeyByIdQuery{ApiKeyId: cmd.Result.Id} + err = GetApiKeyById(query) + So(err, ShouldEqual, models.ErrInvalidApiKey) + }) + + Convey("won't update if one handler fails", func() { + err := ss.InTransaction(context.Background(), func(ctx context.Context) error { + err := DeleteApiKeyCtx(ctx, deleteApiKeyCmd) + if err != nil { + return err + } + + return ProvokedError + }) + + So(err, ShouldEqual, ProvokedError) + + query := &models.GetApiKeyByIdQuery{ApiKeyId: cmd.Result.Id} + err = GetApiKeyById(query) + So(err, ShouldBeNil) + So(query.Result.Id, ShouldEqual, cmd.Result.Id) + }) + }) +} diff --git a/pkg/services/sqlstore/user.go b/pkg/services/sqlstore/user.go index 73ea07f031f..99a77ecabc3 100644 --- a/pkg/services/sqlstore/user.go +++ b/pkg/services/sqlstore/user.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "strconv" "strings" "time" @@ -14,8 +15,9 @@ import ( "github.com/grafana/grafana/pkg/util" ) -func init() { - bus.AddHandler("sql", CreateUser) +func (ss *SqlStore) addUserQueryAndCommandHandlers() { + ss.Bus.AddHandler(ss.GetSignedInUserWithCache) + bus.AddHandler("sql", GetUserById) bus.AddHandler("sql", UpdateUser) bus.AddHandler("sql", ChangeUserPassword) @@ -24,12 +26,12 @@ func init() { bus.AddHandler("sql", SetUsingOrg) bus.AddHandler("sql", UpdateUserLastSeenAt) bus.AddHandler("sql", GetUserProfile) - bus.AddHandler("sql", GetSignedInUser) bus.AddHandler("sql", SearchUsers) bus.AddHandler("sql", GetUserOrgList) bus.AddHandler("sql", DeleteUser) bus.AddHandler("sql", UpdateUserPermissions) bus.AddHandler("sql", SetUserHelpFlag) + bus.AddHandlerCtx("sql", CreateUser) } func getOrgIdForNewUser(cmd *m.CreateUserCommand, sess *DBSession) (int64, error) { @@ -40,16 +42,22 @@ func getOrgIdForNewUser(cmd *m.CreateUserCommand, sess *DBSession) (int64, error var org m.Org if setting.AutoAssignOrg { - // right now auto assign to org with id 1 - has, err := sess.Where("id=?", 1).Get(&org) + has, err := sess.Where("id=?", setting.AutoAssignOrgId).Get(&org) if err != nil { return 0, err } if has { return org.Id, nil } else { - org.Name = "Main Org." - org.Id = 1 + if setting.AutoAssignOrgId == 1 { + org.Name = "Main Org." + org.Id = int64(setting.AutoAssignOrgId) + } else { + sqlog.Info("Could not create user: organization id %v does not exist", + setting.AutoAssignOrgId) + return 0, fmt.Errorf("Could not create user: organization id %v does not exist", + setting.AutoAssignOrgId) + } } } else { org.Name = cmd.OrgName @@ -61,8 +69,14 @@ func getOrgIdForNewUser(cmd *m.CreateUserCommand, sess *DBSession) (int64, error org.Created = time.Now() org.Updated = time.Now() - if _, err := sess.Insert(&org); err != nil { - return 0, err + if org.Id != 0 { + if _, err := sess.InsertId(&org); err != nil { + return 0, err + } + } else { + if _, err := sess.InsertOne(&org); err != nil { + return 0, err + } } sess.publishAfterCommit(&events.OrgCreated{ @@ -74,8 +88,8 @@ func getOrgIdForNewUser(cmd *m.CreateUserCommand, sess *DBSession) (int64, error return org.Id, nil } -func CreateUser(cmd *m.CreateUserCommand) error { - return inTransaction(func(sess *DBSession) error { +func CreateUser(ctx context.Context, cmd *m.CreateUserCommand) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { orgId, err := getOrgIdForNewUser(cmd, sess) if err != nil { return err @@ -99,9 +113,10 @@ func CreateUser(cmd *m.CreateUserCommand) error { LastSeenAt: time.Now().AddDate(-10, 0, 0), } + user.Salt = util.GetRandomString(10) + user.Rands = util.GetRandomString(10) + if len(cmd.Password) > 0 { - user.Salt = util.GetRandomString(10) - user.Rands = util.GetRandomString(10) user.Password = util.EncodePassword(cmd.Password, user.Salt) } @@ -154,7 +169,7 @@ func GetUserById(query *m.GetUserByIdQuery) error { if err != nil { return err - } else if has == false { + } else if !has { return m.ErrUserNotFound } @@ -168,18 +183,16 @@ func GetUserByLogin(query *m.GetUserByLoginQuery) error { return m.ErrUserNotFound } - user := new(m.User) - // Try and find the user by login first. // It's not sufficient to assume that a LoginOrEmail with an "@" is an email. - user = &m.User{Login: query.LoginOrEmail} + user := &m.User{Login: query.LoginOrEmail} has, err := x.Get(user) if err != nil { return err } - if has == false && strings.Contains(query.LoginOrEmail, "@") { + if !has && strings.Contains(query.LoginOrEmail, "@") { // If the user wasn't found, and it contains an "@" fallback to finding the // user by email. user = &m.User{Email: query.LoginOrEmail} @@ -188,7 +201,7 @@ func GetUserByLogin(query *m.GetUserByLoginQuery) error { if err != nil { return err - } else if has == false { + } else if !has { return m.ErrUserNotFound } @@ -202,14 +215,12 @@ func GetUserByEmail(query *m.GetUserByEmailQuery) error { return m.ErrUserNotFound } - user := new(m.User) - - user = &m.User{Email: query.Email} + user := &m.User{Email: query.Email} has, err := x.Get(user) if err != nil { return err - } else if has == false { + } else if !has { return m.ErrUserNotFound } @@ -229,7 +240,7 @@ func UpdateUser(cmd *m.UpdateUserCommand) error { Updated: time.Now(), } - if _, err := sess.Id(cmd.UserId).Update(&user); err != nil { + if _, err := sess.ID(cmd.UserId).Update(&user); err != nil { return err } @@ -253,29 +264,20 @@ func ChangeUserPassword(cmd *m.ChangeUserPasswordCommand) error { Updated: time.Now(), } - if _, err := sess.Id(cmd.UserId).Update(&user); err != nil { - return err - } - - return nil + _, err := sess.ID(cmd.UserId).Update(&user) + return err }) } func UpdateUserLastSeenAt(cmd *m.UpdateUserLastSeenAtCommand) error { return inTransaction(func(sess *DBSession) error { - if cmd.UserId <= 0 { - } - user := m.User{ Id: cmd.UserId, LastSeenAt: time.Now(), } - if _, err := sess.Id(cmd.UserId).Update(&user); err != nil { - return err - } - - return nil + _, err := sess.ID(cmd.UserId).Update(&user) + return err }) } @@ -295,26 +297,32 @@ func SetUsingOrg(cmd *m.SetUsingOrgCommand) error { } return inTransaction(func(sess *DBSession) error { - user := m.User{} - sess.Id(cmd.UserId).Get(&user) - - user.OrgId = cmd.OrgId - _, err := sess.Id(user.Id).Update(&user) - return err + return setUsingOrgInTransaction(sess, cmd.UserId, cmd.OrgId) }) } +func setUsingOrgInTransaction(sess *DBSession, userID int64, orgID int64) error { + user := m.User{ + Id: userID, + OrgId: orgID, + } + + _, err := sess.ID(userID).Update(&user) + return err +} + func GetUserProfile(query *m.GetUserProfileQuery) error { var user m.User has, err := x.Id(query.UserId).Get(&user) if err != nil { return err - } else if has == false { + } else if !has { return m.ErrUserNotFound } query.Result = m.UserProfileDTO{ + Id: user.Id, Name: user.Name, Email: user.Email, Login: user.Login, @@ -332,10 +340,27 @@ func GetUserOrgList(query *m.GetUserOrgListQuery) error { sess.Join("INNER", "org", "org_user.org_id=org.id") sess.Where("org_user.user_id=?", query.UserId) sess.Cols("org.name", "org_user.role", "org_user.org_id") + sess.OrderBy("org.name") err := sess.Find(&query.Result) return err } +func (ss *SqlStore) GetSignedInUserWithCache(query *m.GetSignedInUserQuery) error { + cacheKey := fmt.Sprintf("signed-in-user-%d-%d", query.UserId, query.OrgId) + if cached, found := ss.CacheService.Get(cacheKey); found { + query.Result = cached.(*m.SignedInUser) + return nil + } + + err := GetSignedInUser(query) + if err != nil { + return err + } + + ss.CacheService.Set(cacheKey, query.Result, time.Second*5) + return nil +} + func GetSignedInUser(query *m.GetSignedInUserQuery) error { orgId := "u.org_id" if query.OrgId > 0 { @@ -360,11 +385,11 @@ func GetSignedInUser(query *m.GetSignedInUserQuery) error { sess := x.Table("user") if query.UserId > 0 { - sess.Sql(rawSql+"WHERE u.id=?", query.UserId) + sess.SQL(rawSql+"WHERE u.id=?", query.UserId) } else if query.Login != "" { - sess.Sql(rawSql+"WHERE u.login=?", query.Login) + sess.SQL(rawSql+"WHERE u.login=?", query.Login) } else if query.Email != "" { - sess.Sql(rawSql+"WHERE u.email=?", query.Email) + sess.SQL(rawSql+"WHERE u.email=?", query.Email) } var user m.SignedInUser @@ -380,6 +405,17 @@ func GetSignedInUser(query *m.GetSignedInUserQuery) error { user.OrgName = "Org missing" } + getTeamsByUserQuery := &m.GetTeamsByUserQuery{OrgId: user.OrgId, UserId: user.UserId} + err = GetTeamsByUser(getTeamsByUserQuery) + if err != nil { + return err + } + + user.Teams = make([]int64, len(getTeamsByUserQuery.Result)) + for i, t := range getTeamsByUserQuery.Result { + user.Teams[i] = t.Id + } + query.Result = &user return err } @@ -436,34 +472,39 @@ func SearchUsers(query *m.SearchUsersQuery) error { func DeleteUser(cmd *m.DeleteUserCommand) error { return inTransaction(func(sess *DBSession) error { - deletes := []string{ - "DELETE FROM star WHERE user_id = ?", - "DELETE FROM " + dialect.Quote("user") + " WHERE id = ?", - "DELETE FROM org_user WHERE user_id = ?", - "DELETE FROM dashboard_acl WHERE user_id = ?", - "DELETE FROM preferences WHERE user_id = ?", - "DELETE FROM team_member WHERE user_id = ?", - } - - for _, sql := range deletes { - _, err := sess.Exec(sql, cmd.UserId) - if err != nil { - return err - } - } - - return nil + return deleteUserInTransaction(sess, cmd) }) } +func deleteUserInTransaction(sess *DBSession, cmd *m.DeleteUserCommand) error { + deletes := []string{ + "DELETE FROM star WHERE user_id = ?", + "DELETE FROM " + dialect.Quote("user") + " WHERE id = ?", + "DELETE FROM org_user WHERE user_id = ?", + "DELETE FROM dashboard_acl WHERE user_id = ?", + "DELETE FROM preferences WHERE user_id = ?", + "DELETE FROM team_member WHERE user_id = ?", + "DELETE FROM user_auth WHERE user_id = ?", + } + + for _, sql := range deletes { + _, err := sess.Exec(sql, cmd.UserId) + if err != nil { + return err + } + } + + return nil +} + func UpdateUserPermissions(cmd *m.UpdateUserPermissionsCommand) error { return inTransaction(func(sess *DBSession) error { user := m.User{} - sess.Id(cmd.UserId).Get(&user) + sess.ID(cmd.UserId).Get(&user) user.IsAdmin = cmd.IsGrafanaAdmin sess.UseBool("is_admin") - _, err := sess.Id(user.Id).Update(&user) + _, err := sess.ID(user.Id).Update(&user) return err }) } @@ -477,10 +518,7 @@ func SetUserHelpFlag(cmd *m.SetUserHelpFlagCommand) error { Updated: time.Now(), } - if _, err := sess.Id(cmd.UserId).Cols("help_flags1").Update(&user); err != nil { - return err - } - - return nil + _, err := sess.ID(cmd.UserId).Cols("help_flags1").Update(&user) + return err }) } diff --git a/pkg/services/sqlstore/user_auth.go b/pkg/services/sqlstore/user_auth.go new file mode 100644 index 00000000000..aec828451a4 --- /dev/null +++ b/pkg/services/sqlstore/user_auth.go @@ -0,0 +1,148 @@ +package sqlstore + +import ( + "time" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" +) + +func init() { + bus.AddHandler("sql", GetUserByAuthInfo) + bus.AddHandler("sql", GetAuthInfo) + bus.AddHandler("sql", SetAuthInfo) + bus.AddHandler("sql", DeleteAuthInfo) +} + +func GetUserByAuthInfo(query *m.GetUserByAuthInfoQuery) error { + user := &m.User{} + has := false + var err error + authQuery := &m.GetAuthInfoQuery{} + + // Try to find the user by auth module and id first + if query.AuthModule != "" && query.AuthId != "" { + authQuery.AuthModule = query.AuthModule + authQuery.AuthId = query.AuthId + + err = GetAuthInfo(authQuery) + if err != m.ErrUserNotFound { + if err != nil { + return err + } + + // if user id was specified and doesn't match the user_auth entry, remove it + if query.UserId != 0 && query.UserId != authQuery.Result.UserId { + err = DeleteAuthInfo(&m.DeleteAuthInfoCommand{ + UserAuth: authQuery.Result, + }) + if err != nil { + sqlog.Error("Error removing user_auth entry", "error", err) + } + + authQuery.Result = nil + } else { + has, err = x.Id(authQuery.Result.UserId).Get(user) + if err != nil { + return err + } + + if !has { + // if the user has been deleted then remove the entry + err = DeleteAuthInfo(&m.DeleteAuthInfoCommand{ + UserAuth: authQuery.Result, + }) + if err != nil { + sqlog.Error("Error removing user_auth entry", "error", err) + } + + authQuery.Result = nil + } + } + } + } + + // If not found, try to find the user by id + if !has && query.UserId != 0 { + has, err = x.Id(query.UserId).Get(user) + if err != nil { + return err + } + } + + // If not found, try to find the user by email address + if !has && query.Email != "" { + user = &m.User{Email: query.Email} + has, err = x.Get(user) + if err != nil { + return err + } + } + + // If not found, try to find the user by login + if !has && query.Login != "" { + user = &m.User{Login: query.Login} + has, err = x.Get(user) + if err != nil { + return err + } + } + + // No user found + if !has { + return m.ErrUserNotFound + } + + // create authInfo record to link accounts + if authQuery.Result == nil && query.AuthModule != "" && query.AuthId != "" { + cmd2 := &m.SetAuthInfoCommand{ + UserId: user.Id, + AuthModule: query.AuthModule, + AuthId: query.AuthId, + } + if err := SetAuthInfo(cmd2); err != nil { + return err + } + } + + query.Result = user + return nil +} + +func GetAuthInfo(query *m.GetAuthInfoQuery) error { + userAuth := &m.UserAuth{ + AuthModule: query.AuthModule, + AuthId: query.AuthId, + } + has, err := x.Get(userAuth) + if err != nil { + return err + } + if !has { + return m.ErrUserNotFound + } + + query.Result = userAuth + return nil +} + +func SetAuthInfo(cmd *m.SetAuthInfoCommand) error { + return inTransaction(func(sess *DBSession) error { + authUser := &m.UserAuth{ + UserId: cmd.UserId, + AuthModule: cmd.AuthModule, + AuthId: cmd.AuthId, + Created: time.Now(), + } + + _, err := sess.Insert(authUser) + return err + }) +} + +func DeleteAuthInfo(cmd *m.DeleteAuthInfoCommand) error { + return inTransaction(func(sess *DBSession) error { + _, err := sess.Delete(cmd.UserAuth) + return err + }) +} diff --git a/pkg/services/sqlstore/user_auth_test.go b/pkg/services/sqlstore/user_auth_test.go new file mode 100644 index 00000000000..a0dd714fe6f --- /dev/null +++ b/pkg/services/sqlstore/user_auth_test.go @@ -0,0 +1,130 @@ +package sqlstore + +import ( + "context" + "fmt" + "testing" + + . "github.com/smartystreets/goconvey/convey" + + m "github.com/grafana/grafana/pkg/models" +) + +func TestUserAuth(t *testing.T) { + InitTestDB(t) + + Convey("Given 5 users", t, func() { + var err error + var cmd *m.CreateUserCommand + for i := 0; i < 5; i++ { + cmd = &m.CreateUserCommand{ + Email: fmt.Sprint("user", i, "@test.com"), + Name: fmt.Sprint("user", i), + Login: fmt.Sprint("loginuser", i), + } + err = CreateUser(context.Background(), cmd) + So(err, ShouldBeNil) + } + + Reset(func() { + _, err := x.Exec("DELETE FROM org_user WHERE 1=1") + So(err, ShouldBeNil) + _, err = x.Exec("DELETE FROM org WHERE 1=1") + So(err, ShouldBeNil) + _, err = x.Exec("DELETE FROM " + dialect.Quote("user") + " WHERE 1=1") + So(err, ShouldBeNil) + _, err = x.Exec("DELETE FROM user_auth WHERE 1=1") + So(err, ShouldBeNil) + }) + + Convey("Can find existing user", func() { + // By Login + login := "loginuser0" + + query := &m.GetUserByAuthInfoQuery{Login: login} + err = GetUserByAuthInfo(query) + + So(err, ShouldBeNil) + So(query.Result.Login, ShouldEqual, login) + + // By ID + id := query.Result.Id + + query = &m.GetUserByAuthInfoQuery{UserId: id} + err = GetUserByAuthInfo(query) + + So(err, ShouldBeNil) + So(query.Result.Id, ShouldEqual, id) + + // By Email + email := "user1@test.com" + + query = &m.GetUserByAuthInfoQuery{Email: email} + err = GetUserByAuthInfo(query) + + So(err, ShouldBeNil) + So(query.Result.Email, ShouldEqual, email) + + // Don't find nonexistent user + email = "nonexistent@test.com" + + query = &m.GetUserByAuthInfoQuery{Email: email} + err = GetUserByAuthInfo(query) + + So(err, ShouldEqual, m.ErrUserNotFound) + So(query.Result, ShouldBeNil) + }) + + Convey("Can set & locate by AuthModule and AuthId", func() { + // get nonexistent user_auth entry + query := &m.GetUserByAuthInfoQuery{AuthModule: "test", AuthId: "test"} + err = GetUserByAuthInfo(query) + + So(err, ShouldEqual, m.ErrUserNotFound) + So(query.Result, ShouldBeNil) + + // create user_auth entry + login := "loginuser0" + + query.Login = login + err = GetUserByAuthInfo(query) + + So(err, ShouldBeNil) + So(query.Result.Login, ShouldEqual, login) + + // get via user_auth + query = &m.GetUserByAuthInfoQuery{AuthModule: "test", AuthId: "test"} + err = GetUserByAuthInfo(query) + + So(err, ShouldBeNil) + So(query.Result.Login, ShouldEqual, login) + + // get with non-matching id + id := query.Result.Id + + query.UserId = id + 1 + err = GetUserByAuthInfo(query) + + So(err, ShouldBeNil) + So(query.Result.Login, ShouldEqual, "loginuser1") + + // get via user_auth + query = &m.GetUserByAuthInfoQuery{AuthModule: "test", AuthId: "test"} + err = GetUserByAuthInfo(query) + + So(err, ShouldBeNil) + So(query.Result.Login, ShouldEqual, "loginuser1") + + // remove user + _, err = x.Exec("DELETE FROM "+dialect.Quote("user")+" WHERE id=?", query.Result.Id) + So(err, ShouldBeNil) + + // get via user_auth for deleted user + query = &m.GetUserByAuthInfoQuery{AuthModule: "test", AuthId: "test"} + err = GetUserByAuthInfo(query) + + So(err, ShouldEqual, m.ErrUserNotFound) + So(query.Result, ShouldBeNil) + }) + }) +} diff --git a/pkg/services/sqlstore/user_test.go b/pkg/services/sqlstore/user_test.go index 2830733c96a..b26dd235772 100644 --- a/pkg/services/sqlstore/user_test.go +++ b/pkg/services/sqlstore/user_test.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "fmt" "testing" @@ -14,6 +15,28 @@ func TestUserDataAccess(t *testing.T) { Convey("Testing DB", t, func() { InitTestDB(t) + Convey("Creating a user", func() { + cmd := &m.CreateUserCommand{ + Email: "usertest@test.com", + Name: "user name", + Login: "user_test_login", + } + + err := CreateUser(context.Background(), cmd) + So(err, ShouldBeNil) + + Convey("Loading a user", func() { + query := m.GetUserByIdQuery{Id: cmd.Result.Id} + err := GetUserById(&query) + So(err, ShouldBeNil) + + So(query.Result.Email, ShouldEqual, "usertest@test.com") + So(query.Result.Password, ShouldEqual, "") + So(query.Result.Rands, ShouldHaveLength, 10) + So(query.Result.Salt, ShouldHaveLength, 10) + }) + }) + Convey("Given 5 users", func() { var err error var cmd *m.CreateUserCommand @@ -24,7 +47,7 @@ func TestUserDataAccess(t *testing.T) { Name: fmt.Sprint("user", i), Login: fmt.Sprint("loginuser", i), } - err = CreateUser(cmd) + err = CreateUser(context.Background(), cmd) So(err, ShouldBeNil) users = append(users, cmd.Result) } @@ -96,33 +119,33 @@ func TestUserDataAccess(t *testing.T) { }) Convey("when a user is an org member and has been assigned permissions", func() { - err = AddOrgUser(&m.AddOrgUserCommand{LoginOrEmail: users[0].Login, Role: m.ROLE_VIEWER, OrgId: users[0].OrgId}) + err = AddOrgUser(&m.AddOrgUserCommand{LoginOrEmail: users[1].Login, Role: m.ROLE_VIEWER, OrgId: users[0].OrgId, UserId: users[1].Id}) So(err, ShouldBeNil) - testHelperUpdateDashboardAcl(1, m.DashboardAcl{DashboardId: 1, OrgId: users[0].OrgId, UserId: users[0].Id, Permission: m.PERMISSION_EDIT}) + testHelperUpdateDashboardAcl(1, m.DashboardAcl{DashboardId: 1, OrgId: users[0].OrgId, UserId: users[1].Id, Permission: m.PERMISSION_EDIT}) So(err, ShouldBeNil) - err = SavePreferences(&m.SavePreferencesCommand{UserId: users[0].Id, OrgId: users[0].OrgId, HomeDashboardId: 1, Theme: "dark"}) + err = SavePreferences(&m.SavePreferencesCommand{UserId: users[1].Id, OrgId: users[0].OrgId, HomeDashboardId: 1, Theme: "dark"}) So(err, ShouldBeNil) Convey("when the user is deleted", func() { - err = DeleteUser(&m.DeleteUserCommand{UserId: users[0].Id}) + err = DeleteUser(&m.DeleteUserCommand{UserId: users[1].Id}) So(err, ShouldBeNil) Convey("Should delete connected org users and permissions", func() { - query := &m.GetOrgUsersQuery{OrgId: 1} + query := &m.GetOrgUsersQuery{OrgId: users[0].OrgId} err = GetOrgUsersForTest(query) So(err, ShouldBeNil) So(len(query.Result), ShouldEqual, 1) - permQuery := &m.GetDashboardAclInfoListQuery{DashboardId: 1, OrgId: 1} + permQuery := &m.GetDashboardAclInfoListQuery{DashboardId: 1, OrgId: users[0].OrgId} err = GetDashboardAclInfoList(permQuery) So(err, ShouldBeNil) So(len(permQuery.Result), ShouldEqual, 0) - prefsQuery := &m.GetPreferencesQuery{OrgId: users[0].OrgId, UserId: users[0].Id} + prefsQuery := &m.GetPreferencesQuery{OrgId: users[0].OrgId, UserId: users[1].Id} err = GetPreferences(prefsQuery) So(err, ShouldBeNil) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 6099388f668..afae642f5b3 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -13,13 +13,12 @@ import ( "regexp" "runtime" "strings" - - "gopkg.in/ini.v1" + "time" "github.com/go-macaron/session" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/util" + "gopkg.in/ini.v1" ) type Scheme string @@ -32,33 +31,38 @@ const ( ) const ( - DEV string = "development" - PROD string = "production" - TEST string = "test" + DEV = "development" + PROD = "production" + TEST = "test" + APP_NAME = "Grafana" + APP_NAME_ENTERPRISE = "Grafana Enterprise" +) + +var ( + ERR_TEMPLATE_NAME = "error" ) var ( // App settings. - Env string = DEV + Env = DEV AppUrl string AppSubUrl string InstanceName string // build - BuildVersion string - BuildCommit string - BuildStamp int64 + BuildVersion string + BuildCommit string + BuildBranch string + BuildStamp int64 + IsEnterprise bool + ApplicationName string // Paths - LogsPath string - HomePath string - DataPath string - PluginsPath string - ProvisioningPath string - CustomInitPath = "conf/custom.ini" + HomePath string + PluginsPath string + CustomInitPath = "conf/custom.ini" // Log settings. - LogModes []string LogConfigs []util.DynMap // Http server options @@ -97,12 +101,14 @@ var ( AllowUserSignUp bool AllowUserOrgCreate bool AutoAssignOrg bool + AutoAssignOrgId int AutoAssignOrgRole string VerifyEmailEnabled bool LoginHint string DefaultTheme string DisableLoginForm bool DisableSignoutMenu bool + SignoutRedirectUrl string ExternalUserMngLinkUrl string ExternalUserMngLinkName string ExternalUserMngInfo string @@ -123,6 +129,7 @@ var ( AuthProxyAutoSignUp bool AuthProxyLdapSyncTtl int AuthProxyWhitelist string + AuthProxyHeaders map[string]string // Basic Auth BasicAuthEnabled bool @@ -131,17 +138,14 @@ var ( PluginAppsSkipVerifyTLS bool // Session settings. - SessionOptions session.Options + SessionOptions session.Options + SessionConnMaxLifetime int64 // Global setting objects. - Cfg *ini.File + Raw *ini.File ConfRootPath string IsWindows bool - // PhantomJs Rendering - ImagesDir string - PhantomDir string - // for logging purposes configFiles []string appliedCommandLineProperties []string @@ -155,17 +159,20 @@ var ( // LDAP LdapEnabled bool LdapConfigFile string - LdapAllowSignup bool = true - - // SMTP email settings - Smtp SmtpSettings + LdapAllowSignup = true // QUOTA Quota QuotaSettings // Alerting - AlertingEnabled bool - ExecuteAlerts bool + AlertingEnabled bool + ExecuteAlerts bool + AlertingRenderLimit int + AlertingErrorOrTimeout string + AlertingNoDataOrNullValues string + + // Explore UI + ExploreEnabled bool // logger logger log.Logger @@ -181,6 +188,37 @@ var ( ImageUploadProvider string ) +// TODO move all global vars to this struct +type Cfg struct { + Raw *ini.File + + // HTTP Server Settings + AppUrl string + AppSubUrl string + + // Paths + ProvisioningPath string + DataPath string + LogsPath string + + // SMTP email settings + Smtp SmtpSettings + + // Rendering + ImagesDir string + PhantomDir string + RendererUrl string + RendererCallbackUrl string + RendererLimit int + RendererLimitAlerting int + + DisableBruteForceLoginProtection bool + TempDataLifetime time.Duration + MetricsEndpointEnabled bool + EnableAlphaPanels bool + EnterpriseLicensePath string +} + type CommandLineArgs struct { Config string HomePath string @@ -222,9 +260,9 @@ func shouldRedactURLKey(s string) bool { return strings.Contains(uppercased, "DATABASE_URL") } -func applyEnvVariableOverrides() { +func applyEnvVariableOverrides(file *ini.File) error { appliedEnvOverrides = make([]string, 0) - for _, section := range Cfg.Sections() { + for _, section := range file.Sections() { for _, key := range section.Keys() { sectionName := strings.ToUpper(strings.Replace(section.Name(), ".", "_", -1)) keyName := strings.ToUpper(strings.Replace(key.Name(), ".", "_", -1)) @@ -237,7 +275,10 @@ func applyEnvVariableOverrides() { envValue = "*********" } if shouldRedactURLKey(envKey) { - u, _ := url.Parse(envValue) + u, err := url.Parse(envValue) + if err != nil { + return fmt.Errorf("could not parse environment variable. key: %s, value: %s. error: %v", envKey, envValue, err) + } ui := u.User if ui != nil { _, exists := ui.Password() @@ -251,11 +292,13 @@ func applyEnvVariableOverrides() { } } } + + return nil } -func applyCommandLineDefaultProperties(props map[string]string) { +func applyCommandLineDefaultProperties(props map[string]string, file *ini.File) { appliedCommandLineProperties = make([]string, 0) - for _, section := range Cfg.Sections() { + for _, section := range file.Sections() { for _, key := range section.Keys() { keyString := fmt.Sprintf("default.%s.%s", section.Name(), key.Name()) value, exists := props[keyString] @@ -270,8 +313,8 @@ func applyCommandLineDefaultProperties(props map[string]string) { } } -func applyCommandLineProperties(props map[string]string) { - for _, section := range Cfg.Sections() { +func applyCommandLineProperties(props map[string]string, file *ini.File) { + for _, section := range file.Sections() { sectionName := section.Name() + "." if section.Name() == ini.DEFAULT_SECTION { sectionName = "" @@ -298,7 +341,7 @@ func getCommandLineProperties(args []string) map[string]string { trimmed := strings.TrimPrefix(arg, "cfg:") parts := strings.Split(trimmed, "=") if len(parts) != 2 { - log.Fatal(3, "Invalid command line argument", arg) + log.Fatal(3, "Invalid command line argument. argument: %v", arg) return nil } @@ -330,15 +373,15 @@ func evalEnvVarExpression(value string) string { }) } -func evalConfigValues() { - for _, section := range Cfg.Sections() { +func evalConfigValues(file *ini.File) { + for _, section := range file.Sections() { for _, key := range section.Keys() { key.SetValue(evalEnvVarExpression(key.Value())) } } } -func loadSpecifedConfigFile(configFile string) error { +func loadSpecifedConfigFile(configFile string, masterFile *ini.File) error { if configFile == "" { configFile = filepath.Join(HomePath, CustomInitPath) // return without error if custom file does not exist @@ -360,9 +403,9 @@ func loadSpecifedConfigFile(configFile string) error { continue } - defaultSec, err := Cfg.GetSection(section.Name()) + defaultSec, err := masterFile.GetSection(section.Name()) if err != nil { - defaultSec, _ = Cfg.NewSection(section.Name()) + defaultSec, _ = masterFile.NewSection(section.Name()) } defaultKey, err := defaultSec.GetKey(key.Name()) if err != nil { @@ -376,7 +419,7 @@ func loadSpecifedConfigFile(configFile string) error { return nil } -func loadConfiguration(args *CommandLineArgs) { +func (cfg *Cfg) loadConfiguration(args *CommandLineArgs) (*ini.File, error) { var err error // load config defaults @@ -390,39 +433,44 @@ func loadConfiguration(args *CommandLineArgs) { } // load defaults - Cfg, err = ini.Load(defaultConfigFile) + parsedFile, err := ini.Load(defaultConfigFile) if err != nil { fmt.Println(fmt.Sprintf("Failed to parse defaults.ini, %v", err)) os.Exit(1) - return + return nil, err } - Cfg.BlockMode = false + parsedFile.BlockMode = false // command line props commandLineProps := getCommandLineProperties(args.Args) // load default overrides - applyCommandLineDefaultProperties(commandLineProps) + applyCommandLineDefaultProperties(commandLineProps, parsedFile) // load specified config file - err = loadSpecifedConfigFile(args.Config) + err = loadSpecifedConfigFile(args.Config, parsedFile) if err != nil { - initLogging() + cfg.initLogging(parsedFile) log.Fatal(3, err.Error()) } // apply environment overrides - applyEnvVariableOverrides() + err = applyEnvVariableOverrides(parsedFile) + if err != nil { + return nil, err + } // apply command line overrides - applyCommandLineProperties(commandLineProps) + applyCommandLineProperties(commandLineProps, parsedFile) // evaluate config values containing environment variables - evalConfigValues() + evalConfigValues(parsedFile) // update data path and logging config - DataPath = makeAbsolute(Cfg.Section("paths").Key("data").String(), HomePath) - initLogging() + cfg.DataPath = makeAbsolute(parsedFile.Section("paths").Key("data").String(), HomePath) + cfg.initLogging(parsedFile) + + return parsedFile, err } func pathExists(path string) bool { @@ -454,7 +502,7 @@ func setHomePath(args *CommandLineArgs) { } } -var skipStaticRootValidation bool = false +var skipStaticRootValidation = false func validateStaticRootPath() error { if skipStaticRootValidation { @@ -468,16 +516,38 @@ func validateStaticRootPath() error { return nil } -func NewConfigContext(args *CommandLineArgs) error { - setHomePath(args) - loadConfiguration(args) +func NewCfg() *Cfg { + return &Cfg{ + Raw: ini.Empty(), + } +} - Env = Cfg.Section("").Key("app_mode").MustString("development") - InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name") - PluginsPath = makeAbsolute(Cfg.Section("paths").Key("plugins").String(), HomePath) - ProvisioningPath = makeAbsolute(Cfg.Section("paths").Key("provisioning").String(), HomePath) - server := Cfg.Section("server") +func (cfg *Cfg) Load(args *CommandLineArgs) error { + setHomePath(args) + + iniFile, err := cfg.loadConfiguration(args) + if err != nil { + return err + } + + cfg.Raw = iniFile + + // Temporary keep global, to make refactor in steps + Raw = cfg.Raw + + ApplicationName = APP_NAME + if IsEnterprise { + ApplicationName = APP_NAME_ENTERPRISE + } + + Env = iniFile.Section("").Key("app_mode").MustString("development") + InstanceName = iniFile.Section("").Key("instance_name").MustString("unknown_instance_name") + PluginsPath = makeAbsolute(iniFile.Section("paths").Key("plugins").String(), HomePath) + cfg.ProvisioningPath = makeAbsolute(iniFile.Section("paths").Key("provisioning").String(), HomePath) + server := iniFile.Section("server") AppUrl, AppSubUrl = parseAppUrlAndSubUrl(server) + cfg.AppUrl = AppUrl + cfg.AppSubUrl = AppSubUrl Protocol = HTTP if server.Key("protocol").MustString("http") == "https" { @@ -504,27 +574,28 @@ func NewConfigContext(args *CommandLineArgs) error { } // read data proxy settings - dataproxy := Cfg.Section("dataproxy") + dataproxy := iniFile.Section("dataproxy") DataProxyLogging = dataproxy.Key("logging").MustBool(false) // read security settings - security := Cfg.Section("security") + security := iniFile.Section("security") SecretKey = security.Key("secret_key").String() LogInRememberDays = security.Key("login_remember_days").MustInt() CookieUserName = security.Key("cookie_username").String() CookieRememberName = security.Key("cookie_remember_name").String() DisableGravatar = security.Key("disable_gravatar").MustBool(true) - DisableBruteForceLoginProtection = security.Key("disable_brute_force_login_protection").MustBool(false) + cfg.DisableBruteForceLoginProtection = security.Key("disable_brute_force_login_protection").MustBool(false) + DisableBruteForceLoginProtection = cfg.DisableBruteForceLoginProtection // read snapshots settings - snapshots := Cfg.Section("snapshots") + snapshots := iniFile.Section("snapshots") ExternalSnapshotUrl = snapshots.Key("external_snapshot_url").String() ExternalSnapshotName = snapshots.Key("external_snapshot_name").String() ExternalEnabled = snapshots.Key("external_enabled").MustBool(true) SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true) // read dashboard settings - dashboards := Cfg.Section("dashboards") + dashboards := iniFile.Section("dashboards") DashboardVersionsToKeep = dashboards.Key("versions_to_keep").MustInt(20) // read data source proxy white list @@ -537,10 +608,11 @@ func NewConfigContext(args *CommandLineArgs) error { AdminUser = security.Key("admin_user").String() AdminPassword = security.Key("admin_password").String() - users := Cfg.Section("users") + users := iniFile.Section("users") AllowUserSignUp = users.Key("allow_sign_up").MustBool(true) AllowUserOrgCreate = users.Key("allow_org_create").MustBool(true) AutoAssignOrg = users.Key("auto_assign_org").MustBool(true) + AutoAssignOrgId = users.Key("auto_assign_org_id").MustInt(1) AutoAssignOrgRole = users.Key("auto_assign_org_role").In("Editor", []string{"Editor", "Admin", "Viewer"}) VerifyEmailEnabled = users.Key("verify_email_enabled").MustBool(false) LoginHint = users.Key("login_hint").String() @@ -551,17 +623,18 @@ func NewConfigContext(args *CommandLineArgs) error { ViewersCanEdit = users.Key("viewers_can_edit").MustBool(false) // auth - auth := Cfg.Section("auth") + auth := iniFile.Section("auth") DisableLoginForm = auth.Key("disable_login_form").MustBool(false) DisableSignoutMenu = auth.Key("disable_signout_menu").MustBool(false) + SignoutRedirectUrl = auth.Key("signout_redirect_url").String() // anonymous access - AnonymousEnabled = Cfg.Section("auth.anonymous").Key("enabled").MustBool(false) - AnonymousOrgName = Cfg.Section("auth.anonymous").Key("org_name").String() - AnonymousOrgRole = Cfg.Section("auth.anonymous").Key("org_role").String() + AnonymousEnabled = iniFile.Section("auth.anonymous").Key("enabled").MustBool(false) + AnonymousOrgName = iniFile.Section("auth.anonymous").Key("org_name").String() + AnonymousOrgRole = iniFile.Section("auth.anonymous").Key("org_role").String() // auth proxy - authProxy := Cfg.Section("auth.proxy") + authProxy := iniFile.Section("auth.proxy") AuthProxyEnabled = authProxy.Key("enabled").MustBool(false) AuthProxyHeaderName = authProxy.Key("header_name").String() AuthProxyHeaderProperty = authProxy.Key("header_property").String() @@ -569,85 +642,124 @@ func NewConfigContext(args *CommandLineArgs) error { AuthProxyLdapSyncTtl = authProxy.Key("ldap_sync_ttl").MustInt() AuthProxyWhitelist = authProxy.Key("whitelist").String() + AuthProxyHeaders = make(map[string]string) + for _, propertyAndHeader := range util.SplitString(authProxy.Key("headers").String()) { + split := strings.SplitN(propertyAndHeader, ":", 2) + if len(split) == 2 { + AuthProxyHeaders[split[0]] = split[1] + } + } + // basic auth - authBasic := Cfg.Section("auth.basic") + authBasic := iniFile.Section("auth.basic") BasicAuthEnabled = authBasic.Key("enabled").MustBool(true) // global plugin settings - PluginAppsSkipVerifyTLS = Cfg.Section("plugins").Key("app_tls_skip_verify_insecure").MustBool(false) + PluginAppsSkipVerifyTLS = iniFile.Section("plugins").Key("app_tls_skip_verify_insecure").MustBool(false) - // PhantomJS rendering - ImagesDir = filepath.Join(DataPath, "png") - PhantomDir = filepath.Join(HomePath, "tools/phantomjs") + // Rendering + renderSec := iniFile.Section("rendering") + cfg.RendererUrl = renderSec.Key("server_url").String() + cfg.RendererCallbackUrl = renderSec.Key("callback_url").String() + if cfg.RendererCallbackUrl == "" { + cfg.RendererCallbackUrl = AppUrl + } else { + if cfg.RendererCallbackUrl[len(cfg.RendererCallbackUrl)-1] != '/' { + cfg.RendererCallbackUrl += "/" + } + _, err := url.Parse(cfg.RendererCallbackUrl) + if err != nil { + log.Fatal(4, "Invalid callback_url(%s): %s", cfg.RendererCallbackUrl, err) + } + } + cfg.ImagesDir = filepath.Join(cfg.DataPath, "png") + cfg.PhantomDir = filepath.Join(HomePath, "tools/phantomjs") + cfg.TempDataLifetime = iniFile.Section("paths").Key("temp_data_lifetime").MustDuration(time.Second * 3600 * 24) + cfg.MetricsEndpointEnabled = iniFile.Section("metrics").Key("enabled").MustBool(true) - analytics := Cfg.Section("analytics") + analytics := iniFile.Section("analytics") ReportingEnabled = analytics.Key("reporting_enabled").MustBool(true) CheckForUpdates = analytics.Key("check_for_updates").MustBool(true) GoogleAnalyticsId = analytics.Key("google_analytics_ua_id").String() GoogleTagManagerId = analytics.Key("google_tag_manager_id").String() - ldapSec := Cfg.Section("auth.ldap") + ldapSec := iniFile.Section("auth.ldap") LdapEnabled = ldapSec.Key("enabled").MustBool(false) LdapConfigFile = ldapSec.Key("config_file").String() LdapAllowSignup = ldapSec.Key("allow_sign_up").MustBool(true) - alerting := Cfg.Section("alerting") + alerting := iniFile.Section("alerting") AlertingEnabled = alerting.Key("enabled").MustBool(true) ExecuteAlerts = alerting.Key("execute_alerts").MustBool(true) + AlertingRenderLimit = alerting.Key("concurrent_render_limit").MustInt(5) + AlertingErrorOrTimeout = alerting.Key("error_or_timeout").MustString("alerting") + AlertingNoDataOrNullValues = alerting.Key("nodata_or_nullvalues").MustString("no_data") - readSessionConfig() - readSmtpSettings() - readQuotaSettings() + explore := iniFile.Section("explore") + ExploreEnabled = explore.Key("enabled").MustBool(false) - if VerifyEmailEnabled && !Smtp.Enabled { + panels := iniFile.Section("panels") + cfg.EnableAlphaPanels = panels.Key("enable_alpha").MustBool(false) + + cfg.readSessionConfig() + cfg.readSmtpSettings() + cfg.readQuotaSettings() + + if VerifyEmailEnabled && !cfg.Smtp.Enabled { log.Warn("require_email_validation is enabled but smtp is disabled") } // check old key name - GrafanaComUrl = Cfg.Section("grafana_net").Key("url").MustString("") + GrafanaComUrl = iniFile.Section("grafana_net").Key("url").MustString("") if GrafanaComUrl == "" { - GrafanaComUrl = Cfg.Section("grafana_com").Key("url").MustString("https://grafana.com") + GrafanaComUrl = iniFile.Section("grafana_com").Key("url").MustString("https://grafana.com") } - imageUploadingSection := Cfg.Section("external_image_storage") + imageUploadingSection := iniFile.Section("external_image_storage") ImageUploadProvider = imageUploadingSection.Key("provider").MustString("") + + enterprise := iniFile.Section("enterprise") + cfg.EnterpriseLicensePath = enterprise.Key("license_path").MustString(filepath.Join(cfg.DataPath, "license.jwt")) + return nil } -func readSessionConfig() { - sec := Cfg.Section("session") +func (cfg *Cfg) readSessionConfig() { + sec := cfg.Raw.Section("session") SessionOptions = session.Options{} SessionOptions.Provider = sec.Key("provider").In("memory", []string{"memory", "file", "redis", "mysql", "postgres", "memcache"}) SessionOptions.ProviderConfig = strings.Trim(sec.Key("provider_config").String(), "\" ") SessionOptions.CookieName = sec.Key("cookie_name").MustString("grafana_sess") SessionOptions.CookiePath = AppSubUrl SessionOptions.Secure = sec.Key("cookie_secure").MustBool() - SessionOptions.Gclifetime = Cfg.Section("session").Key("gc_interval_time").MustInt64(86400) - SessionOptions.Maxlifetime = Cfg.Section("session").Key("session_life_time").MustInt64(86400) + SessionOptions.Gclifetime = cfg.Raw.Section("session").Key("gc_interval_time").MustInt64(86400) + SessionOptions.Maxlifetime = cfg.Raw.Section("session").Key("session_life_time").MustInt64(86400) SessionOptions.IDLength = 16 if SessionOptions.Provider == "file" { - SessionOptions.ProviderConfig = makeAbsolute(SessionOptions.ProviderConfig, DataPath) + SessionOptions.ProviderConfig = makeAbsolute(SessionOptions.ProviderConfig, cfg.DataPath) os.MkdirAll(path.Dir(SessionOptions.ProviderConfig), os.ModePerm) } if SessionOptions.CookiePath == "" { SessionOptions.CookiePath = "/" } + + SessionConnMaxLifetime = cfg.Raw.Section("session").Key("conn_max_lifetime").MustInt64(14400) } -func initLogging() { +func (cfg *Cfg) initLogging(file *ini.File) { // split on comma - LogModes = strings.Split(Cfg.Section("log").Key("mode").MustString("console"), ",") + logModes := strings.Split(file.Section("log").Key("mode").MustString("console"), ",") // also try space - if len(LogModes) == 1 { - LogModes = strings.Split(Cfg.Section("log").Key("mode").MustString("console"), " ") + if len(logModes) == 1 { + logModes = strings.Split(file.Section("log").Key("mode").MustString("console"), " ") } - LogsPath = makeAbsolute(Cfg.Section("paths").Key("logs").String(), HomePath) - log.ReadLoggingConfig(LogModes, LogsPath, Cfg) + cfg.LogsPath = makeAbsolute(file.Section("paths").Key("logs").String(), HomePath) + log.ReadLoggingConfig(logModes, cfg.LogsPath, file) } -func LogConfigurationInfo() { +func (cfg *Cfg) LogConfigSources() { var text bytes.Buffer for _, file := range configFiles { @@ -668,9 +780,9 @@ func LogConfigurationInfo() { } logger.Info("Path Home", "path", HomePath) - logger.Info("Path Data", "path", DataPath) - logger.Info("Path Logs", "path", LogsPath) + logger.Info("Path Data", "path", cfg.DataPath) + logger.Info("Path Logs", "path", cfg.LogsPath) logger.Info("Path Plugins", "path", PluginsPath) - logger.Info("Path Provisioning", "path", ProvisioningPath) + logger.Info("Path Provisioning", "path", cfg.ProvisioningPath) logger.Info("App mode " + Env) } diff --git a/pkg/setting/setting_oauth.go b/pkg/setting/setting_oauth.go index ee2e812415b..93b1ab6f101 100644 --- a/pkg/setting/setting_oauth.go +++ b/pkg/setting/setting_oauth.go @@ -5,6 +5,7 @@ type OAuthInfo struct { Scopes []string AuthUrl, TokenUrl string Enabled bool + EmailAttributeName string AllowedDomains []string HostedDomain string ApiUrl string diff --git a/pkg/setting/setting_quota.go b/pkg/setting/setting_quota.go index 49769d9930f..c3a509219db 100644 --- a/pkg/setting/setting_quota.go +++ b/pkg/setting/setting_quota.go @@ -63,9 +63,9 @@ type QuotaSettings struct { Global *GlobalQuota } -func readQuotaSettings() { +func (cfg *Cfg) readQuotaSettings() { // set global defaults. - quota := Cfg.Section("quota") + quota := cfg.Raw.Section("quota") Quota.Enabled = quota.Key("enabled").MustBool(false) // per ORG Limits diff --git a/pkg/setting/setting_smtp.go b/pkg/setting/setting_smtp.go index 9d8b8a529a5..5df774dc691 100644 --- a/pkg/setting/setting_smtp.go +++ b/pkg/setting/setting_smtp.go @@ -16,20 +16,20 @@ type SmtpSettings struct { TemplatesPattern string } -func readSmtpSettings() { - sec := Cfg.Section("smtp") - Smtp.Enabled = sec.Key("enabled").MustBool(false) - Smtp.Host = sec.Key("host").String() - Smtp.User = sec.Key("user").String() - Smtp.Password = sec.Key("password").String() - Smtp.CertFile = sec.Key("cert_file").String() - Smtp.KeyFile = sec.Key("key_file").String() - Smtp.FromAddress = sec.Key("from_address").String() - Smtp.FromName = sec.Key("from_name").String() - Smtp.EhloIdentity = sec.Key("ehlo_identity").String() - Smtp.SkipVerify = sec.Key("skip_verify").MustBool(false) +func (cfg *Cfg) readSmtpSettings() { + sec := cfg.Raw.Section("smtp") + cfg.Smtp.Enabled = sec.Key("enabled").MustBool(false) + cfg.Smtp.Host = sec.Key("host").String() + cfg.Smtp.User = sec.Key("user").String() + cfg.Smtp.Password = sec.Key("password").String() + cfg.Smtp.CertFile = sec.Key("cert_file").String() + cfg.Smtp.KeyFile = sec.Key("key_file").String() + cfg.Smtp.FromAddress = sec.Key("from_address").String() + cfg.Smtp.FromName = sec.Key("from_name").String() + cfg.Smtp.EhloIdentity = sec.Key("ehlo_identity").String() + cfg.Smtp.SkipVerify = sec.Key("skip_verify").MustBool(false) - emails := Cfg.Section("emails") - Smtp.SendWelcomeEmailOnSignUp = emails.Key("welcome_email_on_sign_up").MustBool(false) - Smtp.TemplatesPattern = emails.Key("templates_pattern").MustString("emails/*.html") + emails := cfg.Raw.Section("emails") + cfg.Smtp.SendWelcomeEmailOnSignUp = emails.Key("welcome_email_on_sign_up").MustBool(false) + cfg.Smtp.TemplatesPattern = emails.Key("templates_pattern").MustString("emails/*.html") } diff --git a/pkg/setting/setting_test.go b/pkg/setting/setting_test.go index 640a1648340..72dbe2378c7 100644 --- a/pkg/setting/setting_test.go +++ b/pkg/setting/setting_test.go @@ -15,31 +15,48 @@ func TestLoadingSettings(t *testing.T) { skipStaticRootValidation = true Convey("Given the default ini files", func() { - err := NewConfigContext(&CommandLineArgs{HomePath: "../../"}) + cfg := NewCfg() + err := cfg.Load(&CommandLineArgs{HomePath: "../../"}) So(err, ShouldBeNil) So(AdminUser, ShouldEqual, "admin") + So(cfg.RendererCallbackUrl, ShouldEqual, "http://localhost:3000/") }) Convey("Should be able to override via environment variables", func() { os.Setenv("GF_SECURITY_ADMIN_USER", "superduper") - NewConfigContext(&CommandLineArgs{HomePath: "../../"}) + + cfg := NewCfg() + cfg.Load(&CommandLineArgs{HomePath: "../../"}) So(AdminUser, ShouldEqual, "superduper") - So(DataPath, ShouldEqual, filepath.Join(HomePath, "data")) - So(LogsPath, ShouldEqual, filepath.Join(DataPath, "log")) + So(cfg.DataPath, ShouldEqual, filepath.Join(HomePath, "data")) + So(cfg.LogsPath, ShouldEqual, filepath.Join(cfg.DataPath, "log")) }) Convey("Should replace password when defined in environment", func() { os.Setenv("GF_SECURITY_ADMIN_PASSWORD", "supersecret") - NewConfigContext(&CommandLineArgs{HomePath: "../../"}) + + cfg := NewCfg() + cfg.Load(&CommandLineArgs{HomePath: "../../"}) So(appliedEnvOverrides, ShouldContain, "GF_SECURITY_ADMIN_PASSWORD=*********") }) + Convey("Should return an error when url is invalid", func() { + os.Setenv("GF_DATABASE_URL", "postgres.%31://grafana:secret@postgres:5432/grafana") + + cfg := NewCfg() + err := cfg.Load(&CommandLineArgs{HomePath: "../../"}) + + So(err, ShouldNotBeNil) + }) + Convey("Should replace password in URL when url environment is defined", func() { os.Setenv("GF_DATABASE_URL", "mysql://user:secret@localhost:3306/database") - NewConfigContext(&CommandLineArgs{HomePath: "../../"}) + + cfg := NewCfg() + cfg.Load(&CommandLineArgs{HomePath: "../../"}) So(appliedEnvOverrides, ShouldContain, "GF_DATABASE_URL=mysql://user:-redacted-@localhost:3306/database") }) @@ -54,30 +71,33 @@ func TestLoadingSettings(t *testing.T) { Convey("Should be able to override via command line", func() { if runtime.GOOS == "windows" { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", Args: []string{`cfg:paths.data=c:\tmp\data`, `cfg:paths.logs=c:\tmp\logs`}, }) - So(DataPath, ShouldEqual, `c:\tmp\data`) - So(LogsPath, ShouldEqual, `c:\tmp\logs`) + So(cfg.DataPath, ShouldEqual, `c:\tmp\data`) + So(cfg.LogsPath, ShouldEqual, `c:\tmp\logs`) } else { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", Args: []string{"cfg:paths.data=/tmp/data", "cfg:paths.logs=/tmp/logs"}, }) - So(DataPath, ShouldEqual, "/tmp/data") - So(LogsPath, ShouldEqual, "/tmp/logs") + So(cfg.DataPath, ShouldEqual, "/tmp/data") + So(cfg.LogsPath, ShouldEqual, "/tmp/logs") } }) Convey("Should be able to override defaults via command line", func() { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", Args: []string{ "cfg:default.server.domain=test2", }, - Config: filepath.Join(HomePath, "tests/config-files/override.ini"), + Config: filepath.Join(HomePath, "pkg/setting/testdata/override.ini"), }) So(Domain, ShouldEqual, "test2") @@ -85,66 +105,73 @@ func TestLoadingSettings(t *testing.T) { Convey("Defaults can be overridden in specified config file", func() { if runtime.GOOS == "windows" { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", - Config: filepath.Join(HomePath, "tests/config-files/override_windows.ini"), + Config: filepath.Join(HomePath, "pkg/setting/testdata/override_windows.ini"), Args: []string{`cfg:default.paths.data=c:\tmp\data`}, }) - So(DataPath, ShouldEqual, `c:\tmp\override`) + So(cfg.DataPath, ShouldEqual, `c:\tmp\override`) } else { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", - Config: filepath.Join(HomePath, "tests/config-files/override.ini"), + Config: filepath.Join(HomePath, "pkg/setting/testdata/override.ini"), Args: []string{"cfg:default.paths.data=/tmp/data"}, }) - So(DataPath, ShouldEqual, "/tmp/override") + So(cfg.DataPath, ShouldEqual, "/tmp/override") } }) Convey("Command line overrides specified config file", func() { if runtime.GOOS == "windows" { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", - Config: filepath.Join(HomePath, "tests/config-files/override_windows.ini"), + Config: filepath.Join(HomePath, "pkg/setting/testdata/override_windows.ini"), Args: []string{`cfg:paths.data=c:\tmp\data`}, }) - So(DataPath, ShouldEqual, `c:\tmp\data`) + So(cfg.DataPath, ShouldEqual, `c:\tmp\data`) } else { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", - Config: filepath.Join(HomePath, "tests/config-files/override.ini"), + Config: filepath.Join(HomePath, "pkg/setting/testdata/override.ini"), Args: []string{"cfg:paths.data=/tmp/data"}, }) - So(DataPath, ShouldEqual, "/tmp/data") + So(cfg.DataPath, ShouldEqual, "/tmp/data") } }) Convey("Can use environment variables in config values", func() { if runtime.GOOS == "windows" { os.Setenv("GF_DATA_PATH", `c:\tmp\env_override`) - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", Args: []string{"cfg:paths.data=${GF_DATA_PATH}"}, }) - So(DataPath, ShouldEqual, `c:\tmp\env_override`) + So(cfg.DataPath, ShouldEqual, `c:\tmp\env_override`) } else { os.Setenv("GF_DATA_PATH", "/tmp/env_override") - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", Args: []string{"cfg:paths.data=${GF_DATA_PATH}"}, }) - So(DataPath, ShouldEqual, "/tmp/env_override") + So(cfg.DataPath, ShouldEqual, "/tmp/env_override") } }) Convey("instance_name default to hostname even if hostname env is empty", func() { - NewConfigContext(&CommandLineArgs{ + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ HomePath: "../../", }) @@ -152,5 +179,15 @@ func TestLoadingSettings(t *testing.T) { So(InstanceName, ShouldEqual, hostname) }) + Convey("Reading callback_url should add trailing slash", func() { + cfg := NewCfg() + cfg.Load(&CommandLineArgs{ + HomePath: "../../", + Args: []string{"cfg:rendering.callback_url=http://myserver/renderer"}, + }) + + So(cfg.RendererCallbackUrl, ShouldEqual, "http://myserver/renderer/") + }) + }) } diff --git a/tests/config-files/override.ini b/pkg/setting/testdata/override.ini similarity index 100% rename from tests/config-files/override.ini rename to pkg/setting/testdata/override.ini diff --git a/tests/config-files/override_windows.ini b/pkg/setting/testdata/override_windows.ini similarity index 100% rename from tests/config-files/override_windows.ini rename to pkg/setting/testdata/override_windows.ini diff --git a/pkg/social/generic_oauth.go b/pkg/social/generic_oauth.go index b92d64ad9fc..a97d58334c7 100644 --- a/pkg/social/generic_oauth.go +++ b/pkg/social/generic_oauth.go @@ -20,6 +20,7 @@ type SocialGenericOAuth struct { allowedOrganizations []string apiUrl string allowSignup bool + emailAttributeName string teamIds []int } @@ -182,7 +183,7 @@ func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token) var data UserInfoJson var err error - if s.extractToken(&data, token) != true { + if !s.extractToken(&data, token) { response, err := HttpGet(client, s.apiUrl) if err != nil { return nil, fmt.Errorf("Error getting user info: %s", err) @@ -264,8 +265,9 @@ func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson) string { return data.Email } - if data.Attributes["email:primary"] != nil { - return data.Attributes["email:primary"][0] + emails, ok := data.Attributes[s.emailAttributeName] + if ok && len(emails) != 0 { + return emails[0] } if data.Upn != "" { diff --git a/pkg/social/github_oauth.go b/pkg/social/github_oauth.go index 815c684cf03..b07f112b8d3 100644 --- a/pkg/social/github_oauth.go +++ b/pkg/social/github_oauth.go @@ -213,6 +213,7 @@ func (s *SocialGithub) UserInfo(client *http.Client, token *oauth2.Token) (*Basi userInfo := &BasicUserInfo{ Name: data.Login, Login: data.Login, + Id: fmt.Sprintf("%d", data.Id), Email: data.Email, } diff --git a/pkg/social/gitlab_oauth.go b/pkg/social/gitlab_oauth.go new file mode 100644 index 00000000000..21463dabf8f --- /dev/null +++ b/pkg/social/gitlab_oauth.go @@ -0,0 +1,132 @@ +package social + +import ( + "encoding/json" + "fmt" + "net/http" + "regexp" + + "github.com/grafana/grafana/pkg/models" + + "golang.org/x/oauth2" +) + +type SocialGitlab struct { + *SocialBase + allowedDomains []string + allowedGroups []string + apiUrl string + allowSignup bool +} + +var ( + ErrMissingGroupMembership = &Error{"User not a member of one of the required groups"} +) + +func (s *SocialGitlab) Type() int { + return int(models.GITLAB) +} + +func (s *SocialGitlab) IsEmailAllowed(email string) bool { + return isEmailAllowed(email, s.allowedDomains) +} + +func (s *SocialGitlab) IsSignupAllowed() bool { + return s.allowSignup +} + +func (s *SocialGitlab) IsGroupMember(client *http.Client) bool { + if len(s.allowedGroups) == 0 { + return true + } + + for groups, url := s.GetGroups(client, s.apiUrl+"/groups"); groups != nil; groups, url = s.GetGroups(client, url) { + for _, allowedGroup := range s.allowedGroups { + for _, group := range groups { + if group == allowedGroup { + return true + } + } + } + } + + return false +} + +func (s *SocialGitlab) GetGroups(client *http.Client, url string) ([]string, string) { + type Group struct { + FullPath string `json:"full_path"` + } + + var ( + groups []Group + next string + ) + + if url == "" { + return nil, next + } + + response, err := HttpGet(client, url) + if err != nil { + s.log.Error("Error getting groups from GitLab API", "err", err) + return nil, next + } + + if err := json.Unmarshal(response.Body, &groups); err != nil { + s.log.Error("Error parsing JSON from GitLab API", "err", err) + return nil, next + } + + fullPaths := make([]string, len(groups)) + for i, group := range groups { + fullPaths[i] = group.FullPath + } + + if link, ok := response.Headers["Link"]; ok { + pattern := regexp.MustCompile(`<([^>]+)>; rel="next"`) + if matches := pattern.FindStringSubmatch(link[0]); matches != nil { + next = matches[1] + } + } + + return fullPaths, next +} + +func (s *SocialGitlab) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) { + + var data struct { + Id int + Username string + Email string + Name string + State string + } + + response, err := HttpGet(client, s.apiUrl+"/user") + if err != nil { + return nil, fmt.Errorf("Error getting user info: %s", err) + } + + err = json.Unmarshal(response.Body, &data) + if err != nil { + return nil, fmt.Errorf("Error getting user info: %s", err) + } + + if data.State != "active" { + return nil, fmt.Errorf("User %s is inactive", data.Username) + } + + userInfo := &BasicUserInfo{ + Id: fmt.Sprintf("%d", data.Id), + Name: data.Name, + Login: data.Username, + Email: data.Email, + } + + if !s.IsGroupMember(client) { + return nil, ErrMissingGroupMembership + } + + return userInfo, nil +} diff --git a/pkg/social/grafana_com_oauth.go b/pkg/social/grafana_com_oauth.go index d3614520d61..87601788c3f 100644 --- a/pkg/social/grafana_com_oauth.go +++ b/pkg/social/grafana_com_oauth.go @@ -51,6 +51,7 @@ func (s *SocialGrafanaCom) IsOrganizationMember(organizations []OrgRecord) bool func (s *SocialGrafanaCom) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) { var data struct { + Id int `json:"id"` Name string `json:"name"` Login string `json:"username"` Email string `json:"email"` @@ -69,6 +70,7 @@ func (s *SocialGrafanaCom) UserInfo(client *http.Client, token *oauth2.Token) (* } userInfo := &BasicUserInfo{ + Id: fmt.Sprintf("%d", data.Id), Name: data.Name, Login: data.Login, Email: data.Email, diff --git a/pkg/social/social.go b/pkg/social/social.go index b763e2d71b2..8918507f3b9 100644 --- a/pkg/social/social.go +++ b/pkg/social/social.go @@ -14,6 +14,7 @@ import ( ) type BasicUserInfo struct { + Id string Name string Email string Login string @@ -45,35 +46,39 @@ func (e *Error) Error() string { return e.s } +const ( + grafanaCom = "grafana_com" +) + var ( SocialBaseUrl = "/login/" SocialMap = make(map[string]SocialConnector) + allOauthes = []string{"github", "gitlab", "google", "generic_oauth", "grafananet", grafanaCom} ) func NewOAuthService() { setting.OAuthService = &setting.OAuther{} setting.OAuthService.OAuthInfos = make(map[string]*setting.OAuthInfo) - allOauthes := []string{"github", "google", "generic_oauth", "grafananet", "grafana_com"} - for _, name := range allOauthes { - sec := setting.Cfg.Section("auth." + name) + sec := setting.Raw.Section("auth." + name) info := &setting.OAuthInfo{ - ClientId: sec.Key("client_id").String(), - ClientSecret: sec.Key("client_secret").String(), - Scopes: util.SplitString(sec.Key("scopes").String()), - AuthUrl: sec.Key("auth_url").String(), - TokenUrl: sec.Key("token_url").String(), - ApiUrl: sec.Key("api_url").String(), - Enabled: sec.Key("enabled").MustBool(), - AllowedDomains: util.SplitString(sec.Key("allowed_domains").String()), - HostedDomain: sec.Key("hosted_domain").String(), - AllowSignup: sec.Key("allow_sign_up").MustBool(), - Name: sec.Key("name").MustString(name), - TlsClientCert: sec.Key("tls_client_cert").String(), - TlsClientKey: sec.Key("tls_client_key").String(), - TlsClientCa: sec.Key("tls_client_ca").String(), - TlsSkipVerify: sec.Key("tls_skip_verify_insecure").MustBool(), + ClientId: sec.Key("client_id").String(), + ClientSecret: sec.Key("client_secret").String(), + Scopes: util.SplitString(sec.Key("scopes").String()), + AuthUrl: sec.Key("auth_url").String(), + TokenUrl: sec.Key("token_url").String(), + ApiUrl: sec.Key("api_url").String(), + Enabled: sec.Key("enabled").MustBool(), + EmailAttributeName: sec.Key("email_attribute_name").String(), + AllowedDomains: util.SplitString(sec.Key("allowed_domains").String()), + HostedDomain: sec.Key("hosted_domain").String(), + AllowSignup: sec.Key("allow_sign_up").MustBool(), + Name: sec.Key("name").MustString(name), + TlsClientCert: sec.Key("tls_client_cert").String(), + TlsClientKey: sec.Key("tls_client_key").String(), + TlsClientCa: sec.Key("tls_client_ca").String(), + TlsSkipVerify: sec.Key("tls_skip_verify_insecure").MustBool(), } if !info.Enabled { @@ -81,7 +86,7 @@ func NewOAuthService() { } if name == "grafananet" { - name = "grafana_com" + name = grafanaCom } setting.OAuthService.OAuthInfos[name] = info @@ -114,6 +119,20 @@ func NewOAuthService() { } } + // GitLab. + if name == "gitlab" { + SocialMap["gitlab"] = &SocialGitlab{ + SocialBase: &SocialBase{ + Config: &config, + log: logger, + }, + allowedDomains: info.AllowedDomains, + apiUrl: info.ApiUrl, + allowSignup: info.AllowSignup, + allowedGroups: util.SplitString(sec.Key("allowed_groups").String()), + } + } + // Google. if name == "google" { SocialMap["google"] = &SocialGoogle{ @@ -138,12 +157,13 @@ func NewOAuthService() { allowedDomains: info.AllowedDomains, apiUrl: info.ApiUrl, allowSignup: info.AllowSignup, + emailAttributeName: info.EmailAttributeName, teamIds: sec.Key("team_ids").Ints(","), allowedOrganizations: util.SplitString(sec.Key("allowed_organizations").String()), } } - if name == "grafana_com" { + if name == grafanaCom { config = oauth2.Config{ ClientID: info.ClientId, ClientSecret: info.ClientSecret, @@ -155,7 +175,7 @@ func NewOAuthService() { Scopes: info.Scopes, } - SocialMap["grafana_com"] = &SocialGrafanaCom{ + SocialMap[grafanaCom] = &SocialGrafanaCom{ SocialBase: &SocialBase{ Config: &config, log: logger, @@ -167,3 +187,26 @@ func NewOAuthService() { } } } + +// GetOAuthProviders returns available oauth providers and if they're enabled or not +var GetOAuthProviders = func(cfg *setting.Cfg) map[string]bool { + result := map[string]bool{} + + if cfg == nil || cfg.Raw == nil { + return result + } + + for _, name := range allOauthes { + if name == "grafananet" { + name = grafanaCom + } + + sec := cfg.Raw.Section("auth." + name) + if sec == nil { + continue + } + result[name] = sec.Key("enabled").MustBool() + } + + return result +} diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index 921996d155d..fd7258b7a0a 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -1,68 +1,72 @@ package tracing import ( + "context" "io" "strings" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/setting" opentracing "github.com/opentracing/opentracing-go" jaegercfg "github.com/uber/jaeger-client-go/config" - ini "gopkg.in/ini.v1" ) -var ( - logger log.Logger = log.New("tracing") -) - -type TracingSettings struct { - Enabled bool - Address string - CustomTags map[string]string - SamplerType string - SamplerParam float64 +func init() { + registry.RegisterService(&TracingService{}) } -func Init(file *ini.File) (io.Closer, error) { - settings := parseSettings(file) - return internalInit(settings) +type TracingService struct { + enabled bool + address string + customTags map[string]string + samplerType string + samplerParam float64 + log log.Logger + closer io.Closer + + Cfg *setting.Cfg `inject:""` } -func parseSettings(file *ini.File) *TracingSettings { - settings := &TracingSettings{} +func (ts *TracingService) Init() error { + ts.log = log.New("tracing") + ts.parseSettings() - var section, err = setting.Cfg.GetSection("tracing.jaeger") + if ts.enabled { + ts.initGlobalTracer() + } + + return nil +} + +func (ts *TracingService) parseSettings() { + var section, err = ts.Cfg.Raw.GetSection("tracing.jaeger") if err != nil { - return settings + return } - settings.Address = section.Key("address").MustString("") - if settings.Address != "" { - settings.Enabled = true + ts.address = section.Key("address").MustString("") + if ts.address != "" { + ts.enabled = true } - settings.CustomTags = splitTagSettings(section.Key("always_included_tag").MustString("")) - settings.SamplerType = section.Key("sampler_type").MustString("") - settings.SamplerParam = section.Key("sampler_param").MustFloat64(1) - - return settings + ts.customTags = splitTagSettings(section.Key("always_included_tag").MustString("")) + ts.samplerType = section.Key("sampler_type").MustString("") + ts.samplerParam = section.Key("sampler_param").MustFloat64(1) } -func internalInit(settings *TracingSettings) (io.Closer, error) { - if !settings.Enabled { - return &nullCloser{}, nil - } - +func (ts *TracingService) initGlobalTracer() error { cfg := jaegercfg.Configuration{ - Disabled: !settings.Enabled, + ServiceName: "grafana", + Disabled: !ts.enabled, Sampler: &jaegercfg.SamplerConfig{ - Type: settings.SamplerType, - Param: settings.SamplerParam, + Type: ts.samplerType, + Param: ts.samplerParam, }, Reporter: &jaegercfg.ReporterConfig{ LogSpans: false, - LocalAgentHostPort: settings.Address, + LocalAgentHostPort: ts.address, }, } @@ -71,18 +75,31 @@ func internalInit(settings *TracingSettings) (io.Closer, error) { options := []jaegercfg.Option{} options = append(options, jaegercfg.Logger(jLogger)) - for tag, value := range settings.CustomTags { + for tag, value := range ts.customTags { options = append(options, jaegercfg.Tag(tag, value)) } - tracer, closer, err := cfg.New("grafana", options...) + tracer, closer, err := cfg.NewTracer(options...) if err != nil { - return nil, err + return err } opentracing.InitGlobalTracer(tracer) - logger.Info("Initializing Jaeger tracer", "address", settings.Address) - return closer, nil + + ts.closer = closer + + return nil +} + +func (ts *TracingService) Run(ctx context.Context) error { + <-ctx.Done() + + if ts.closer != nil { + ts.log.Info("Closing tracing") + ts.closer.Close() + } + + return nil } func splitTagSettings(input string) map[string]string { @@ -110,7 +127,3 @@ func (jlw *jaegerLogWrapper) Error(msg string) { func (jlw *jaegerLogWrapper) Infof(msg string, args ...interface{}) { jlw.logger.Info(msg, args) } - -type nullCloser struct{} - -func (*nullCloser) Close() error { return nil } diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go index 287f4e770ef..e0d9158435e 100644 --- a/pkg/tsdb/cloudwatch/annotation_query.go +++ b/pkg/tsdb/cloudwatch/annotation_query.go @@ -72,7 +72,7 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo MetricName: aws.String(metricName), Dimensions: qd, Statistic: aws.String(s), - Period: aws.Int64(int64(period)), + Period: aws.Int64(period), } resp, err := svc.DescribeAlarmsForMetric(params) if err != nil { @@ -88,7 +88,7 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo MetricName: aws.String(metricName), Dimensions: qd, ExtendedStatistic: aws.String(s), - Period: aws.Int64(int64(period)), + Period: aws.Int64(period), } resp, err := svc.DescribeAlarmsForMetric(params) if err != nil { diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 3879dce4ea6..437457df52a 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -3,6 +3,7 @@ package cloudwatch import ( "context" "errors" + "fmt" "regexp" "sort" "strconv" @@ -13,8 +14,10 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb" + "golang.org/x/sync/errgroup" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" @@ -71,67 +74,105 @@ func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSourc switch queryType { case "metricFindQuery": result, err = e.executeMetricFindQuery(ctx, queryContext) - break case "annotationQuery": result, err = e.executeAnnotationQuery(ctx, queryContext) - break case "timeSeriesQuery": fallthrough default: result, err = e.executeTimeSeriesQuery(ctx, queryContext) - break } return result, err } func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) { - result := &tsdb.Response{ + results := &tsdb.Response{ Results: make(map[string]*tsdb.QueryResult), } + resultChan := make(chan *tsdb.QueryResult, len(queryContext.Queries)) - errCh := make(chan error, 1) - resCh := make(chan *tsdb.QueryResult, 1) + eg, ectx := errgroup.WithContext(ctx) - currentlyExecuting := 0 + getMetricDataQueries := make(map[string]map[string]*CloudWatchQuery) for i, model := range queryContext.Queries { queryType := model.Model.Get("type").MustString() if queryType != "timeSeriesQuery" && queryType != "" { continue } - currentlyExecuting++ - go func(refId string, index int) { - queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext) - currentlyExecuting-- - if err != nil { - errCh <- err - } else { - queryRes.RefId = refId - resCh <- queryRes + + RefId := queryContext.Queries[i].RefId + query, err := parseQuery(queryContext.Queries[i].Model) + if err != nil { + results.Results[RefId] = &tsdb.QueryResult{ + Error: err, } - }(model.RefId, i) + return results, nil + } + query.RefId = RefId + + if query.Id != "" { + if _, ok := getMetricDataQueries[query.Region]; !ok { + getMetricDataQueries[query.Region] = make(map[string]*CloudWatchQuery) + } + getMetricDataQueries[query.Region][query.Id] = query + continue + } + + if query.Id == "" && query.Expression != "" { + results.Results[query.RefId] = &tsdb.QueryResult{ + Error: fmt.Errorf("Invalid query: id should be set if using expression"), + } + return results, nil + } + + eg.Go(func() error { + queryRes, err := e.executeQuery(ectx, query, queryContext) + if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" { + return err + } + if err != nil { + resultChan <- &tsdb.QueryResult{ + RefId: query.RefId, + Error: err, + } + return nil + } + resultChan <- queryRes + return nil + }) } - for currentlyExecuting != 0 { - select { - case res := <-resCh: - result.Results[res.RefId] = res - case err := <-errCh: - return result, err - case <-ctx.Done(): - return result, ctx.Err() + if len(getMetricDataQueries) > 0 { + for region, getMetricDataQuery := range getMetricDataQueries { + q := getMetricDataQuery + eg.Go(func() error { + queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext) + if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" { + return err + } + for _, queryRes := range queryResponses { + if err != nil { + queryRes.Error = err + } + resultChan <- queryRes + } + return nil + }) } } - return result, nil -} - -func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) { - query, err := parseQuery(parameters) - if err != nil { + if err := eg.Wait(); err != nil { return nil, err } + close(resultChan) + for result := range resultChan { + results.Results[result.RefId] = result + } + return results, nil +} + +func (e *CloudWatchExecutor) executeQuery(ctx context.Context, query *CloudWatchQuery, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) { client, err := e.getClient(query.Region) if err != nil { return nil, err @@ -147,6 +188,10 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl return nil, err } + if endTime.Before(startTime) { + return nil, fmt.Errorf("Invalid time range: End time can't be before start time") + } + params := &cloudwatch.GetMetricStatisticsInput{ Namespace: aws.String(query.Namespace), MetricName: aws.String(query.MetricName), @@ -160,7 +205,7 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl params.ExtendedStatistics = query.ExtendedStatistics } - // 1 minutes resolutin metrics is stored for 15 days, 15 * 24 * 60 = 21600 + // 1 minutes resolution metrics is stored for 15 days, 15 * 24 * 60 = 21600 if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) { return nil, errors.New("too long query period") } @@ -199,6 +244,140 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl return queryRes, nil } +func (e *CloudWatchExecutor) executeGetMetricDataQuery(ctx context.Context, region string, queries map[string]*CloudWatchQuery, queryContext *tsdb.TsdbQuery) ([]*tsdb.QueryResult, error) { + queryResponses := make([]*tsdb.QueryResult, 0) + + // validate query + for _, query := range queries { + if !(len(query.Statistics) == 1 && len(query.ExtendedStatistics) == 0) && + !(len(query.Statistics) == 0 && len(query.ExtendedStatistics) == 1) { + return queryResponses, errors.New("Statistics count should be 1") + } + } + + client, err := e.getClient(region) + if err != nil { + return queryResponses, err + } + + startTime, err := queryContext.TimeRange.ParseFrom() + if err != nil { + return queryResponses, err + } + + endTime, err := queryContext.TimeRange.ParseTo() + if err != nil { + return queryResponses, err + } + + params := &cloudwatch.GetMetricDataInput{ + StartTime: aws.Time(startTime), + EndTime: aws.Time(endTime), + ScanBy: aws.String("TimestampAscending"), + } + for _, query := range queries { + // 1 minutes resolution metrics is stored for 15 days, 15 * 24 * 60 = 21600 + if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) { + return queryResponses, errors.New("too long query period") + } + + mdq := &cloudwatch.MetricDataQuery{ + Id: aws.String(query.Id), + ReturnData: aws.Bool(query.ReturnData), + } + if query.Expression != "" { + mdq.Expression = aws.String(query.Expression) + } else { + mdq.MetricStat = &cloudwatch.MetricStat{ + Metric: &cloudwatch.Metric{ + Namespace: aws.String(query.Namespace), + MetricName: aws.String(query.MetricName), + }, + Period: aws.Int64(int64(query.Period)), + } + for _, d := range query.Dimensions { + mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions, + &cloudwatch.Dimension{ + Name: d.Name, + Value: d.Value, + }) + } + if len(query.Statistics) == 1 { + mdq.MetricStat.Stat = query.Statistics[0] + } else { + mdq.MetricStat.Stat = query.ExtendedStatistics[0] + } + } + params.MetricDataQueries = append(params.MetricDataQueries, mdq) + } + + nextToken := "" + mdr := make(map[string]*cloudwatch.MetricDataResult) + for { + if nextToken != "" { + params.NextToken = aws.String(nextToken) + } + resp, err := client.GetMetricDataWithContext(ctx, params) + if err != nil { + return queryResponses, err + } + metrics.M_Aws_CloudWatch_GetMetricData.Add(float64(len(params.MetricDataQueries))) + + for _, r := range resp.MetricDataResults { + if _, ok := mdr[*r.Id]; !ok { + mdr[*r.Id] = r + } else { + mdr[*r.Id].Timestamps = append(mdr[*r.Id].Timestamps, r.Timestamps...) + mdr[*r.Id].Values = append(mdr[*r.Id].Values, r.Values...) + } + } + + if resp.NextToken == nil || *resp.NextToken == "" { + break + } + nextToken = *resp.NextToken + } + + for i, r := range mdr { + if *r.StatusCode != "Complete" { + return queryResponses, fmt.Errorf("Part of query is failed: %s", *r.StatusCode) + } + + queryRes := tsdb.NewQueryResult() + queryRes.RefId = queries[i].RefId + query := queries[*r.Id] + + series := tsdb.TimeSeries{ + Tags: map[string]string{}, + Points: make([]tsdb.TimePoint, 0), + } + for _, d := range query.Dimensions { + series.Tags[*d.Name] = *d.Value + } + s := "" + if len(query.Statistics) == 1 { + s = *query.Statistics[0] + } else { + s = *query.ExtendedStatistics[0] + } + series.Name = formatAlias(query, s, series.Tags) + + for j, t := range r.Timestamps { + expectedTimestamp := r.Timestamps[j].Add(time.Duration(query.Period) * time.Second) + if j > 0 && expectedTimestamp.Before(*t) { + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(expectedTimestamp.Unix()*1000))) + } + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(*r.Values[j]), float64((*t).Unix())*1000)) + } + + queryRes.Series = append(queryRes.Series, &series) + queryRes.Meta = simplejson.New() + queryResponses = append(queryResponses, queryRes) + } + + return queryResponses, nil +} + func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) { var result []*cloudwatch.Dimension @@ -255,6 +434,9 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { return nil, err } + id := model.Get("id").MustString("") + expression := model.Get("expression").MustString("") + dimensions, err := parseDimensions(model) if err != nil { return nil, err @@ -274,7 +456,7 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { } } - period := 300 + var period int if regexp.MustCompile(`^\d+$`).Match([]byte(p)) { period, err = strconv.Atoi(p) if err != nil { @@ -293,6 +475,7 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { alias = "{{metric}}_{{stat}}" } + returnData := model.Get("returnData").MustBool(false) highResolution := model.Get("highResolution").MustBool(false) return &CloudWatchQuery{ @@ -304,11 +487,18 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { ExtendedStatistics: aws.StringSlice(extendedStatistics), Period: period, Alias: alias, + Id: id, + Expression: expression, + ReturnData: returnData, HighResolution: highResolution, }, nil } func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string { + if len(query.Id) > 0 && len(query.Expression) > 0 { + return query.Id + } + data := map[string]string{} data["region"] = query.Region data["namespace"] = query.Namespace @@ -336,6 +526,7 @@ func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]stri func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) { queryRes := tsdb.NewQueryResult() + queryRes.RefId = query.RefId var value float64 for _, s := range append(query.Statistics, query.ExtendedStatistics...) { series := tsdb.TimeSeries{ @@ -384,6 +575,12 @@ func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatch } queryRes.Series = append(queryRes.Series, &series) + queryRes.Meta = simplejson.New() + if len(resp.Datapoints) > 0 && resp.Datapoints[0].Unit != nil { + if unit, ok := cloudwatchUnitMappings[*resp.Datapoints[0].Unit]; ok { + queryRes.Meta.Set("unit", unit) + } + } } return queryRes, nil diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go index 719edba08ba..32b8c910f2b 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch_test.go +++ b/pkg/tsdb/cloudwatch/cloudwatch_test.go @@ -71,6 +71,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(30.0), "p90.00": aws.Float64(40.0), }, + Unit: aws.String("Seconds"), }, }, } @@ -103,6 +104,7 @@ func TestCloudWatch(t *testing.T) { So(queryRes.Series[1].Points[0][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) So(queryRes.Series[2].Points[0][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) So(queryRes.Series[3].Points[0][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) + So(queryRes.Meta.Get("unit").MustString(), ShouldEqual, "s") }) Convey("terminate gap of data points", func() { @@ -118,6 +120,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(30.0), "p90.00": aws.Float64(40.0), }, + Unit: aws.String("Seconds"), }, { Timestamp: aws.Time(timestamp.Add(60 * time.Second)), @@ -127,6 +130,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(40.0), "p90.00": aws.Float64(50.0), }, + Unit: aws.String("Seconds"), }, { Timestamp: aws.Time(timestamp.Add(180 * time.Second)), @@ -136,6 +140,7 @@ func TestCloudWatch(t *testing.T) { "p50.00": aws.Float64(50.0), "p90.00": aws.Float64(60.0), }, + Unit: aws.String("Seconds"), }, }, } diff --git a/pkg/tsdb/cloudwatch/constants.go b/pkg/tsdb/cloudwatch/constants.go new file mode 100644 index 00000000000..23817b1d133 --- /dev/null +++ b/pkg/tsdb/cloudwatch/constants.go @@ -0,0 +1,30 @@ +package cloudwatch + +var cloudwatchUnitMappings = map[string]string{ + "Seconds": "s", + "Microseconds": "µs", + "Milliseconds": "ms", + "Bytes": "bytes", + "Kilobytes": "kbytes", + "Megabytes": "mbytes", + "Gigabytes": "gbytes", + //"Terabytes": "", + "Bits": "bits", + //"Kilobits": "", + //"Megabits": "", + //"Gigabits": "", + //"Terabits": "", + "Percent": "percent", + //"Count": "", + "Bytes/Second": "Bps", + "Kilobytes/Second": "KBs", + "Megabytes/Second": "MBs", + "Gigabytes/Second": "GBs", + //"Terabytes/Second": "", + "Bits/Second": "bps", + "Kilobits/Second": "Kbits", + "Megabits/Second": "Mbits", + "Gigabits/Second": "Gbits", + //"Terabits/Second": "", + //"Count/Second": "", +} diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 06848323fbb..165f8fdbe97 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -23,7 +23,7 @@ type cache struct { expiration *time.Time } -var awsCredentialCache map[string]cache = make(map[string]cache) +var awsCredentialCache = make(map[string]cache) var credentialCacheLock sync.RWMutex func GetCredentials(dsInfo *DatasourceInfo) (*credentials.Credentials, error) { @@ -42,8 +42,7 @@ func GetCredentials(dsInfo *DatasourceInfo) (*credentials.Credentials, error) { accessKeyId := "" secretAccessKey := "" sessionToken := "" - var expiration *time.Time - expiration = nil + var expiration *time.Time = nil if dsInfo.AuthType == "arn" && strings.Index(dsInfo.AssumeRoleArn, "arn:aws:iam:") == 0 { params := &sts.AssumeRoleInput{ RoleArn: aws.String(dsInfo.AssumeRoleArn), diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index c82cff390c3..1a860519f2b 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -35,6 +35,7 @@ type CustomMetricsCache struct { var customMetricsMetricsMap map[string]map[string]map[string]*CustomMetricsCache var customMetricsDimensionsMap map[string]map[string]map[string]*CustomMetricsCache +var regionCache sync.Map func init() { metricsMap = map[string][]string{ @@ -45,7 +46,9 @@ func init() { "AWS/Billing": {"EstimatedCharges"}, "AWS/CloudFront": {"Requests", "BytesDownloaded", "BytesUploaded", "TotalErrorRate", "4xxErrorRate", "5xxErrorRate"}, "AWS/CloudSearch": {"SuccessfulRequests", "SearchableDocuments", "IndexUtilization", "Partitions"}, + "AWS/Connect": {"CallsBreachingConcurrencyQuota", "CallBackNotDialableNumber", "CallRecordingUploadError", "CallsPerInterval", "ConcurrentCalls", "ConcurrentCallsPercentage", "ContactFlowErrors", "ContactFlowFatalErrors", "LongestQueueWaitTime", "MissedCalls", "MisconfiguredPhoneNumbers", "PublicSigningKeyUsage", "QueueCapacityExceededError", "QueueSize", "ThrottledCalls", "ToInstancePacketLossRate"}, "AWS/DMS": {"FreeableMemory", "WriteIOPS", "ReadIOPS", "WriteThroughput", "ReadThroughput", "WriteLatency", "ReadLatency", "SwapUsage", "NetworkTransmitThroughput", "NetworkReceiveThroughput", "FullLoadThroughputBandwidthSource", "FullLoadThroughputBandwidthTarget", "FullLoadThroughputRowsSource", "FullLoadThroughputRowsTarget", "CDCIncomingChanges", "CDCChangesMemorySource", "CDCChangesMemoryTarget", "CDCChangesDiskSource", "CDCChangesDiskTarget", "CDCThroughputBandwidthTarget", "CDCThroughputRowsSource", "CDCThroughputRowsTarget", "CDCLatencySource", "CDCLatencyTarget"}, + "AWS/DX": {"ConnectionState", "ConnectionBpsEgress", "ConnectionBpsIngress", "ConnectionPpsEgress", "ConnectionPpsIngress", "ConnectionCRCErrorCount", "ConnectionLightLevelTx", "ConnectionLightLevelRx"}, "AWS/DynamoDB": {"ConditionalCheckFailedRequests", "ConsumedReadCapacityUnits", "ConsumedWriteCapacityUnits", "OnlineIndexConsumedWriteCapacity", "OnlineIndexPercentageProgress", "OnlineIndexThrottleEvents", "ProvisionedReadCapacityUnits", "ProvisionedWriteCapacityUnits", "ReadThrottleEvents", "ReturnedBytes", "ReturnedItemCount", "ReturnedRecordsCount", "SuccessfulRequestLatency", "SystemErrors", "TimeToLiveDeletedItemCount", "ThrottledRequests", "UserErrors", "WriteThrottleEvents"}, "AWS/EBS": {"VolumeReadBytes", "VolumeWriteBytes", "VolumeReadOps", "VolumeWriteOps", "VolumeTotalReadTime", "VolumeTotalWriteTime", "VolumeIdleTime", "VolumeQueueLength", "VolumeThroughputPercentage", "VolumeConsumedReadWriteOps", "BurstBalance"}, "AWS/EC2": {"CPUCreditUsage", "CPUCreditBalance", "CPUUtilization", "DiskReadOps", "DiskWriteOps", "DiskReadBytes", "DiskWriteBytes", "NetworkIn", "NetworkOut", "NetworkPacketsIn", "NetworkPacketsOut", "StatusCheckFailed", "StatusCheckFailed_Instance", "StatusCheckFailed_System"}, @@ -86,13 +89,14 @@ func init() { "AWS/Kinesis": {"GetRecords.Bytes", "GetRecords.IteratorAge", "GetRecords.IteratorAgeMilliseconds", "GetRecords.Latency", "GetRecords.Records", "GetRecords.Success", "IncomingBytes", "IncomingRecords", "PutRecord.Bytes", "PutRecord.Latency", "PutRecord.Success", "PutRecords.Bytes", "PutRecords.Latency", "PutRecords.Records", "PutRecords.Success", "ReadProvisionedThroughputExceeded", "WriteProvisionedThroughputExceeded", "IteratorAgeMilliseconds", "OutgoingBytes", "OutgoingRecords"}, "AWS/KinesisAnalytics": {"Bytes", "MillisBehindLatest", "Records", "Success"}, "AWS/Lambda": {"Invocations", "Errors", "Duration", "Throttles", "IteratorAge"}, + "AWS/AppSync": {"Latency", "4XXError", "5XXError"}, "AWS/Logs": {"IncomingBytes", "IncomingLogEvents", "ForwardedBytes", "ForwardedLogEvents", "DeliveryErrors", "DeliveryThrottling"}, "AWS/ML": {"PredictCount", "PredictFailureCount"}, "AWS/NATGateway": {"PacketsOutToDestination", "PacketsOutToSource", "PacketsInFromSource", "PacketsInFromDestination", "BytesOutToDestination", "BytesOutToSource", "BytesInFromSource", "BytesInFromDestination", "ErrorPortAllocation", "ActiveConnectionCount", "ConnectionAttemptCount", "ConnectionEstablishedCount", "IdleTimeoutCount", "PacketsDropCount"}, "AWS/NetworkELB": {"ActiveFlowCount", "ConsumedLCUs", "HealthyHostCount", "NewFlowCount", "ProcessedBytes", "TCP_Client_Reset_Count", "TCP_ELB_Reset_Count", "TCP_Target_Reset_Count", "UnHealthyHostCount"}, "AWS/OpsWorks": {"cpu_idle", "cpu_nice", "cpu_system", "cpu_user", "cpu_waitio", "load_1", "load_5", "load_15", "memory_buffers", "memory_cached", "memory_free", "memory_swap", "memory_total", "memory_used", "procs"}, - "AWS/Redshift": {"CPUUtilization", "DatabaseConnections", "HealthStatus", "MaintenanceMode", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "PercentageDiskSpaceUsed", "ReadIOPS", "ReadLatency", "ReadThroughput", "WriteIOPS", "WriteLatency", "WriteThroughput"}, - "AWS/RDS": {"ActiveTransactions", "AuroraBinlogReplicaLag", "AuroraReplicaLag", "AuroraReplicaLagMaximum", "AuroraReplicaLagMinimum", "BinLogDiskUsage", "BlockedTransactions", "BufferCacheHitRatio", "CommitLatency", "CommitThroughput", "BinLogDiskUsage", "CPUCreditBalance", "CPUCreditUsage", "CPUUtilization", "DatabaseConnections", "DDLLatency", "DDLThroughput", "Deadlocks", "DeleteLatency", "DeleteThroughput", "DiskQueueDepth", "DMLLatency", "DMLThroughput", "EngineUptime", "FailedSqlStatements", "FreeableMemory", "FreeLocalStorage", "FreeStorageSpace", "InsertLatency", "InsertThroughput", "LoginFailures", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "NetworkThroughput", "Queries", "ReadIOPS", "ReadLatency", "ReadThroughput", "ReplicaLag", "ResultSetCacheHitRatio", "SelectLatency", "SelectThroughput", "SwapUsage", "TotalConnections", "UpdateLatency", "UpdateThroughput", "VolumeBytesUsed", "VolumeReadIOPS", "VolumeWriteIOPS", "WriteIOPS", "WriteLatency", "WriteThroughput"}, + "AWS/Redshift": {"CPUUtilization", "DatabaseConnections", "HealthStatus", "MaintenanceMode", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "PercentageDiskSpaceUsed", "QueriesCompletedPerSecond", "QueryDuration", "QueryRuntimeBreakdown", "ReadIOPS", "ReadLatency", "ReadThroughput", "WLMQueriesCompletedPerSecond", "WLMQueryDuration", "WLMQueueLength", "WriteIOPS", "WriteLatency", "WriteThroughput"}, + "AWS/RDS": {"ActiveTransactions", "AuroraBinlogReplicaLag", "AuroraReplicaLag", "AuroraReplicaLagMaximum", "AuroraReplicaLagMinimum", "BinLogDiskUsage", "BlockedTransactions", "BufferCacheHitRatio", "BurstBalance", "CommitLatency", "CommitThroughput", "BinLogDiskUsage", "CPUCreditBalance", "CPUCreditUsage", "CPUUtilization", "DatabaseConnections", "DDLLatency", "DDLThroughput", "Deadlocks", "DeleteLatency", "DeleteThroughput", "DiskQueueDepth", "DMLLatency", "DMLThroughput", "EngineUptime", "FailedSqlStatements", "FreeableMemory", "FreeLocalStorage", "FreeStorageSpace", "InsertLatency", "InsertThroughput", "LoginFailures", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "NetworkThroughput", "Queries", "ReadIOPS", "ReadLatency", "ReadThroughput", "ReplicaLag", "ResultSetCacheHitRatio", "SelectLatency", "SelectThroughput", "SwapUsage", "TotalConnections", "UpdateLatency", "UpdateThroughput", "VolumeBytesUsed", "VolumeReadIOPS", "VolumeWriteIOPS", "WriteIOPS", "WriteLatency", "WriteThroughput"}, "AWS/Route53": {"ChildHealthCheckHealthyCount", "HealthCheckStatus", "HealthCheckPercentageHealthy", "ConnectionTime", "SSLHandshakeTime", "TimeToFirstByte"}, "AWS/S3": {"BucketSizeBytes", "NumberOfObjects", "AllRequests", "GetRequests", "PutRequests", "DeleteRequests", "HeadRequests", "PostRequests", "ListRequests", "BytesDownloaded", "BytesUploaded", "4xxErrors", "5xxErrors", "FirstByteLatency", "TotalRequestLatency"}, "AWS/SES": {"Bounce", "Complaint", "Delivery", "Reject", "Send"}, @@ -117,7 +121,9 @@ func init() { "AWS/Billing": {"ServiceName", "LinkedAccount", "Currency"}, "AWS/CloudFront": {"DistributionId", "Region"}, "AWS/CloudSearch": {}, + "AWS/Connect": {"InstanceId", "MetricGroup", "Participant", "QueueName", "Stream Type", "Type of Connection"}, "AWS/DMS": {"ReplicationInstanceIdentifier", "ReplicationTaskIdentifier"}, + "AWS/DX": {"ConnectionId"}, "AWS/DynamoDB": {"TableName", "GlobalSecondaryIndexName", "Operation", "StreamLabel"}, "AWS/EBS": {"VolumeId"}, "AWS/EC2": {"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"}, @@ -135,12 +141,13 @@ func init() { "AWS/Kinesis": {"StreamName", "ShardId"}, "AWS/KinesisAnalytics": {"Flow", "Id", "Application"}, "AWS/Lambda": {"FunctionName", "Resource", "Version", "Alias"}, + "AWS/AppSync": {"GraphQLAPIId"}, "AWS/Logs": {"LogGroupName", "DestinationType", "FilterName"}, "AWS/ML": {"MLModelId", "RequestMode"}, "AWS/NATGateway": {"NatGatewayId"}, "AWS/NetworkELB": {"LoadBalancer", "TargetGroup", "AvailabilityZone"}, "AWS/OpsWorks": {"StackId", "LayerId", "InstanceId"}, - "AWS/Redshift": {"NodeID", "ClusterIdentifier"}, + "AWS/Redshift": {"NodeID", "ClusterIdentifier", "latency", "service class", "wmlid"}, "AWS/RDS": {"DBInstanceIdentifier", "DBClusterIdentifier", "DbClusterIdentifier", "DatabaseClass", "EngineName", "Role"}, "AWS/Route53": {"HealthCheckId", "Region"}, "AWS/S3": {"BucketName", "StorageType", "FilterId"}, @@ -175,25 +182,18 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queryCo switch subType { case "regions": data, err = e.handleGetRegions(ctx, parameters, queryContext) - break case "namespaces": data, err = e.handleGetNamespaces(ctx, parameters, queryContext) - break case "metrics": data, err = e.handleGetMetrics(ctx, parameters, queryContext) - break case "dimension_keys": data, err = e.handleGetDimensions(ctx, parameters, queryContext) - break case "dimension_values": data, err = e.handleGetDimensionValues(ctx, parameters, queryContext) - break case "ebs_volume_ids": data, err = e.handleGetEbsVolumeIds(ctx, parameters, queryContext) - break case "ec2_instance_attribute": data, err = e.handleGetEc2InstanceAttribute(ctx, parameters, queryContext) - break } transformToTable(data, queryResult) @@ -229,23 +229,57 @@ func parseMultiSelectValue(input string) []string { trimValues[i] = strings.TrimSpace(v) } return trimValues - } else { - return []string{trimmedInput} } + return []string{trimmedInput} } // Whenever this list is updated, frontend list should also be updated. // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html func (e *CloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { - regions := []string{ - "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-south-1", "ca-central-1", "cn-north-1", - "eu-central-1", "eu-west-1", "eu-west-2", "sa-east-1", "us-east-1", "us-east-2", "us-gov-west-1", "us-west-1", "us-west-2", + dsInfo := e.getDsInfo("default") + profile := dsInfo.Profile + if cache, ok := regionCache.Load(profile); ok { + if cache2, ok2 := cache.([]suggestData); ok2 { + return cache2, nil + } } + regions := []string{ + "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", + "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2", + "cn-north-1", "cn-northwest-1", "us-gov-east-1", "us-gov-west-1", "us-isob-east-1", "us-iso-east-1", + } + err := e.ensureClientSession("default") + if err != nil { + return nil, err + } + r, err := e.ec2Svc.DescribeRegions(&ec2.DescribeRegionsInput{}) + if err != nil { + // ignore error for backward compatibility + plog.Error("Failed to get regions", "error", err) + } else { + for _, region := range r.Regions { + exists := false + + for _, existingRegion := range regions { + if existingRegion == *region.RegionName { + exists = true + break + } + } + + if !exists { + regions = append(regions, *region.RegionName) + } + } + } + sort.Strings(regions) + result := make([]suggestData, 0) for _, region := range regions { result = append(result, suggestData{Text: region, Value: region}) } + regionCache.Store(profile, result) return result, nil } @@ -261,7 +295,7 @@ func (e *CloudWatchExecutor) handleGetNamespaces(ctx context.Context, parameters keys = append(keys, strings.Split(customNamespaces, ",")...) } - sort.Sort(sort.StringSlice(keys)) + sort.Strings(keys) result := make([]suggestData, 0) for _, key := range keys { @@ -290,7 +324,7 @@ func (e *CloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s return nil, errors.New("Unable to call AWS API") } } - sort.Sort(sort.StringSlice(namespaceMetrics)) + sort.Strings(namespaceMetrics) result := make([]suggestData, 0) for _, name := range namespaceMetrics { @@ -319,7 +353,7 @@ func (e *CloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters return nil, errors.New("Unable to call AWS API") } } - sort.Sort(sort.StringSlice(dimensionValues)) + sort.Strings(dimensionValues) result := make([]suggestData, 0) for _, name := range dimensionValues { @@ -470,6 +504,9 @@ func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, return nil, errors.New("invalid attribute path") } v = v.FieldByName(key) + if !v.IsValid() { + return nil, errors.New("invalid attribute path") + } } if attr, ok := v.Interface().(*string); ok { data = *attr @@ -573,11 +610,7 @@ func getAllMetrics(cwData *DatasourceInfo) (cloudwatch.ListMetricsOutput, error) } return !lastPage }) - if err != nil { - return resp, err - } - - return resp, nil + return resp, err } var metricsCacheLock sync.Mutex diff --git a/pkg/tsdb/cloudwatch/metric_find_query_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go index bf87e7b7d41..34c3379b4df 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query_test.go +++ b/pkg/tsdb/cloudwatch/metric_find_query_test.go @@ -9,20 +9,26 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/bmizerany/assert" + "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) type mockedEc2 struct { ec2iface.EC2API - Resp ec2.DescribeInstancesOutput + Resp ec2.DescribeInstancesOutput + RespRegions ec2.DescribeRegionsOutput } func (m mockedEc2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error { fn(&m.Resp, true) return nil } +func (m mockedEc2) DescribeRegions(in *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) { + return &m.RespRegions, nil +} func TestCloudWatchMetrics(t *testing.T) { @@ -82,6 +88,31 @@ func TestCloudWatchMetrics(t *testing.T) { }) }) + Convey("When calling handleGetRegions", t, func() { + executor := &CloudWatchExecutor{ + ec2Svc: mockedEc2{RespRegions: ec2.DescribeRegionsOutput{ + Regions: []*ec2.Region{ + { + RegionName: aws.String("ap-northeast-2"), + }, + }, + }}, + } + jsonData := simplejson.New() + jsonData.Set("defaultRegion", "default") + executor.DataSource = &models.DataSource{ + JsonData: jsonData, + SecureJsonData: securejsondata.SecureJsonData{}, + } + + result, _ := executor.handleGetRegions(context.Background(), simplejson.New(), &tsdb.TsdbQuery{}) + + Convey("Should return regions", func() { + So(result[0].Text, ShouldEqual, "ap-northeast-1") + So(result[1].Text, ShouldEqual, "ap-northeast-2") + }) + }) + Convey("When calling handleGetEc2InstanceAttribute", t, func() { executor := &CloudWatchExecutor{ ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{ @@ -181,10 +212,7 @@ func TestCloudWatchMetrics(t *testing.T) { } func TestParseMultiSelectValue(t *testing.T) { - - var values []string - - values = parseMultiSelectValue(" i-someInstance ") + values := parseMultiSelectValue(" i-someInstance ") assert.Equal(t, []string{"i-someInstance"}, values) values = parseMultiSelectValue("{i-05}") diff --git a/pkg/tsdb/cloudwatch/types.go b/pkg/tsdb/cloudwatch/types.go index 0737b64686d..1225fb9b31b 100644 --- a/pkg/tsdb/cloudwatch/types.go +++ b/pkg/tsdb/cloudwatch/types.go @@ -5,6 +5,7 @@ import ( ) type CloudWatchQuery struct { + RefId string Region string Namespace string MetricName string @@ -13,5 +14,8 @@ type CloudWatchQuery struct { ExtendedStatistics []*string Period int Alias string + Id string + Expression string + ReturnData bool HighResolution bool } diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go new file mode 100644 index 00000000000..4ebe0db8f89 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -0,0 +1,257 @@ +package es + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/tsdb" + + "github.com/grafana/grafana/pkg/models" + "golang.org/x/net/context/ctxhttp" +) + +const loggerName = "tsdb.elasticsearch.client" + +var ( + clientLog = log.New(loggerName) +) + +var newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { + return ds.GetHttpClient() +} + +// Client represents a client which can interact with elasticsearch api +type Client interface { + GetVersion() int + GetTimeField() string + GetMinInterval(queryInterval string) (time.Duration, error) + ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) + MultiSearch() *MultiSearchRequestBuilder +} + +// NewClient creates a new elasticsearch client +var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb.TimeRange) (Client, error) { + version, err := ds.JsonData.Get("esVersion").Int() + if err != nil { + return nil, fmt.Errorf("elasticsearch version is required, err=%v", err) + } + + timeField, err := ds.JsonData.Get("timeField").String() + if err != nil { + return nil, fmt.Errorf("elasticsearch time field name is required, err=%v", err) + } + + indexInterval := ds.JsonData.Get("interval").MustString() + ip, err := newIndexPattern(indexInterval, ds.Database) + if err != nil { + return nil, err + } + + indices, err := ip.GetIndices(timeRange) + if err != nil { + return nil, err + } + + clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", ")) + + switch version { + case 2, 5, 56: + return &baseClientImpl{ + ctx: ctx, + ds: ds, + version: version, + timeField: timeField, + indices: indices, + timeRange: timeRange, + }, nil + } + + return nil, fmt.Errorf("elasticsearch version=%d is not supported", version) +} + +type baseClientImpl struct { + ctx context.Context + ds *models.DataSource + version int + timeField string + indices []string + timeRange *tsdb.TimeRange +} + +func (c *baseClientImpl) GetVersion() int { + return c.version +} + +func (c *baseClientImpl) GetTimeField() string { + return c.timeField +} + +func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { + return tsdb.GetIntervalFrom(c.ds, simplejson.NewFromAny(map[string]interface{}{ + "interval": queryInterval, + }), 5*time.Second) +} + +func (c *baseClientImpl) getSettings() *simplejson.Json { + return c.ds.JsonData +} + +type multiRequest struct { + header map[string]interface{} + body interface{} + interval tsdb.Interval +} + +func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { + bytes, err := c.encodeBatchRequests(requests) + if err != nil { + return nil, err + } + return c.executeRequest(http.MethodPost, uriPath, bytes) +} + +func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) { + clientLog.Debug("Encoding batch requests to json", "batch requests", len(requests)) + start := time.Now() + + payload := bytes.Buffer{} + for _, r := range requests { + reqHeader, err := json.Marshal(r.header) + if err != nil { + return nil, err + } + payload.WriteString(string(reqHeader) + "\n") + + reqBody, err := json.Marshal(r.body) + if err != nil { + return nil, err + } + + body := string(reqBody) + body = strings.Replace(body, "$__interval_ms", strconv.FormatInt(r.interval.Milliseconds(), 10), -1) + body = strings.Replace(body, "$__interval", r.interval.Text, -1) + + payload.WriteString(body + "\n") + } + + elapsed := time.Since(start) + clientLog.Debug("Encoded batch requests to json", "took", elapsed) + + return payload.Bytes(), nil +} + +func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { + u, _ := url.Parse(c.ds.Url) + u.Path = path.Join(u.Path, uriPath) + + var req *http.Request + var err error + if method == http.MethodPost { + req, err = http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(body)) + } else { + req, err = http.NewRequest(http.MethodGet, u.String(), nil) + } + if err != nil { + return nil, err + } + + clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + + req.Header.Set("User-Agent", "Grafana") + req.Header.Set("Content-Type", "application/json") + + if c.ds.BasicAuth { + clientLog.Debug("Request configured to use basic authentication") + req.SetBasicAuth(c.ds.BasicAuthUser, c.ds.BasicAuthPassword) + } + + if !c.ds.BasicAuth && c.ds.User != "" { + clientLog.Debug("Request configured to use basic authentication") + req.SetBasicAuth(c.ds.User, c.ds.Password) + } + + httpClient, err := newDatasourceHttpClient(c.ds) + if err != nil { + return nil, err + } + + start := time.Now() + defer func() { + elapsed := time.Since(start) + clientLog.Debug("Executed request", "took", elapsed) + }() + return ctxhttp.Do(c.ctx, httpClient, req) +} + +func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) { + clientLog.Debug("Executing multisearch", "search requests", len(r.Requests)) + + multiRequests := c.createMultiSearchRequests(r.Requests) + res, err := c.executeBatchRequest("_msearch", multiRequests) + if err != nil { + return nil, err + } + + clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) + + start := time.Now() + clientLog.Debug("Decoding multisearch json response") + + var msr MultiSearchResponse + defer res.Body.Close() + dec := json.NewDecoder(res.Body) + err = dec.Decode(&msr) + if err != nil { + return nil, err + } + + elapsed := time.Since(start) + clientLog.Debug("Decoded multisearch json response", "took", elapsed) + + msr.Status = res.StatusCode + + return &msr, nil +} + +func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { + multiRequests := []*multiRequest{} + + for _, searchReq := range searchRequests { + mr := multiRequest{ + header: map[string]interface{}{ + "search_type": "query_then_fetch", + "ignore_unavailable": true, + "index": strings.Join(c.indices, ","), + }, + body: searchReq, + interval: searchReq.Interval, + } + + if c.version == 2 { + mr.header["search_type"] = "count" + } + + if c.version >= 56 { + maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256) + mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests + } + + multiRequests = append(multiRequests, &mr) + } + + return multiRequests +} + +func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { + return NewMultiSearchRequestBuilder(c.GetVersion()) +} diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go new file mode 100644 index 00000000000..540a999688a --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -0,0 +1,304 @@ +package es + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + + "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestClient(t *testing.T) { + Convey("Test elasticsearch client", t, func() { + Convey("NewClient", func() { + Convey("When no version set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(make(map[string]interface{})), + } + + _, err := NewClient(context.Background(), ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When no time field name set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + }), + } + + _, err := NewClient(context.Background(), ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When unsupported version set should return error", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 6, + "timeField": "@timestamp", + }), + } + + _, err := NewClient(context.Background(), ds, nil) + So(err, ShouldNotBeNil) + }) + + Convey("When version 2 should return v2 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 2, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 2) + }) + + Convey("When version 5 should return v5 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 5) + }) + + Convey("When version 56 should return v5.6 client", func() { + ds := &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + "timeField": "@timestamp", + }), + } + + c, err := NewClient(context.Background(), ds, nil) + So(err, ShouldBeNil) + So(c.GetVersion(), ShouldEqual, 56) + }) + }) + + Convey("Given a fake http client", func() { + var responseBuffer *bytes.Buffer + var req *http.Request + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + req = r + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("Failed to read response body, err=%v", err) + } + responseBuffer = bytes.NewBuffer(buf) + })) + + currentNewDatasourceHttpClient := newDatasourceHttpClient + + newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { + return ts.Client(), nil + } + + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + timeRange := tsdb.NewTimeRange(fromStr, toStr) + + Convey("and a v2.x client", func() { + ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 2, + "timeField": "@timestamp", + "interval": "Daily", + }), + } + + c, err := NewClient(context.Background(), &ds, timeRange) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "count") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) + }) + }) + }) + + Convey("and a v5.x client", func() { + ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 5, + "maxConcurrentShardRequests": 100, + "timeField": "@timestamp", + "interval": "Daily", + }), + } + + c, err := NewClient(context.Background(), &ds, timeRange) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(10), ShouldEqual, 10) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) + }) + }) + }) + + Convey("and a v5.6 client", func() { + ds := models.DataSource{ + Database: "[metrics-]YYYY.MM.DD", + Url: ts.URL, + JsonData: simplejson.NewFromAny(map[string]interface{}{ + "esVersion": 56, + "maxConcurrentShardRequests": 100, + "timeField": "@timestamp", + "interval": "Daily", + }), + } + + c, err := NewClient(context.Background(), &ds, timeRange) + So(err, ShouldBeNil) + So(c, ShouldNotBeNil) + + Convey("When executing multi search", func() { + ms, err := createMultisearchForTest(c) + So(err, ShouldBeNil) + c.ExecuteMultisearch(ms) + + Convey("Should send correct request and payload", func() { + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.URL.Path, ShouldEqual, "/_msearch") + + So(responseBuffer, ShouldNotBeNil) + + headerBytes, err := responseBuffer.ReadBytes('\n') + So(err, ShouldBeNil) + bodyBytes := responseBuffer.Bytes() + + jHeader, err := simplejson.NewJson(headerBytes) + So(err, ShouldBeNil) + + jBody, err := simplejson.NewJson(bodyBytes) + So(err, ShouldBeNil) + + fmt.Println("body", string(headerBytes)) + + So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") + So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) + So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") + So(jHeader.Get("max_concurrent_shard_requests").MustInt(), ShouldEqual, 100) + + Convey("and replace $__interval variable", func() { + So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") + }) + + Convey("and replace $__interval_ms variable", func() { + So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") + }) + }) + }) + }) + + Reset(func() { + newDatasourceHttpClient = currentNewDatasourceHttpClient + }) + }) + }) +} + +func createMultisearchForTest(c Client) (*MultiSearchRequest, error) { + msb := c.MultiSearch() + s := msb.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + s.Agg().DateHistogram("2", "@timestamp", func(a *DateHistogramAgg, ab AggBuilder) { + a.Interval = "$__interval" + + ab.Metric("1", "avg", "@hostname", func(a *MetricAggregation) { + a.Settings["script"] = "$__interval_ms*@hostname" + }) + }) + return msb.Build() +} diff --git a/pkg/tsdb/elasticsearch/client/index_pattern.go b/pkg/tsdb/elasticsearch/client/index_pattern.go new file mode 100644 index 00000000000..952b5c4f806 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/index_pattern.go @@ -0,0 +1,331 @@ +package es + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/grafana/grafana/pkg/tsdb" +) + +const ( + noInterval = "" + intervalHourly = "hourly" + intervalDaily = "daily" + intervalWeekly = "weekly" + intervalMonthly = "monthly" + intervalYearly = "yearly" +) + +type indexPattern interface { + GetIndices(timeRange *tsdb.TimeRange) ([]string, error) +} + +var newIndexPattern = func(interval string, pattern string) (indexPattern, error) { + if interval == noInterval { + return &staticIndexPattern{indexName: pattern}, nil + } + + return newDynamicIndexPattern(interval, pattern) +} + +type staticIndexPattern struct { + indexName string +} + +func (ip *staticIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) { + return []string{ip.indexName}, nil +} + +type intervalGenerator interface { + Generate(from, to time.Time) []time.Time +} + +type dynamicIndexPattern struct { + interval string + pattern string + intervalGenerator intervalGenerator +} + +func newDynamicIndexPattern(interval, pattern string) (*dynamicIndexPattern, error) { + var generator intervalGenerator + + switch strings.ToLower(interval) { + case intervalHourly: + generator = &hourlyInterval{} + case intervalDaily: + generator = &dailyInterval{} + case intervalWeekly: + generator = &weeklyInterval{} + case intervalMonthly: + generator = &monthlyInterval{} + case intervalYearly: + generator = &yearlyInterval{} + default: + return nil, fmt.Errorf("unsupported interval '%s'", interval) + } + + return &dynamicIndexPattern{ + interval: interval, + pattern: pattern, + intervalGenerator: generator, + }, nil +} + +func (ip *dynamicIndexPattern) GetIndices(timeRange *tsdb.TimeRange) ([]string, error) { + from := timeRange.GetFromAsTimeUTC() + to := timeRange.GetToAsTimeUTC() + intervals := ip.intervalGenerator.Generate(from, to) + indices := make([]string, 0) + + for _, t := range intervals { + indices = append(indices, formatDate(t, ip.pattern)) + } + + return indices, nil +} + +type hourlyInterval struct{} + +func (i *hourlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), from.Hour(), 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), to.Hour(), 0, 0, 0, time.UTC) + + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(time.Hour) + intervals = append(intervals, start) + } + + return intervals +} + +type dailyInterval struct{} + +func (i *dailyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC) + + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + intervals = append(intervals, start) + } + + return intervals +} + +type weeklyInterval struct{} + +func (i *weeklyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), to.Day(), 0, 0, 0, 0, time.UTC) + + for start.Weekday() != time.Monday { + start = start.Add(-24 * time.Hour) + } + + for end.Weekday() != time.Monday { + end = end.Add(-24 * time.Hour) + } + + year, week := start.ISOWeek() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextYear, nextWeek := start.ISOWeek() + if nextYear != year || nextWeek != week { + intervals = append(intervals, start) + } + year = nextYear + week = nextWeek + } + + return intervals +} + +type monthlyInterval struct{} + +func (i *monthlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC) + + month := start.Month() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextMonth := start.Month() + if nextMonth != month { + intervals = append(intervals, start) + } + month = nextMonth + } + + return intervals +} + +type yearlyInterval struct{} + +func (i *yearlyInterval) Generate(from, to time.Time) []time.Time { + intervals := []time.Time{} + start := time.Date(from.Year(), 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(to.Year(), 1, 1, 0, 0, 0, 0, time.UTC) + + year := start.Year() + intervals = append(intervals, start) + + for start.Before(end) { + start = start.Add(24 * time.Hour) + nextYear := start.Year() + if nextYear != year { + intervals = append(intervals, start) + } + year = nextYear + } + + return intervals +} + +var datePatternRegex = regexp.MustCompile("(LT|LL?L?L?|l{1,4}|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|Q)") + +var datePatternReplacements = map[string]string{ + "M": "1", // stdNumMonth 1 2 ... 11 12 + "MM": "01", // stdZeroMonth 01 02 ... 11 12 + "MMM": "Jan", // stdMonth Jan Feb ... Nov Dec + "MMMM": "January", // stdLongMonth January February ... November December + "D": "2", // stdDay 1 2 ... 30 30 + "DD": "02", // stdZeroDay 01 02 ... 30 31 + "DDD": "", // Day of the year 1 2 ... 364 365 + "DDDD": "", // Day of the year 001 002 ... 364 365 @todo**** + "d": "", // Numeric representation of day of the week 0 1 ... 5 6 + "dd": "Mon", // ***Su Mo ... Fr Sa @todo + "ddd": "Mon", // Sun Mon ... Fri Sat + "dddd": "Monday", // stdLongWeekDay Sunday Monday ... Friday Saturday + "e": "", // Numeric representation of day of the week 0 1 ... 5 6 @todo + "E": "", // ISO-8601 numeric representation of the day of the week (added in PHP 5.1.0) 1 2 ... 6 7 @todo + "w": "", // 1 2 ... 52 53 + "ww": "", // ***01 02 ... 52 53 @todo + "W": "", // 1 2 ... 52 53 + "WW": "", // ***01 02 ... 52 53 @todo + "YY": "06", // stdYear 70 71 ... 29 30 + "YYYY": "2006", // stdLongYear 1970 1971 ... 2029 2030 + "gg": "", // ISO-8601 year number 70 71 ... 29 30 + "gggg": "", // ***1970 1971 ... 2029 2030 + "GG": "", //70 71 ... 29 30 + "GGGG": "", // ***1970 1971 ... 2029 2030 + "Q": "", // 1, 2, 3, 4 + "A": "PM", // stdPM AM PM + "a": "pm", // stdpm am pm + "H": "", // stdHour 0 1 ... 22 23 + "HH": "15", // 00 01 ... 22 23 + "h": "3", // stdHour12 1 2 ... 11 12 + "hh": "03", // stdZeroHour12 01 02 ... 11 12 + "m": "4", // stdZeroMinute 0 1 ... 58 59 + "mm": "04", // stdZeroMinute 00 01 ... 58 59 + "s": "5", // stdSecond 0 1 ... 58 59 + "ss": "05", // stdZeroSecond ***00 01 ... 58 59 + "z": "MST", //EST CST ... MST PST + "zz": "MST", //EST CST ... MST PST + "Z": "Z07:00", // stdNumColonTZ -07:00 -06:00 ... +06:00 +07:00 + "ZZ": "-0700", // stdNumTZ -0700 -0600 ... +0600 +0700 + "X": "", // Seconds since unix epoch 1360013296 + "LT": "3:04 PM", // 8:30 PM + "L": "01/02/2006", //09/04/1986 + "l": "1/2/2006", //9/4/1986 + "ll": "Jan 2 2006", //Sep 4 1986 + "lll": "Jan 2 2006 3:04 PM", //Sep 4 1986 8:30 PM + "llll": "Mon, Jan 2 2006 3:04 PM", //Thu, Sep 4 1986 8:30 PM +} + +func formatDate(t time.Time, pattern string) string { + var datePattern string + base := "" + ltr := false + + if strings.HasPrefix(pattern, "[") { + parts := strings.Split(strings.TrimLeft(pattern, "["), "]") + base = parts[0] + if len(parts) == 2 { + datePattern = parts[1] + } else { + datePattern = base + base = "" + } + ltr = true + } else if strings.HasSuffix(pattern, "]") { + parts := strings.Split(strings.TrimRight(pattern, "]"), "[") + datePattern = parts[0] + if len(parts) == 2 { + base = parts[1] + } else { + base = "" + } + ltr = false + } + + formatted := t.Format(patternToLayout(datePattern)) + + if strings.Contains(formatted, "", fmt.Sprintf("%d", isoYear), -1) + formatted = strings.Replace(formatted, "", isoYearShort, -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", isoWeek), -1) + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.Unix()), -1) + + day := t.Weekday() + dayOfWeekIso := int(day) + if day == time.Sunday { + dayOfWeekIso = 7 + } + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", day), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", dayOfWeekIso), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.YearDay()), -1) + + quarter := 4 + + switch t.Month() { + case time.January, time.February, time.March: + quarter = 1 + case time.April, time.May, time.June: + quarter = 2 + case time.July, time.August, time.September: + quarter = 3 + } + + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", quarter), -1) + formatted = strings.Replace(formatted, "", fmt.Sprintf("%d", t.Hour()), -1) + } + + if ltr { + return base + formatted + } + + return formatted + base +} + +func patternToLayout(pattern string) string { + var match [][]string + if match = datePatternRegex.FindAllStringSubmatch(pattern, -1); match == nil { + return pattern + } + + for i := range match { + if replace, ok := datePatternReplacements[match[i][0]]; ok { + pattern = strings.Replace(pattern, match[i][0], replace, 1) + } + } + + return pattern +} diff --git a/pkg/tsdb/elasticsearch/client/index_pattern_test.go b/pkg/tsdb/elasticsearch/client/index_pattern_test.go new file mode 100644 index 00000000000..ca20b39d532 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/index_pattern_test.go @@ -0,0 +1,269 @@ +package es + +import ( + "fmt" + "testing" + "time" + + "github.com/grafana/grafana/pkg/tsdb" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestIndexPattern(t *testing.T) { + Convey("Static index patterns", t, func() { + indexPatternScenario(noInterval, "data-*", nil, func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-*") + }) + + indexPatternScenario(noInterval, "es-index-name", nil, func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "es-index-name") + }) + }) + + Convey("Dynamic index patterns", t, func() { + from := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond)) + to := fmt.Sprintf("%d", time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC).UnixNano()/int64(time.Millisecond)) + + indexPatternScenario(intervalHourly, "[data-]YYYY.MM.DD.HH", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05.15.17") + }) + + indexPatternScenario(intervalHourly, "YYYY.MM.DD.HH[-data]", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "2018.05.15.17-data") + }) + + indexPatternScenario(intervalDaily, "[data-]YYYY.MM.DD", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05.15") + }) + + indexPatternScenario(intervalDaily, "YYYY.MM.DD[-data]", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "2018.05.15-data") + }) + + indexPatternScenario(intervalWeekly, "[data-]GGGG.WW", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.20") + }) + + indexPatternScenario(intervalWeekly, "GGGG.WW[-data]", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "2018.20-data") + }) + + indexPatternScenario(intervalMonthly, "[data-]YYYY.MM", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018.05") + }) + + indexPatternScenario(intervalMonthly, "YYYY.MM[-data]", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "2018.05-data") + }) + + indexPatternScenario(intervalYearly, "[data-]YYYY", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "data-2018") + }) + + indexPatternScenario(intervalYearly, "YYYY[-data]", tsdb.NewTimeRange(from, to), func(indices []string) { + So(indices, ShouldHaveLength, 1) + So(indices[0], ShouldEqual, "2018-data") + }) + }) + + Convey("Hourly interval", t, func() { + Convey("Should return 1 interval", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 intervals", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 0, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 10 intervals", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 8, 6, 0, 0, time.UTC) + intervals := (&hourlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 10) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 23, 0, 0, 0, time.UTC)) + So(intervals[4], ShouldEqual, time.Date(2018, 1, 2, 3, 0, 0, 0, time.UTC)) + So(intervals[9], ShouldEqual, time.Date(2018, 1, 2, 8, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Daily interval", t, func() { + Convey("Should return 1 day", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 days", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 2, 0, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 32 days", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 8, 6, 0, 0, time.UTC) + intervals := (&dailyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 32) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[30], ShouldEqual, time.Date(2018, 1, 31, 0, 0, 0, 0, time.UTC)) + So(intervals[31], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Weekly interval", t, func() { + Convey("Should return 1 week (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 1 week (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 weeks (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 10, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 8, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 weeks (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 8, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2017, 1, 2, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 3 weeks (1)", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 21, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 3) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 1, 8, 0, 0, 0, 0, time.UTC)) + So(intervals[2], ShouldEqual, time.Date(2018, 1, 15, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 3 weeks (2)", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2017, 1, 9, 23, 6, 0, 0, time.UTC) + intervals := (&weeklyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 3) + So(intervals[0], ShouldEqual, time.Date(2016, 12, 26, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2017, 1, 2, 0, 0, 0, 0, time.UTC)) + So(intervals[2], ShouldEqual, time.Date(2017, 1, 9, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Monthly interval", t, func() { + Convey("Should return 1 month", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 1, 1, 23, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 months", func() { + from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 2, 0, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 14 months", func() { + from := time.Date(2017, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 8, 6, 0, 0, time.UTC) + intervals := (&monthlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 14) + So(intervals[0], ShouldEqual, time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[13], ShouldEqual, time.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC)) + }) + }) + + Convey("Yearly interval", t, func() { + Convey("Should return 1 year (hour diff)", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 2, 1, 23, 6, 0, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 1 year (month diff)", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 1) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 2 years", func() { + from := time.Date(2018, 2, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2019, 1, 1, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 2) + So(intervals[0], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[1], ShouldEqual, time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + + Convey("Should return 5 years", func() { + from := time.Date(2014, 1, 1, 23, 1, 1, 0, time.UTC) + to := time.Date(2018, 11, 1, 23, 59, 59, 0, time.UTC) + intervals := (&yearlyInterval{}).Generate(from, to) + So(intervals, ShouldHaveLength, 5) + So(intervals[0], ShouldEqual, time.Date(2014, 1, 1, 0, 0, 0, 0, time.UTC)) + So(intervals[4], ShouldEqual, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)) + }) + }) +} + +func indexPatternScenario(interval string, pattern string, timeRange *tsdb.TimeRange, fn func(indices []string)) { + Convey(fmt.Sprintf("Index pattern (interval=%s, index=%s", interval, pattern), func() { + ip, err := newIndexPattern(interval, pattern) + So(err, ShouldBeNil) + So(ip, ShouldNotBeNil) + indices, err := ip.GetIndices(timeRange) + So(err, ShouldBeNil) + fn(indices) + }) +} diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go new file mode 100644 index 00000000000..a0d257d01a6 --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -0,0 +1,311 @@ +package es + +import ( + "encoding/json" + + "github.com/grafana/grafana/pkg/tsdb" +) + +// SearchRequest represents a search request +type SearchRequest struct { + Index string + Interval tsdb.Interval + Size int + Sort map[string]interface{} + Query *Query + Aggs AggArray + CustomProps map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the request. +func (r *SearchRequest) MarshalJSON() ([]byte, error) { + root := make(map[string]interface{}) + + root["size"] = r.Size + if len(r.Sort) > 0 { + root["sort"] = r.Sort + } + + for key, value := range r.CustomProps { + root[key] = value + } + + root["query"] = r.Query + + if len(r.Aggs) > 0 { + root["aggs"] = r.Aggs + } + + return json.Marshal(root) +} + +// SearchResponseHits represents search response hits +type SearchResponseHits struct { + Hits []map[string]interface{} + Total int64 +} + +// SearchResponse represents a search response +type SearchResponse struct { + Error map[string]interface{} `json:"error"` + Aggregations map[string]interface{} `json:"aggregations"` + Hits *SearchResponseHits `json:"hits"` +} + +// func (r *Response) getErrMsg() string { +// var msg bytes.Buffer +// errJson := simplejson.NewFromAny(r.Err) +// errType, err := errJson.Get("type").String() +// if err == nil { +// msg.WriteString(fmt.Sprintf("type:%s", errType)) +// } + +// reason, err := errJson.Get("type").String() +// if err == nil { +// msg.WriteString(fmt.Sprintf("reason:%s", reason)) +// } +// return msg.String() +// } + +// MultiSearchRequest represents a multi search request +type MultiSearchRequest struct { + Requests []*SearchRequest +} + +// MultiSearchResponse represents a multi search response +type MultiSearchResponse struct { + Status int `json:"status,omitempty"` + Responses []*SearchResponse `json:"responses"` +} + +// Query represents a query +type Query struct { + Bool *BoolQuery `json:"bool"` +} + +// BoolQuery represents a bool query +type BoolQuery struct { + Filters []Filter +} + +// NewBoolQuery create a new bool query +func NewBoolQuery() *BoolQuery { + return &BoolQuery{Filters: make([]Filter, 0)} +} + +// MarshalJSON returns the JSON encoding of the boolean query. +func (q *BoolQuery) MarshalJSON() ([]byte, error) { + root := make(map[string]interface{}) + + if len(q.Filters) > 0 { + if len(q.Filters) == 1 { + root["filter"] = q.Filters[0] + } else { + root["filter"] = q.Filters + } + } + return json.Marshal(root) +} + +// Filter represents a search filter +type Filter interface{} + +// QueryStringFilter represents a query string search filter +type QueryStringFilter struct { + Filter + Query string + AnalyzeWildcard bool +} + +// MarshalJSON returns the JSON encoding of the query string filter. +func (f *QueryStringFilter) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "query_string": map[string]interface{}{ + "query": f.Query, + "analyze_wildcard": f.AnalyzeWildcard, + }, + } + + return json.Marshal(root) +} + +// RangeFilter represents a range search filter +type RangeFilter struct { + Filter + Key string + Gte string + Lte string + Format string +} + +// DateFormatEpochMS represents a date format of epoch milliseconds (epoch_millis) +const DateFormatEpochMS = "epoch_millis" + +// MarshalJSON returns the JSON encoding of the query string filter. +func (f *RangeFilter) MarshalJSON() ([]byte, error) { + root := map[string]map[string]map[string]interface{}{ + "range": { + f.Key: { + "lte": f.Lte, + "gte": f.Gte, + }, + }, + } + + if f.Format != "" { + root["range"][f.Key]["format"] = f.Format + } + + return json.Marshal(root) +} + +// Aggregation represents an aggregation +type Aggregation interface{} + +// Agg represents a key and aggregation +type Agg struct { + Key string + Aggregation *aggContainer +} + +// MarshalJSON returns the JSON encoding of the agg +func (a *Agg) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + a.Key: a.Aggregation, + } + + return json.Marshal(root) +} + +// AggArray represents a collection of key/aggregation pairs +type AggArray []*Agg + +// MarshalJSON returns the JSON encoding of the agg +func (a AggArray) MarshalJSON() ([]byte, error) { + aggsMap := make(map[string]Aggregation) + + for _, subAgg := range a { + aggsMap[subAgg.Key] = subAgg.Aggregation + } + + return json.Marshal(aggsMap) +} + +type aggContainer struct { + Type string + Aggregation Aggregation + Aggs AggArray +} + +// MarshalJSON returns the JSON encoding of the aggregation container +func (a *aggContainer) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + a.Type: a.Aggregation, + } + + if len(a.Aggs) > 0 { + root["aggs"] = a.Aggs + } + + return json.Marshal(root) +} + +type aggDef struct { + key string + aggregation *aggContainer + builders []AggBuilder +} + +func newAggDef(key string, aggregation *aggContainer) *aggDef { + return &aggDef{ + key: key, + aggregation: aggregation, + builders: make([]AggBuilder, 0), + } +} + +// HistogramAgg represents a histogram aggregation +type HistogramAgg struct { + Interval int `json:"interval,omitempty"` + Field string `json:"field"` + MinDocCount int `json:"min_doc_count"` + Missing *int `json:"missing,omitempty"` +} + +// DateHistogramAgg represents a date histogram aggregation +type DateHistogramAgg struct { + Field string `json:"field"` + Interval string `json:"interval,omitempty"` + MinDocCount int `json:"min_doc_count"` + Missing *string `json:"missing,omitempty"` + ExtendedBounds *ExtendedBounds `json:"extended_bounds"` + Format string `json:"format"` +} + +// FiltersAggregation represents a filters aggregation +type FiltersAggregation struct { + Filters map[string]interface{} `json:"filters"` +} + +// TermsAggregation represents a terms aggregation +type TermsAggregation struct { + Field string `json:"field"` + Size int `json:"size"` + Order map[string]interface{} `json:"order"` + MinDocCount *int `json:"min_doc_count,omitempty"` + Missing *string `json:"missing,omitempty"` +} + +// ExtendedBounds represents extended bounds +type ExtendedBounds struct { + Min string `json:"min"` + Max string `json:"max"` +} + +// GeoHashGridAggregation represents a geo hash grid aggregation +type GeoHashGridAggregation struct { + Field string `json:"field"` + Precision int `json:"precision"` +} + +// MetricAggregation represents a metric aggregation +type MetricAggregation struct { + Field string + Settings map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the metric aggregation +func (a *MetricAggregation) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "field": a.Field, + } + + for k, v := range a.Settings { + if k != "" && v != nil { + root[k] = v + } + } + + return json.Marshal(root) +} + +// PipelineAggregation represents a metric aggregation +type PipelineAggregation struct { + BucketPath string + Settings map[string]interface{} +} + +// MarshalJSON returns the JSON encoding of the pipeline aggregation +func (a *PipelineAggregation) MarshalJSON() ([]byte, error) { + root := map[string]interface{}{ + "buckets_path": a.BucketPath, + } + + for k, v := range a.Settings { + if k != "" && v != nil { + root[k] = v + } + } + + return json.Marshal(root) +} diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go new file mode 100644 index 00000000000..d89a98cbadb --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -0,0 +1,458 @@ +package es + +import ( + "strings" + + "github.com/grafana/grafana/pkg/tsdb" +) + +// SearchRequestBuilder represents a builder which can build a search request +type SearchRequestBuilder struct { + version int + interval tsdb.Interval + index string + size int + sort map[string]interface{} + queryBuilder *QueryBuilder + aggBuilders []AggBuilder + customProps map[string]interface{} +} + +// NewSearchRequestBuilder create a new search request builder +func NewSearchRequestBuilder(version int, interval tsdb.Interval) *SearchRequestBuilder { + builder := &SearchRequestBuilder{ + version: version, + interval: interval, + sort: make(map[string]interface{}), + customProps: make(map[string]interface{}), + aggBuilders: make([]AggBuilder, 0), + } + return builder +} + +// Build builds and return a search request +func (b *SearchRequestBuilder) Build() (*SearchRequest, error) { + sr := SearchRequest{ + Index: b.index, + Interval: b.interval, + Size: b.size, + Sort: b.sort, + CustomProps: b.customProps, + } + + if b.queryBuilder != nil { + q, err := b.queryBuilder.Build() + if err != nil { + return nil, err + } + sr.Query = q + } + + if len(b.aggBuilders) > 0 { + sr.Aggs = make(AggArray, 0) + + for _, ab := range b.aggBuilders { + aggArray, err := ab.Build() + if err != nil { + return nil, err + } + sr.Aggs = append(sr.Aggs, aggArray...) + } + } + + return &sr, nil +} + +// Size sets the size of the search request +func (b *SearchRequestBuilder) Size(size int) *SearchRequestBuilder { + b.size = size + return b +} + +// SortDesc adds a sort to the search request +func (b *SearchRequestBuilder) SortDesc(field, unmappedType string) *SearchRequestBuilder { + props := map[string]string{ + "order": "desc", + } + + if unmappedType != "" { + props["unmapped_type"] = unmappedType + } + + b.sort[field] = props + + return b +} + +// AddDocValueField adds a doc value field to the search request +func (b *SearchRequestBuilder) AddDocValueField(field string) *SearchRequestBuilder { + // fields field not supported on version >= 5 + if b.version < 5 { + b.customProps["fields"] = []string{"*", "_source"} + } + + b.customProps["script_fields"] = make(map[string]interface{}) + + if b.version < 5 { + b.customProps["fielddata_fields"] = []string{field} + } else { + b.customProps["docvalue_fields"] = []string{field} + } + + return b +} + +// Query creates and return a query builder +func (b *SearchRequestBuilder) Query() *QueryBuilder { + if b.queryBuilder == nil { + b.queryBuilder = NewQueryBuilder() + } + return b.queryBuilder +} + +// Agg initiate and returns a new aggregation builder +func (b *SearchRequestBuilder) Agg() AggBuilder { + aggBuilder := newAggBuilder(b.version) + b.aggBuilders = append(b.aggBuilders, aggBuilder) + return aggBuilder +} + +// MultiSearchRequestBuilder represents a builder which can build a multi search request +type MultiSearchRequestBuilder struct { + version int + requestBuilders []*SearchRequestBuilder +} + +// NewMultiSearchRequestBuilder creates a new multi search request builder +func NewMultiSearchRequestBuilder(version int) *MultiSearchRequestBuilder { + return &MultiSearchRequestBuilder{ + version: version, + } +} + +// Search initiates and returns a new search request builder +func (m *MultiSearchRequestBuilder) Search(interval tsdb.Interval) *SearchRequestBuilder { + b := NewSearchRequestBuilder(m.version, interval) + m.requestBuilders = append(m.requestBuilders, b) + return b +} + +// Build builds and return a multi search request +func (m *MultiSearchRequestBuilder) Build() (*MultiSearchRequest, error) { + requests := []*SearchRequest{} + for _, sb := range m.requestBuilders { + searchRequest, err := sb.Build() + if err != nil { + return nil, err + } + requests = append(requests, searchRequest) + } + + return &MultiSearchRequest{ + Requests: requests, + }, nil +} + +// QueryBuilder represents a query builder +type QueryBuilder struct { + boolQueryBuilder *BoolQueryBuilder +} + +// NewQueryBuilder create a new query builder +func NewQueryBuilder() *QueryBuilder { + return &QueryBuilder{} +} + +// Build builds and return a query builder +func (b *QueryBuilder) Build() (*Query, error) { + q := Query{} + + if b.boolQueryBuilder != nil { + b, err := b.boolQueryBuilder.Build() + if err != nil { + return nil, err + } + q.Bool = b + } + + return &q, nil +} + +// Bool creates and return a query builder +func (b *QueryBuilder) Bool() *BoolQueryBuilder { + if b.boolQueryBuilder == nil { + b.boolQueryBuilder = NewBoolQueryBuilder() + } + return b.boolQueryBuilder +} + +// BoolQueryBuilder represents a bool query builder +type BoolQueryBuilder struct { + filterQueryBuilder *FilterQueryBuilder +} + +// NewBoolQueryBuilder create a new bool query builder +func NewBoolQueryBuilder() *BoolQueryBuilder { + return &BoolQueryBuilder{} +} + +// Filter creates and return a filter query builder +func (b *BoolQueryBuilder) Filter() *FilterQueryBuilder { + if b.filterQueryBuilder == nil { + b.filterQueryBuilder = NewFilterQueryBuilder() + } + return b.filterQueryBuilder +} + +// Build builds and return a bool query builder +func (b *BoolQueryBuilder) Build() (*BoolQuery, error) { + boolQuery := BoolQuery{} + + if b.filterQueryBuilder != nil { + filters, err := b.filterQueryBuilder.Build() + if err != nil { + return nil, err + } + boolQuery.Filters = filters + } + + return &boolQuery, nil +} + +// FilterQueryBuilder represents a filter query builder +type FilterQueryBuilder struct { + filters []Filter +} + +// NewFilterQueryBuilder creates a new filter query builder +func NewFilterQueryBuilder() *FilterQueryBuilder { + return &FilterQueryBuilder{ + filters: make([]Filter, 0), + } +} + +// Build builds and return a filter query builder +func (b *FilterQueryBuilder) Build() ([]Filter, error) { + return b.filters, nil +} + +// AddDateRangeFilter adds a new time range filter +func (b *FilterQueryBuilder) AddDateRangeFilter(timeField, lte, gte, format string) *FilterQueryBuilder { + b.filters = append(b.filters, &RangeFilter{ + Key: timeField, + Lte: lte, + Gte: gte, + Format: format, + }) + return b +} + +// AddQueryStringFilter adds a new query string filter +func (b *FilterQueryBuilder) AddQueryStringFilter(querystring string, analyseWildcard bool) *FilterQueryBuilder { + if len(strings.TrimSpace(querystring)) == 0 { + return b + } + + b.filters = append(b.filters, &QueryStringFilter{ + Query: querystring, + AnalyzeWildcard: analyseWildcard, + }) + return b +} + +// AggBuilder represents an aggregation builder +type AggBuilder interface { + Histogram(key, field string, fn func(a *HistogramAgg, b AggBuilder)) AggBuilder + DateHistogram(key, field string, fn func(a *DateHistogramAgg, b AggBuilder)) AggBuilder + Terms(key, field string, fn func(a *TermsAggregation, b AggBuilder)) AggBuilder + Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder + GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder + Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder + Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder + Build() (AggArray, error) +} + +type aggBuilderImpl struct { + AggBuilder + aggDefs []*aggDef + version int +} + +func newAggBuilder(version int) *aggBuilderImpl { + return &aggBuilderImpl{ + aggDefs: make([]*aggDef, 0), + version: version, + } +} + +func (b *aggBuilderImpl) Build() (AggArray, error) { + aggs := make(AggArray, 0) + + for _, aggDef := range b.aggDefs { + agg := &Agg{ + Key: aggDef.key, + Aggregation: aggDef.aggregation, + } + + for _, cb := range aggDef.builders { + childAggs, err := cb.Build() + if err != nil { + return nil, err + } + + agg.Aggregation.Aggs = append(agg.Aggregation.Aggs, childAggs...) + } + + aggs = append(aggs, agg) + } + + return aggs, nil +} + +func (b *aggBuilderImpl) Histogram(key, field string, fn func(a *HistogramAgg, b AggBuilder)) AggBuilder { + innerAgg := &HistogramAgg{ + Field: field, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "histogram", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder(b.version) + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) DateHistogram(key, field string, fn func(a *DateHistogramAgg, b AggBuilder)) AggBuilder { + innerAgg := &DateHistogramAgg{ + Field: field, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "date_histogram", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder(b.version) + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +const termsOrderTerm = "_term" + +func (b *aggBuilderImpl) Terms(key, field string, fn func(a *TermsAggregation, b AggBuilder)) AggBuilder { + innerAgg := &TermsAggregation{ + Field: field, + Order: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: "terms", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder(b.version) + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + if b.version >= 60 && len(innerAgg.Order) > 0 { + if orderBy, exists := innerAgg.Order[termsOrderTerm]; exists { + innerAgg.Order["_key"] = orderBy + delete(innerAgg.Order, termsOrderTerm) + } + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder { + innerAgg := &FiltersAggregation{ + Filters: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: "filters", + Aggregation: innerAgg, + }) + if fn != nil { + builder := newAggBuilder(b.version) + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder { + innerAgg := &GeoHashGridAggregation{ + Field: field, + Precision: 5, + } + aggDef := newAggDef(key, &aggContainer{ + Type: "geohash_grid", + Aggregation: innerAgg, + }) + + if fn != nil { + builder := newAggBuilder(b.version) + aggDef.builders = append(aggDef.builders, builder) + fn(innerAgg, builder) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder { + innerAgg := &MetricAggregation{ + Field: field, + Settings: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: metricType, + Aggregation: innerAgg, + }) + + if fn != nil { + fn(innerAgg) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} + +func (b *aggBuilderImpl) Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder { + innerAgg := &PipelineAggregation{ + BucketPath: bucketPath, + Settings: make(map[string]interface{}), + } + aggDef := newAggDef(key, &aggContainer{ + Type: pipelineType, + Aggregation: innerAgg, + }) + + if fn != nil { + fn(innerAgg) + } + + b.aggDefs = append(b.aggDefs, aggDef) + + return b +} diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go new file mode 100644 index 00000000000..862b8058cba --- /dev/null +++ b/pkg/tsdb/elasticsearch/client/search_request_test.go @@ -0,0 +1,473 @@ +package es + +import ( + "encoding/json" + "testing" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestSearchRequest(t *testing.T) { + Convey("Test elasticsearch search request", t, func() { + timeField := "@timestamp" + Convey("Given new search request builder for es version 5", func() { + b := NewSearchRequestBuilder(5, tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have size of zero", func() { + So(sr.Size, ShouldEqual, 0) + }) + + Convey("Should have no sorting", func() { + So(sr.Sort, ShouldHaveLength, 0) + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + So(json.Get("size").MustInt(500), ShouldEqual, 0) + So(json.Get("sort").Interface(), ShouldBeNil) + So(json.Get("aggs").Interface(), ShouldBeNil) + So(json.Get("query").Interface(), ShouldBeNil) + }) + }) + + Convey("When adding size, sort, filters", func() { + b.Size(200) + b.SortDesc(timeField, "boolean") + filters := b.Query().Bool().Filter() + filters.AddDateRangeFilter(timeField, "$timeTo", "$timeFrom", DateFormatEpochMS) + filters.AddQueryStringFilter("test", true) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have correct size", func() { + So(sr.Size, ShouldEqual, 200) + }) + + Convey("Should have correct sorting", func() { + sort, ok := sr.Sort[timeField].(map[string]string) + So(ok, ShouldBeTrue) + So(sort["order"], ShouldEqual, "desc") + So(sort["unmapped_type"], ShouldEqual, "boolean") + }) + + Convey("Should have range filter", func() { + f, ok := sr.Query.Bool.Filters[0].(*RangeFilter) + So(ok, ShouldBeTrue) + So(f.Gte, ShouldEqual, "$timeFrom") + So(f.Lte, ShouldEqual, "$timeTo") + So(f.Format, ShouldEqual, "epoch_millis") + }) + + Convey("Should have query string filter", func() { + f, ok := sr.Query.Bool.Filters[1].(*QueryStringFilter) + So(ok, ShouldBeTrue) + So(f.Query, ShouldEqual, "test") + So(f.AnalyzeWildcard, ShouldBeTrue) + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + So(json.Get("size").MustInt(0), ShouldEqual, 200) + + sort := json.GetPath("sort", timeField) + So(sort.Get("order").MustString(), ShouldEqual, "desc") + So(sort.Get("unmapped_type").MustString(), ShouldEqual, "boolean") + + timeRangeFilter := json.GetPath("query", "bool", "filter").GetIndex(0).Get("range").Get(timeField) + So(timeRangeFilter.Get("gte").MustString(""), ShouldEqual, "$timeFrom") + So(timeRangeFilter.Get("lte").MustString(""), ShouldEqual, "$timeTo") + So(timeRangeFilter.Get("format").MustString(""), ShouldEqual, DateFormatEpochMS) + + queryStringFilter := json.GetPath("query", "bool", "filter").GetIndex(1).Get("query_string") + So(queryStringFilter.Get("analyze_wildcard").MustBool(false), ShouldEqual, true) + So(queryStringFilter.Get("query").MustString(""), ShouldEqual, "test") + }) + }) + }) + + Convey("When adding doc value field", func() { + b.AddDocValueField(timeField) + + Convey("should set correct props", func() { + So(b.customProps["fields"], ShouldBeNil) + + scriptFields, ok := b.customProps["script_fields"].(map[string]interface{}) + So(ok, ShouldBeTrue) + So(scriptFields, ShouldHaveLength, 0) + + docValueFields, ok := b.customProps["docvalue_fields"].([]string) + So(ok, ShouldBeTrue) + So(docValueFields, ShouldHaveLength, 1) + So(docValueFields[0], ShouldEqual, timeField) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + + scriptFields, err := json.Get("script_fields").Map() + So(err, ShouldBeNil) + So(scriptFields, ShouldHaveLength, 0) + + _, err = json.Get("fields").StringArray() + So(err, ShouldNotBeNil) + + docValueFields, err := json.Get("docvalue_fields").StringArray() + So(err, ShouldBeNil) + So(docValueFields, ShouldHaveLength, 1) + So(docValueFields[0], ShouldEqual, timeField) + }) + }) + }) + + Convey("and adding multiple top level aggs", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", nil) + aggBuilder.DateHistogram("2", "@timestamp", nil) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 2 top level aggs", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 2) + So(aggs[0].Key, ShouldEqual, "1") + So(aggs[0].Aggregation.Type, ShouldEqual, "terms") + So(aggs[1].Key, ShouldEqual, "2") + So(aggs[1].Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + + So(json.Get("aggs").MustMap(), ShouldHaveLength, 2) + So(json.GetPath("aggs", "1", "terms", "field").MustString(), ShouldEqual, "@hostname") + So(json.GetPath("aggs", "2", "date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding top level agg with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.DateHistogram("2", "@timestamp", nil) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg and one child agg", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := aggs[0].Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + + So(json.Get("aggs").MustMap(), ShouldHaveLength, 1) + firstLevelAgg := json.GetPath("aggs", "1") + secondLevelAgg := firstLevelAgg.GetPath("aggs", "2") + So(firstLevelAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + So(secondLevelAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding two top level aggs with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Histogram("1", "@hostname", func(a *HistogramAgg, ib AggBuilder) { + ib.DateHistogram("2", "@timestamp", nil) + }) + aggBuilder.Filters("3", func(a *FiltersAggregation, ib AggBuilder) { + ib.Terms("4", "@test", nil) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 2 top level aggs with one child agg each", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 2) + + topAggOne := aggs[0] + So(topAggOne.Key, ShouldEqual, "1") + So(topAggOne.Aggregation.Type, ShouldEqual, "histogram") + So(topAggOne.Aggregation.Aggs, ShouldHaveLength, 1) + + topAggOnechildAgg := topAggOne.Aggregation.Aggs[0] + So(topAggOnechildAgg.Key, ShouldEqual, "2") + So(topAggOnechildAgg.Aggregation.Type, ShouldEqual, "date_histogram") + + topAggTwo := aggs[1] + So(topAggTwo.Key, ShouldEqual, "3") + So(topAggTwo.Aggregation.Type, ShouldEqual, "filters") + So(topAggTwo.Aggregation.Aggs, ShouldHaveLength, 1) + + topAggTwochildAgg := topAggTwo.Aggregation.Aggs[0] + So(topAggTwochildAgg.Key, ShouldEqual, "4") + So(topAggTwochildAgg.Aggregation.Type, ShouldEqual, "terms") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + + topAggOne := json.GetPath("aggs", "1") + So(topAggOne.GetPath("histogram", "field").MustString(), ShouldEqual, "@hostname") + topAggOnechildAgg := topAggOne.GetPath("aggs", "2") + So(topAggOnechildAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + + topAggTwo := json.GetPath("aggs", "3") + topAggTwochildAgg := topAggTwo.GetPath("aggs", "4") + So(topAggTwo.GetPath("filters").MustArray(), ShouldHaveLength, 0) + So(topAggTwochildAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@test") + }) + }) + }) + + Convey("and adding top level agg with child agg with child agg", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.Terms("2", "@app", func(a *TermsAggregation, ib AggBuilder) { + ib.DateHistogram("3", "@timestamp", nil) + }) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg with one child having a child", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := topAgg.Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "terms") + + childChildAgg := childAgg.Aggregation.Aggs[0] + So(childChildAgg.Key, ShouldEqual, "3") + So(childChildAgg.Aggregation.Type, ShouldEqual, "date_histogram") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + + topAgg := json.GetPath("aggs", "1") + So(topAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + + childAgg := topAgg.GetPath("aggs", "2") + So(childAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@app") + + childChildAgg := childAgg.GetPath("aggs", "3") + So(childChildAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + }) + }) + }) + + Convey("and adding bucket and metric aggs", func() { + aggBuilder := b.Agg() + aggBuilder.Terms("1", "@hostname", func(a *TermsAggregation, ib AggBuilder) { + ib.Terms("2", "@app", func(a *TermsAggregation, ib AggBuilder) { + ib.Metric("4", "avg", "@value", nil) + ib.DateHistogram("3", "@timestamp", func(a *DateHistogramAgg, ib AggBuilder) { + ib.Metric("4", "avg", "@value", nil) + ib.Metric("5", "max", "@value", nil) + }) + }) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("Should have 1 top level agg with one child having a child", func() { + aggs := sr.Aggs + So(aggs, ShouldHaveLength, 1) + + topAgg := aggs[0] + So(topAgg.Key, ShouldEqual, "1") + So(topAgg.Aggregation.Type, ShouldEqual, "terms") + So(topAgg.Aggregation.Aggs, ShouldHaveLength, 1) + + childAgg := topAgg.Aggregation.Aggs[0] + So(childAgg.Key, ShouldEqual, "2") + So(childAgg.Aggregation.Type, ShouldEqual, "terms") + + childChildOneAgg := childAgg.Aggregation.Aggs[0] + So(childChildOneAgg.Key, ShouldEqual, "4") + So(childChildOneAgg.Aggregation.Type, ShouldEqual, "avg") + + childChildTwoAgg := childAgg.Aggregation.Aggs[1] + So(childChildTwoAgg.Key, ShouldEqual, "3") + So(childChildTwoAgg.Aggregation.Type, ShouldEqual, "date_histogram") + + childChildTwoChildOneAgg := childChildTwoAgg.Aggregation.Aggs[0] + So(childChildTwoChildOneAgg.Key, ShouldEqual, "4") + So(childChildTwoChildOneAgg.Aggregation.Type, ShouldEqual, "avg") + + childChildTwoChildTwoAgg := childChildTwoAgg.Aggregation.Aggs[1] + So(childChildTwoChildTwoAgg.Key, ShouldEqual, "5") + So(childChildTwoChildTwoAgg.Aggregation.Type, ShouldEqual, "max") + }) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + + termsAgg := json.GetPath("aggs", "1") + So(termsAgg.GetPath("terms", "field").MustString(), ShouldEqual, "@hostname") + + termsAggTwo := termsAgg.GetPath("aggs", "2") + So(termsAggTwo.GetPath("terms", "field").MustString(), ShouldEqual, "@app") + + termsAggTwoAvg := termsAggTwo.GetPath("aggs", "4") + So(termsAggTwoAvg.GetPath("avg", "field").MustString(), ShouldEqual, "@value") + + dateHistAgg := termsAggTwo.GetPath("aggs", "3") + So(dateHistAgg.GetPath("date_histogram", "field").MustString(), ShouldEqual, "@timestamp") + + avgAgg := dateHistAgg.GetPath("aggs", "4") + So(avgAgg.GetPath("avg", "field").MustString(), ShouldEqual, "@value") + + maxAgg := dateHistAgg.GetPath("aggs", "5") + So(maxAgg.GetPath("max", "field").MustString(), ShouldEqual, "@value") + }) + }) + }) + }) + + Convey("Given new search request builder for es version 2", func() { + b := NewSearchRequestBuilder(2, tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + + Convey("When adding doc value field", func() { + b.AddDocValueField(timeField) + + Convey("should set correct props", func() { + fields, ok := b.customProps["fields"].([]string) + So(ok, ShouldBeTrue) + So(fields, ShouldHaveLength, 2) + So(fields[0], ShouldEqual, "*") + So(fields[1], ShouldEqual, "_source") + + scriptFields, ok := b.customProps["script_fields"].(map[string]interface{}) + So(ok, ShouldBeTrue) + So(scriptFields, ShouldHaveLength, 0) + + fieldDataFields, ok := b.customProps["fielddata_fields"].([]string) + So(ok, ShouldBeTrue) + So(fieldDataFields, ShouldHaveLength, 1) + So(fieldDataFields[0], ShouldEqual, timeField) + }) + + Convey("When building search request", func() { + sr, err := b.Build() + So(err, ShouldBeNil) + + Convey("When marshal to JSON should generate correct json", func() { + body, err := json.Marshal(sr) + So(err, ShouldBeNil) + json, err := simplejson.NewJson(body) + So(err, ShouldBeNil) + + scriptFields, err := json.Get("script_fields").Map() + So(err, ShouldBeNil) + So(scriptFields, ShouldHaveLength, 0) + + fields, err := json.Get("fields").StringArray() + So(err, ShouldBeNil) + So(fields, ShouldHaveLength, 2) + So(fields[0], ShouldEqual, "*") + So(fields[1], ShouldEqual, "_source") + + fieldDataFields, err := json.Get("fielddata_fields").StringArray() + So(err, ShouldBeNil) + So(fieldDataFields, ShouldHaveLength, 1) + So(fieldDataFields[0], ShouldEqual, timeField) + }) + }) + }) + }) + }) +} + +func TestMultiSearchRequest(t *testing.T) { + Convey("Test elasticsearch multi search request", t, func() { + Convey("Given new multi search request builder", func() { + b := NewMultiSearchRequestBuilder(0) + + Convey("When adding one search request", func() { + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + + Convey("When building search request should contain one search request", func() { + mr, err := b.Build() + So(err, ShouldBeNil) + So(mr.Requests, ShouldHaveLength, 1) + }) + }) + + Convey("When adding two search requests", func() { + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"}) + + Convey("When building search request should contain two search requests", func() { + mr, err := b.Build() + So(err, ShouldBeNil) + So(mr.Requests, ShouldHaveLength, 2) + }) + }) + }) + }) +} diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go new file mode 100644 index 00000000000..857b847f0f9 --- /dev/null +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -0,0 +1,45 @@ +package elasticsearch + +import ( + "context" + "fmt" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" +) + +// ElasticsearchExecutor represents a handler for handling elasticsearch datasource request +type ElasticsearchExecutor struct{} + +var ( + glog log.Logger + intervalCalculator tsdb.IntervalCalculator +) + +// NewElasticsearchExecutor creates a new elasticsearch executor +func NewElasticsearchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + return &ElasticsearchExecutor{}, nil +} + +func init() { + glog = log.New("tsdb.elasticsearch") + intervalCalculator = tsdb.NewIntervalCalculator(nil) + tsdb.RegisterTsdbQueryEndpoint("elasticsearch", NewElasticsearchExecutor) +} + +// Query handles an elasticsearch datasource request +func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + if len(tsdbQuery.Queries) == 0 { + return nil, fmt.Errorf("query contains no queries") + } + + client, err := es.NewClient(ctx, dsInfo, tsdbQuery.TimeRange) + if err != nil { + return nil, err + } + + query := newTimeSeriesQuery(client, tsdbQuery, intervalCalculator) + return query.execute() +} diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go new file mode 100644 index 00000000000..b3fdee95b91 --- /dev/null +++ b/pkg/tsdb/elasticsearch/models.go @@ -0,0 +1,77 @@ +package elasticsearch + +import ( + "github.com/grafana/grafana/pkg/components/simplejson" +) + +// Query represents the time series query model of the datasource +type Query struct { + TimeField string `json:"timeField"` + RawQuery string `json:"query"` + BucketAggs []*BucketAgg `json:"bucketAggs"` + Metrics []*MetricAgg `json:"metrics"` + Alias string `json:"alias"` + Interval string + RefID string +} + +// BucketAgg represents a bucket aggregation of the time series query model of the datasource +type BucketAgg struct { + Field string `json:"field"` + ID string `json:"id"` + Settings *simplejson.Json `json:"settings"` + Type string `jsons:"type"` +} + +// MetricAgg represents a metric aggregation of the time series query model of the datasource +type MetricAgg struct { + Field string `json:"field"` + Hide bool `json:"hide"` + ID string `json:"id"` + PipelineAggregate string `json:"pipelineAgg"` + Settings *simplejson.Json `json:"settings"` + Meta *simplejson.Json `json:"meta"` + Type string `json:"type"` +} + +var metricAggType = map[string]string{ + "count": "Count", + "avg": "Average", + "sum": "Sum", + "max": "Max", + "min": "Min", + "extended_stats": "Extended Stats", + "percentiles": "Percentiles", + "cardinality": "Unique Count", + "moving_avg": "Moving Average", + "derivative": "Derivative", + "raw_document": "Raw Document", +} + +var extendedStats = map[string]string{ + "avg": "Avg", + "min": "Min", + "max": "Max", + "sum": "Sum", + "count": "Count", + "std_deviation": "Std Dev", + "std_deviation_bounds_upper": "Std Dev Upper", + "std_deviation_bounds_lower": "Std Dev Lower", +} + +var pipelineAggType = map[string]string{ + "moving_avg": "moving_avg", + "derivative": "derivative", +} + +func isPipelineAgg(metricType string) bool { + if _, ok := pipelineAggType[metricType]; ok { + return true + } + return false +} + +func describeMetric(metricType, field string) string { + text := metricAggType[metricType] + return text + " " + field +} diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go new file mode 100644 index 00000000000..0837c3dd9d5 --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -0,0 +1,548 @@ +package elasticsearch + +import ( + "errors" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" +) + +const ( + // Metric types + countType = "count" + percentilesType = "percentiles" + extendedStatsType = "extended_stats" + // Bucket types + dateHistType = "date_histogram" + histogramType = "histogram" + filtersType = "filters" + termsType = "terms" + geohashGridType = "geohash_grid" +) + +type responseParser struct { + Responses []*es.SearchResponse + Targets []*Query +} + +var newResponseParser = func(responses []*es.SearchResponse, targets []*Query) *responseParser { + return &responseParser{ + Responses: responses, + Targets: targets, + } +} + +func (rp *responseParser) getTimeSeries() (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + + if rp.Responses == nil { + return result, nil + } + + for i, res := range rp.Responses { + target := rp.Targets[i] + + if res.Error != nil { + result.Results[target.RefID] = getErrorFromElasticResponse(res) + continue + } + + queryRes := tsdb.NewQueryResult() + props := make(map[string]string) + table := tsdb.Table{ + Columns: make([]tsdb.TableColumn, 0), + Rows: make([]tsdb.RowValues, 0), + } + err := rp.processBuckets(res.Aggregations, target, &queryRes.Series, &table, props, 0) + if err != nil { + return nil, err + } + rp.nameSeries(&queryRes.Series, target) + rp.trimDatapoints(&queryRes.Series, target) + + if len(table.Rows) > 0 { + queryRes.Tables = append(queryRes.Tables, &table) + } + + result.Results[target.RefID] = queryRes + } + return result, nil +} + +func (rp *responseParser) processBuckets(aggs map[string]interface{}, target *Query, series *tsdb.TimeSeriesSlice, table *tsdb.Table, props map[string]string, depth int) error { + var err error + maxDepth := len(target.BucketAggs) - 1 + + aggIDs := make([]string, 0) + for k := range aggs { + aggIDs = append(aggIDs, k) + } + sort.Strings(aggIDs) + for _, aggID := range aggIDs { + v := aggs[aggID] + aggDef, _ := findAgg(target, aggID) + esAgg := simplejson.NewFromAny(v) + if aggDef == nil { + continue + } + + if depth == maxDepth { + if aggDef.Type == dateHistType { + err = rp.processMetrics(esAgg, target, series, props) + } else { + err = rp.processAggregationDocs(esAgg, aggDef, target, table, props) + } + if err != nil { + return err + } + } else { + for _, b := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(b) + newProps := make(map[string]string) + + for k, v := range props { + newProps[k] = v + } + + if key, err := bucket.Get("key").String(); err == nil { + newProps[aggDef.Field] = key + } else if key, err := bucket.Get("key").Int64(); err == nil { + newProps[aggDef.Field] = strconv.FormatInt(key, 10) + } + + if key, err := bucket.Get("key_as_string").String(); err == nil { + newProps[aggDef.Field] = key + } + err = rp.processBuckets(bucket.MustMap(), target, series, table, newProps, depth+1) + if err != nil { + return err + } + } + + buckets := esAgg.Get("buckets").MustMap() + bucketKeys := make([]string, 0) + for k := range buckets { + bucketKeys = append(bucketKeys, k) + } + sort.Strings(bucketKeys) + + for _, bucketKey := range bucketKeys { + bucket := simplejson.NewFromAny(buckets[bucketKey]) + newProps := make(map[string]string) + + for k, v := range props { + newProps[k] = v + } + + newProps["filter"] = bucketKey + + err = rp.processBuckets(bucket.MustMap(), target, series, table, newProps, depth+1) + if err != nil { + return err + } + } + } + + } + return nil + +} + +func (rp *responseParser) processMetrics(esAgg *simplejson.Json, target *Query, series *tsdb.TimeSeriesSlice, props map[string]string) error { + for _, metric := range target.Metrics { + if metric.Hide { + continue + } + + switch metric.Type { + case countType: + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + value := castToNullFloat(bucket.Get("doc_count")) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = countType + *series = append(*series, &newSeries) + + case percentilesType: + buckets := esAgg.Get("buckets").MustArray() + if len(buckets) == 0 { + break + } + + firstBucket := simplejson.NewFromAny(buckets[0]) + percentiles := firstBucket.GetPath(metric.ID, "values").MustMap() + + percentileKeys := make([]string, 0) + for k := range percentiles { + percentileKeys = append(percentileKeys, k) + } + sort.Strings(percentileKeys) + for _, percentileName := range percentileKeys { + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = "p" + percentileName + newSeries.Tags["field"] = metric.Field + for _, v := range buckets { + bucket := simplejson.NewFromAny(v) + value := castToNullFloat(bucket.GetPath(metric.ID, "values", percentileName)) + key := castToNullFloat(bucket.Get("key")) + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } + case extendedStatsType: + buckets := esAgg.Get("buckets").MustArray() + + metaKeys := make([]string, 0) + meta := metric.Meta.MustMap() + for k := range meta { + metaKeys = append(metaKeys, k) + } + sort.Strings(metaKeys) + for _, statName := range metaKeys { + v := meta[statName] + if enabled, ok := v.(bool); !ok || !enabled { + continue + } + + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for k, v := range props { + newSeries.Tags[k] = v + } + newSeries.Tags["metric"] = statName + newSeries.Tags["field"] = metric.Field + + for _, v := range buckets { + bucket := simplejson.NewFromAny(v) + key := castToNullFloat(bucket.Get("key")) + var value null.Float + if statName == "std_deviation_bounds_upper" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "upper")) + } else if statName == "std_deviation_bounds_lower" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "lower")) + } else { + value = castToNullFloat(bucket.GetPath(metric.ID, statName)) + } + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } + default: + newSeries := tsdb.TimeSeries{ + Tags: make(map[string]string), + } + for k, v := range props { + newSeries.Tags[k] = v + } + + newSeries.Tags["metric"] = metric.Type + newSeries.Tags["field"] = metric.Field + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + key := castToNullFloat(bucket.Get("key")) + valueObj, err := bucket.Get(metric.ID).Map() + if err != nil { + continue + } + var value null.Float + if _, ok := valueObj["normalized_value"]; ok { + value = castToNullFloat(bucket.GetPath(metric.ID, "normalized_value")) + } else { + value = castToNullFloat(bucket.GetPath(metric.ID, "value")) + } + newSeries.Points = append(newSeries.Points, tsdb.TimePoint{value, key}) + } + *series = append(*series, &newSeries) + } + } + return nil +} + +func (rp *responseParser) processAggregationDocs(esAgg *simplejson.Json, aggDef *BucketAgg, target *Query, table *tsdb.Table, props map[string]string) error { + propKeys := make([]string, 0) + for k := range props { + propKeys = append(propKeys, k) + } + sort.Strings(propKeys) + + if len(table.Columns) == 0 { + for _, propKey := range propKeys { + table.Columns = append(table.Columns, tsdb.TableColumn{Text: propKey}) + } + table.Columns = append(table.Columns, tsdb.TableColumn{Text: aggDef.Field}) + } + + addMetricValue := func(values *tsdb.RowValues, metricName string, value null.Float) { + found := false + for _, c := range table.Columns { + if c.Text == metricName { + found = true + break + } + } + if !found { + table.Columns = append(table.Columns, tsdb.TableColumn{Text: metricName}) + } + *values = append(*values, value) + } + + for _, v := range esAgg.Get("buckets").MustArray() { + bucket := simplejson.NewFromAny(v) + values := make(tsdb.RowValues, 0) + + for _, propKey := range propKeys { + values = append(values, props[propKey]) + } + + if key, err := bucket.Get("key").String(); err == nil { + values = append(values, key) + } else { + values = append(values, castToNullFloat(bucket.Get("key"))) + } + + for _, metric := range target.Metrics { + switch metric.Type { + case countType: + addMetricValue(&values, rp.getMetricName(metric.Type), castToNullFloat(bucket.Get("doc_count"))) + case extendedStatsType: + metaKeys := make([]string, 0) + meta := metric.Meta.MustMap() + for k := range meta { + metaKeys = append(metaKeys, k) + } + sort.Strings(metaKeys) + for _, statName := range metaKeys { + v := meta[statName] + if enabled, ok := v.(bool); !ok || !enabled { + continue + } + + var value null.Float + if statName == "std_deviation_bounds_upper" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "upper")) + } else if statName == "std_deviation_bounds_lower" { + value = castToNullFloat(bucket.GetPath(metric.ID, "std_deviation_bounds", "lower")) + } else { + value = castToNullFloat(bucket.GetPath(metric.ID, statName)) + } + + addMetricValue(&values, rp.getMetricName(metric.Type), value) + break + } + default: + metricName := rp.getMetricName(metric.Type) + otherMetrics := make([]*MetricAgg, 0) + + for _, m := range target.Metrics { + if m.Type == metric.Type { + otherMetrics = append(otherMetrics, m) + } + } + + if len(otherMetrics) > 1 { + metricName += " " + metric.Field + } + + addMetricValue(&values, metricName, castToNullFloat(bucket.GetPath(metric.ID, "value"))) + } + } + + table.Rows = append(table.Rows, values) + } + + return nil +} + +func (rp *responseParser) trimDatapoints(series *tsdb.TimeSeriesSlice, target *Query) { + var histogram *BucketAgg + for _, bucketAgg := range target.BucketAggs { + if bucketAgg.Type == dateHistType { + histogram = bucketAgg + break + } + } + + if histogram == nil { + return + } + + trimEdges, err := histogram.Settings.Get("trimEdges").Int() + if err != nil { + return + } + + for _, s := range *series { + if len(s.Points) > trimEdges*2 { + s.Points = s.Points[trimEdges : len(s.Points)-trimEdges] + } + } +} + +func (rp *responseParser) nameSeries(seriesList *tsdb.TimeSeriesSlice, target *Query) { + set := make(map[string]string) + for _, v := range *seriesList { + if metricType, exists := v.Tags["metric"]; exists { + if _, ok := set[metricType]; !ok { + set[metricType] = "" + } + } + } + metricTypeCount := len(set) + for _, series := range *seriesList { + series.Name = rp.getSeriesName(series, target, metricTypeCount) + } + +} + +var aliasPatternRegex = regexp.MustCompile(`\{\{([\s\S]+?)\}\}`) + +func (rp *responseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, metricTypeCount int) string { + metricType := series.Tags["metric"] + metricName := rp.getMetricName(metricType) + delete(series.Tags, "metric") + + field := "" + if v, ok := series.Tags["field"]; ok { + field = v + delete(series.Tags, "field") + } + + if target.Alias != "" { + seriesName := target.Alias + + subMatches := aliasPatternRegex.FindAllStringSubmatch(target.Alias, -1) + for _, subMatch := range subMatches { + group := subMatch[0] + + if len(subMatch) > 1 { + group = subMatch[1] + } + + if strings.Index(group, "term ") == 0 { + seriesName = strings.Replace(seriesName, subMatch[0], series.Tags[group[5:]], 1) + } + if v, ok := series.Tags[group]; ok { + seriesName = strings.Replace(seriesName, subMatch[0], v, 1) + } + if group == "metric" { + seriesName = strings.Replace(seriesName, subMatch[0], metricName, 1) + } + if group == "field" { + seriesName = strings.Replace(seriesName, subMatch[0], field, 1) + } + } + + return seriesName + } + // todo, if field and pipelineAgg + if field != "" && isPipelineAgg(metricType) { + found := false + for _, metric := range target.Metrics { + if metric.ID == field { + metricName += " " + describeMetric(metric.Type, field) + found = true + } + } + if !found { + metricName = "Unset" + } + } else if field != "" { + metricName += " " + field + } + + if len(series.Tags) == 0 { + return metricName + } + + name := "" + for _, v := range series.Tags { + name += v + " " + } + + if metricTypeCount == 1 { + return strings.TrimSpace(name) + } + + return strings.TrimSpace(name) + " " + metricName + +} + +func (rp *responseParser) getMetricName(metric string) string { + if text, ok := metricAggType[metric]; ok { + return text + } + + if text, ok := extendedStats[metric]; ok { + return text + } + + return metric +} + +func castToNullFloat(j *simplejson.Json) null.Float { + f, err := j.Float64() + if err == nil { + return null.FloatFrom(f) + } + + if s, err := j.String(); err == nil { + if strings.ToLower(s) == "nan" { + return null.NewFloat(0, false) + } + + if v, err := strconv.ParseFloat(s, 64); err == nil { + return null.FloatFromPtr(&v) + } + } + + return null.NewFloat(0, false) +} + +func findAgg(target *Query, aggID string) (*BucketAgg, error) { + for _, v := range target.BucketAggs { + if aggID == v.ID { + return v, nil + } + } + return nil, errors.New("can't found aggDef, aggID:" + aggID) +} + +func getErrorFromElasticResponse(response *es.SearchResponse) *tsdb.QueryResult { + result := tsdb.NewQueryResult() + json := simplejson.NewFromAny(response.Error) + reason := json.Get("reason").MustString() + rootCauseReason := json.Get("root_cause").GetIndex(0).Get("reason").MustString() + + if rootCauseReason != "" { + result.ErrorString = rootCauseReason + } else if reason != "" { + result.ErrorString = reason + } else { + result.ErrorString = "Unkown elasticsearch error response" + } + + return result +} diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go new file mode 100644 index 00000000000..b00c14cf946 --- /dev/null +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -0,0 +1,880 @@ +package elasticsearch + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" + + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestResponseParser(t *testing.T) { + Convey("Elasticsearch response parser test", t, func() { + Convey("Simple query and count", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "doc_count": 10, + "key": 1000 + }, + { + "doc_count": 15, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 1) + series := queryRes.Series[0] + So(series.Name, ShouldEqual, "Count") + So(series.Points, ShouldHaveLength, 2) + So(series.Points[0][0].Float64, ShouldEqual, 10) + So(series.Points[0][1].Float64, ShouldEqual, 1000) + So(series.Points[1][0].Float64, ShouldEqual, 15) + So(series.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Simple query count & avg aggregation", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }, {"type": "avg", "field": "value", "id": "2" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "2": { "value": 88 }, + "doc_count": 10, + "key": 1000 + }, + { + "2": { "value": 99 }, + "doc_count": 15, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "Count") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 10) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 15) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "Average value") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 88) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 99) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Single group by query one metric", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": "server2" + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server2") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Single group by query two metrics", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }, { "type": "avg", "field": "@value", "id": "4" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [ + { "4": { "value": 10 }, "doc_count": 1, "key": 1000 }, + { "4": { "value": 12 }, "doc_count": 3, "key": 2000 } + ] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [ + { "4": { "value": 20 }, "doc_count": 1, "key": 1000 }, + { "4": { "value": 32 }, "doc_count": 3, "key": 2000 } + ] + }, + "doc_count": 10, + "key": "server2" + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 4) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Count") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server1 Average @value") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 10) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 12) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "server2 Count") + So(seriesThree.Points, ShouldHaveLength, 2) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 1) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesThree.Points[1][0].Float64, ShouldEqual, 3) + So(seriesThree.Points[1][1].Float64, ShouldEqual, 2000) + + seriesFour := queryRes.Series[3] + So(seriesFour.Name, ShouldEqual, "server2 Average @value") + So(seriesFour.Points, ShouldHaveLength, 2) + So(seriesFour.Points[0][0].Float64, ShouldEqual, 20) + So(seriesFour.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesFour.Points[1][0].Float64, ShouldEqual, 32) + So(seriesFour.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With percentiles", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "percentiles", "settings": { "percents": [75, 90] }, "id": "1" }], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "1": { "values": { "75": 3.3, "90": 5.5 } }, + "doc_count": 10, + "key": 1000 + }, + { + "1": { "values": { "75": 2.3, "90": 4.5 } }, + "doc_count": 15, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "p75") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 3.3) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 2.3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "p90") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 5.5) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 4.5) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With extended stats", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "extended_stats", "meta": { "max": true, "std_deviation_bounds_upper": true, "std_deviation_bounds_lower": true }, "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "host", "id": "3" }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [ + { + "key": "server1", + "4": { + "buckets": [ + { + "1": { + "max": 10.2, + "min": 5.5, + "std_deviation_bounds": { "upper": 3, "lower": -2 } + }, + "doc_count": 10, + "key": 1000 + } + ] + } + }, + { + "key": "server2", + "4": { + "buckets": [ + { + "1": { + "max": 15.5, + "min": 3.4, + "std_deviation_bounds": { "upper": 4, "lower": -1 } + }, + "doc_count": 10, + "key": 1000 + } + ] + } + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 6) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Max") + So(seriesOne.Points, ShouldHaveLength, 1) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 10.2) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server1 Std Dev Lower") + So(seriesTwo.Points, ShouldHaveLength, 1) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, -2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "server1 Std Dev Upper") + So(seriesThree.Points, ShouldHaveLength, 1) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 3) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + + seriesFour := queryRes.Series[3] + So(seriesFour.Name, ShouldEqual, "server2 Max") + So(seriesFour.Points, ShouldHaveLength, 1) + So(seriesFour.Points[0][0].Float64, ShouldEqual, 15.5) + So(seriesFour.Points[0][1].Float64, ShouldEqual, 1000) + + seriesFive := queryRes.Series[4] + So(seriesFive.Name, ShouldEqual, "server2 Std Dev Lower") + So(seriesFive.Points, ShouldHaveLength, 1) + So(seriesFive.Points[0][0].Float64, ShouldEqual, -1) + So(seriesFive.Points[0][1].Float64, ShouldEqual, 1000) + + seriesSix := queryRes.Series[5] + So(seriesSix.Name, ShouldEqual, "server2 Std Dev Upper") + So(seriesSix.Points, ShouldHaveLength, 1) + So(seriesSix.Points[0][0].Float64, ShouldEqual, 4) + So(seriesSix.Points[0][1].Float64, ShouldEqual, 1000) + }) + + Convey("Single group by with alias pattern", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "alias": "{{term @host}} {{metric}} and {{not_exist}} {{@host}}", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { "type": "terms", "field": "@host", "id": "2" }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + }, + "doc_count": 4, + "key": "server1" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": "server2" + }, + { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + }, + "doc_count": 10, + "key": 0 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 3) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "server1 Count and {{not_exist}} server1") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "server2 Count and {{not_exist}} server2") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "0 Count and {{not_exist}} 0") + So(seriesThree.Points, ShouldHaveLength, 2) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 2) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesThree.Points[1][0].Float64, ShouldEqual, 8) + So(seriesThree.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("Histogram response", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [{ "type": "histogram", "field": "bytes", "id": "3" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }, { "doc_count": 2, "key": 3000 }] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 3) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 2) + + So(cols[0].Text, ShouldEqual, "bytes") + So(cols[1].Text, ShouldEqual, "Count") + + So(rows[0][0].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1) + So(rows[1][0].(null.Float).Float64, ShouldEqual, 2000) + So(rows[1][1].(null.Float).Float64, ShouldEqual, 3) + So(rows[2][0].(null.Float).Float64, ShouldEqual, 3000) + So(rows[2][1].(null.Float).Float64, ShouldEqual, 2) + }) + + Convey("With two filters agg", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "count", "id": "1" }], + "bucketAggs": [ + { + "type": "filters", + "id": "2", + "settings": { + "filters": [{ "query": "@metric:cpu" }, { "query": "@metric:logins.count" }] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": { + "@metric:cpu": { + "3": { + "buckets": [{ "doc_count": 1, "key": 1000 }, { "doc_count": 3, "key": 2000 }] + } + }, + "@metric:logins.count": { + "3": { + "buckets": [{ "doc_count": 2, "key": 1000 }, { "doc_count": 8, "key": 2000 }] + } + } + } + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "@metric:cpu") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 1) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "@metric:logins.count") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 2) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 8) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + }) + + Convey("With dropfirst and last aggregation", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "id": "1" }, { "type": "count" }], + "bucketAggs": [ + { + "type": "date_histogram", + "field": "@timestamp", + "id": "2", + "settings": { "trimEdges": 1 } + } + ] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "key": 1, + "doc_count": 369 + }, + { + "1": { "value": 2000 }, + "key": 2, + "doc_count": 200 + }, + { + "1": { "value": 2000 }, + "key": 3, + "doc_count": 200 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 2) + + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "Average") + So(seriesOne.Points, ShouldHaveLength, 1) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 2000) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 2) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "Count") + So(seriesTwo.Points, ShouldHaveLength, 1) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 200) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 2) + }) + + Convey("No group by time", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "id": "1" }, { "type": "count" }], + "bucketAggs": [{ "type": "terms", "field": "host", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "key": "server-1", + "doc_count": 369 + }, + { + "1": { "value": 2000 }, + "key": "server-2", + "doc_count": 200 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 2) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 3) + + So(cols[0].Text, ShouldEqual, "host") + So(cols[1].Text, ShouldEqual, "Average") + So(cols[2].Text, ShouldEqual, "Count") + + So(rows[0][0].(string), ShouldEqual, "server-1") + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][2].(null.Float).Float64, ShouldEqual, 369) + So(rows[1][0].(string), ShouldEqual, "server-2") + So(rows[1][1].(null.Float).Float64, ShouldEqual, 2000) + So(rows[1][2].(null.Float).Float64, ShouldEqual, 200) + }) + + Convey("Multiple metrics of same type", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [{ "type": "avg", "field": "test", "id": "1" }, { "type": "avg", "field": "test2", "id": "2" }], + "bucketAggs": [{ "type": "terms", "field": "host", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 1000 }, + "2": { "value": 3000 }, + "key": "server-1", + "doc_count": 369 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Tables, ShouldHaveLength, 1) + + rows := queryRes.Tables[0].Rows + So(rows, ShouldHaveLength, 1) + cols := queryRes.Tables[0].Columns + So(cols, ShouldHaveLength, 3) + + So(cols[0].Text, ShouldEqual, "host") + So(cols[1].Text, ShouldEqual, "Average test") + So(cols[2].Text, ShouldEqual, "Average test2") + + So(rows[0][0].(string), ShouldEqual, "server-1") + So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + So(rows[0][2].(null.Float).Float64, ShouldEqual, 3000) + }) + + // Convey("Raw documents query", func() { + // targets := map[string]string{ + // "A": `{ + // "timeField": "@timestamp", + // "metrics": [{ "type": "raw_document", "id": "1" }] + // }`, + // } + // response := `{ + // "responses": [ + // { + // "hits": { + // "total": 100, + // "hits": [ + // { + // "_id": "1", + // "_type": "type", + // "_index": "index", + // "_source": { "sourceProp": "asd" }, + // "fields": { "fieldProp": "field" } + // }, + // { + // "_source": { "sourceProp": "asd2" }, + // "fields": { "fieldProp": "field2" } + // } + // ] + // } + // } + // ] + // }` + // rp, err := newResponseParserForTest(targets, response) + // So(err, ShouldBeNil) + // result, err := rp.getTimeSeries() + // So(err, ShouldBeNil) + // So(result.Results, ShouldHaveLength, 1) + + // queryRes := result.Results["A"] + // So(queryRes, ShouldNotBeNil) + // So(queryRes.Tables, ShouldHaveLength, 1) + + // rows := queryRes.Tables[0].Rows + // So(rows, ShouldHaveLength, 1) + // cols := queryRes.Tables[0].Columns + // So(cols, ShouldHaveLength, 3) + + // So(cols[0].Text, ShouldEqual, "host") + // So(cols[1].Text, ShouldEqual, "Average test") + // So(cols[2].Text, ShouldEqual, "Average test2") + + // So(rows[0][0].(string), ShouldEqual, "server-1") + // So(rows[0][1].(null.Float).Float64, ShouldEqual, 1000) + // So(rows[0][2].(null.Float).Float64, ShouldEqual, 3000) + // }) + }) +} + +func newResponseParserForTest(tsdbQueries map[string]string, responseBody string) (*responseParser, error) { + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{}, + TimeRange: tsdb.NewTimeRange(fromStr, toStr), + } + + for refID, tsdbQueryBody := range tsdbQueries { + tsdbQueryJSON, err := simplejson.NewJson([]byte(tsdbQueryBody)) + if err != nil { + return nil, err + } + + tsdbQuery.Queries = append(tsdbQuery.Queries, &tsdb.Query{ + Model: tsdbQueryJSON, + RefId: refID, + }) + } + + var response es.MultiSearchResponse + err := json.Unmarshal([]byte(responseBody), &response) + if err != nil { + return nil, err + } + + tsQueryParser := newTimeSeriesQueryParser() + queries, err := tsQueryParser.parse(tsdbQuery) + if err != nil { + return nil, err + } + + return newResponseParser(response.Responses, queries), nil +} diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go new file mode 100644 index 00000000000..869e23e21ce --- /dev/null +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -0,0 +1,322 @@ +package elasticsearch + +import ( + "fmt" + "strconv" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" +) + +type timeSeriesQuery struct { + client es.Client + tsdbQuery *tsdb.TsdbQuery + intervalCalculator tsdb.IntervalCalculator +} + +var newTimeSeriesQuery = func(client es.Client, tsdbQuery *tsdb.TsdbQuery, intervalCalculator tsdb.IntervalCalculator) *timeSeriesQuery { + return &timeSeriesQuery{ + client: client, + tsdbQuery: tsdbQuery, + intervalCalculator: intervalCalculator, + } +} + +func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { + result := &tsdb.Response{} + result.Results = make(map[string]*tsdb.QueryResult) + + tsQueryParser := newTimeSeriesQueryParser() + queries, err := tsQueryParser.parse(e.tsdbQuery) + if err != nil { + return nil, err + } + + ms := e.client.MultiSearch() + + from := fmt.Sprintf("%d", e.tsdbQuery.TimeRange.GetFromAsMsEpoch()) + to := fmt.Sprintf("%d", e.tsdbQuery.TimeRange.GetToAsMsEpoch()) + + for _, q := range queries { + minInterval, err := e.client.GetMinInterval(q.Interval) + if err != nil { + return nil, err + } + interval := e.intervalCalculator.Calculate(e.tsdbQuery.TimeRange, minInterval) + + b := ms.Search(interval) + b.Size(0) + filters := b.Query().Bool().Filter() + filters.AddDateRangeFilter(e.client.GetTimeField(), to, from, es.DateFormatEpochMS) + + if q.RawQuery != "" { + filters.AddQueryStringFilter(q.RawQuery, true) + } + + if len(q.BucketAggs) == 0 { + if len(q.Metrics) == 0 || q.Metrics[0].Type != "raw_document" { + result.Results[q.RefID] = &tsdb.QueryResult{ + RefId: q.RefID, + Error: fmt.Errorf("invalid query, missing metrics and aggregations"), + ErrorString: "invalid query, missing metrics and aggregations", + } + continue + } + metric := q.Metrics[0] + b.Size(metric.Settings.Get("size").MustInt(500)) + b.SortDesc("@timestamp", "boolean") + b.AddDocValueField("@timestamp") + continue + } + + aggBuilder := b.Agg() + + // iterate backwards to create aggregations bottom-down + for _, bucketAgg := range q.BucketAggs { + switch bucketAgg.Type { + case dateHistType: + aggBuilder = addDateHistogramAgg(aggBuilder, bucketAgg, from, to) + case histogramType: + aggBuilder = addHistogramAgg(aggBuilder, bucketAgg) + case filtersType: + aggBuilder = addFiltersAgg(aggBuilder, bucketAgg) + case termsType: + aggBuilder = addTermsAgg(aggBuilder, bucketAgg, q.Metrics) + case geohashGridType: + aggBuilder = addGeoHashGridAgg(aggBuilder, bucketAgg) + } + } + + for _, m := range q.Metrics { + if m.Type == "count" { + continue + } + + if isPipelineAgg(m.Type) { + if _, err := strconv.Atoi(m.PipelineAggregate); err == nil { + aggBuilder.Pipeline(m.ID, m.Type, m.PipelineAggregate, func(a *es.PipelineAggregation) { + a.Settings = m.Settings.MustMap() + }) + } else { + continue + } + } else { + aggBuilder.Metric(m.ID, m.Type, m.Field, func(a *es.MetricAggregation) { + a.Settings = m.Settings.MustMap() + }) + } + } + } + + req, err := ms.Build() + if err != nil { + return nil, err + } + + res, err := e.client.ExecuteMultisearch(req) + if err != nil { + return nil, err + } + + rp := newResponseParser(res.Responses, queries) + return rp.getTimeSeries() +} + +func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFrom, timeTo string) es.AggBuilder { + aggBuilder.DateHistogram(bucketAgg.ID, bucketAgg.Field, func(a *es.DateHistogramAgg, b es.AggBuilder) { + a.Interval = bucketAgg.Settings.Get("interval").MustString("auto") + a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) + a.ExtendedBounds = &es.ExtendedBounds{Min: timeFrom, Max: timeTo} + a.Format = bucketAgg.Settings.Get("format").MustString(es.DateFormatEpochMS) + + if a.Interval == "auto" { + a.Interval = "$__interval" + } + + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { + a.Missing = &missing + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + aggBuilder.Histogram(bucketAgg.ID, bucketAgg.Field, func(a *es.HistogramAgg, b es.AggBuilder) { + a.Interval = bucketAgg.Settings.Get("interval").MustInt(1000) + a.MinDocCount = bucketAgg.Settings.Get("min_doc_count").MustInt(0) + + if missing, err := bucketAgg.Settings.Get("missing").Int(); err == nil { + a.Missing = &missing + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addTermsAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, metrics []*MetricAgg) es.AggBuilder { + aggBuilder.Terms(bucketAgg.ID, bucketAgg.Field, func(a *es.TermsAggregation, b es.AggBuilder) { + if size, err := bucketAgg.Settings.Get("size").Int(); err == nil { + a.Size = size + } else if size, err := bucketAgg.Settings.Get("size").String(); err == nil { + a.Size, err = strconv.Atoi(size) + if err != nil { + a.Size = 500 + } + } else { + a.Size = 500 + } + if a.Size == 0 { + a.Size = 500 + } + + if minDocCount, err := bucketAgg.Settings.Get("min_doc_count").Int(); err == nil { + a.MinDocCount = &minDocCount + } + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { + a.Missing = &missing + } + + if orderBy, err := bucketAgg.Settings.Get("orderBy").String(); err == nil { + a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc") + + if _, err := strconv.Atoi(orderBy); err == nil { + for _, m := range metrics { + if m.ID == orderBy { + b.Metric(m.ID, m.Type, m.Field, nil) + break + } + } + } + } + + aggBuilder = b + }) + + return aggBuilder +} + +func addFiltersAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + filters := make(map[string]interface{}) + for _, filter := range bucketAgg.Settings.Get("filters").MustArray() { + json := simplejson.NewFromAny(filter) + query := json.Get("query").MustString() + label := json.Get("label").MustString() + if label == "" { + label = query + } + filters[label] = &es.QueryStringFilter{Query: query, AnalyzeWildcard: true} + } + + if len(filters) > 0 { + aggBuilder.Filters(bucketAgg.ID, func(a *es.FiltersAggregation, b es.AggBuilder) { + a.Filters = filters + aggBuilder = b + }) + } + + return aggBuilder +} + +func addGeoHashGridAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder { + aggBuilder.GeoHashGrid(bucketAgg.ID, bucketAgg.Field, func(a *es.GeoHashGridAggregation, b es.AggBuilder) { + a.Precision = bucketAgg.Settings.Get("precision").MustInt(3) + aggBuilder = b + }) + + return aggBuilder +} + +type timeSeriesQueryParser struct{} + +func newTimeSeriesQueryParser() *timeSeriesQueryParser { + return &timeSeriesQueryParser{} +} + +func (p *timeSeriesQueryParser) parse(tsdbQuery *tsdb.TsdbQuery) ([]*Query, error) { + queries := make([]*Query, 0) + for _, q := range tsdbQuery.Queries { + model := q.Model + timeField, err := model.Get("timeField").String() + if err != nil { + return nil, err + } + rawQuery := model.Get("query").MustString() + bucketAggs, err := p.parseBucketAggs(model) + if err != nil { + return nil, err + } + metrics, err := p.parseMetrics(model) + if err != nil { + return nil, err + } + alias := model.Get("alias").MustString("") + interval := strconv.FormatInt(q.IntervalMs, 10) + "ms" + + queries = append(queries, &Query{ + TimeField: timeField, + RawQuery: rawQuery, + BucketAggs: bucketAggs, + Metrics: metrics, + Alias: alias, + Interval: interval, + RefID: q.RefId, + }) + } + + return queries, nil +} + +func (p *timeSeriesQueryParser) parseBucketAggs(model *simplejson.Json) ([]*BucketAgg, error) { + var err error + var result []*BucketAgg + for _, t := range model.Get("bucketAggs").MustArray() { + aggJSON := simplejson.NewFromAny(t) + agg := &BucketAgg{} + + agg.Type, err = aggJSON.Get("type").String() + if err != nil { + return nil, err + } + + agg.ID, err = aggJSON.Get("id").String() + if err != nil { + return nil, err + } + + agg.Field = aggJSON.Get("field").MustString() + agg.Settings = simplejson.NewFromAny(aggJSON.Get("settings").MustMap()) + + result = append(result, agg) + } + return result, nil +} + +func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*MetricAgg, error) { + var err error + var result []*MetricAgg + for _, t := range model.Get("metrics").MustArray() { + metricJSON := simplejson.NewFromAny(t) + metric := &MetricAgg{} + + metric.Field = metricJSON.Get("field").MustString() + metric.Hide = metricJSON.Get("hide").MustBool(false) + metric.ID = metricJSON.Get("id").MustString() + metric.PipelineAggregate = metricJSON.Get("pipelineAgg").MustString() + metric.Settings = simplejson.NewFromAny(metricJSON.Get("settings").MustMap()) + metric.Meta = simplejson.NewFromAny(metricJSON.Get("meta").MustMap()) + + metric.Type, err = metricJSON.Get("type").String() + if err != nil { + return nil, err + } + + result = append(result, metric) + } + return result, nil +} diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go new file mode 100644 index 00000000000..9660d70c318 --- /dev/null +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -0,0 +1,660 @@ +package elasticsearch + +import ( + "fmt" + "testing" + "time" + + "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + . "github.com/smartystreets/goconvey/convey" +) + +func TestExecuteTimeSeriesQuery(t *testing.T) { + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + + Convey("Test execute time series query", t, func() { + Convey("With defaults on es 2", func() { + c := newFakeClient(2) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], + "metrics": [{"type": "count", "id": "0" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) + So(rangeFilter.Key, ShouldEqual, c.timeField) + So(rangeFilter.Lte, ShouldEqual, toStr) + So(rangeFilter.Gte, ShouldEqual, fromStr) + So(rangeFilter.Format, ShouldEqual, es.DateFormatEpochMS) + So(sr.Aggs[0].Key, ShouldEqual, "2") + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) + So(dateHistogramAgg.Field, ShouldEqual, "@timestamp") + So(dateHistogramAgg.ExtendedBounds.Min, ShouldEqual, fromStr) + So(dateHistogramAgg.ExtendedBounds.Max, ShouldEqual, toStr) + }) + + Convey("With defaults on es 5", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }], + "metrics": [{"type": "count", "id": "0" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + So(sr.Query.Bool.Filters[0].(*es.RangeFilter).Key, ShouldEqual, c.timeField) + So(sr.Aggs[0].Key, ShouldEqual, "2") + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Min, ShouldEqual, fromStr) + So(sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg).ExtendedBounds.Max, ShouldEqual, toStr) + }) + + Convey("With multiple bucket aggs", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "terms", "field": "@host", "id": "2", "settings": { "size": "0", "order": "asc" } }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + termsAgg := firstLevel.Aggregation.Aggregation.(*es.TermsAggregation) + So(termsAgg.Field, ShouldEqual, "@host") + So(termsAgg.Size, ShouldEqual, 500) + secondLevel := firstLevel.Aggregation.Aggs[0] + So(secondLevel.Key, ShouldEqual, "3") + So(secondLevel.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With select field", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "2" } + ], + "metrics": [{"type": "avg", "field": "@value", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + secondLevel := firstLevel.Aggregation.Aggs[0] + So(secondLevel.Key, ShouldEqual, "1") + So(secondLevel.Aggregation.Type, ShouldEqual, "avg") + So(secondLevel.Aggregation.Aggregation.(*es.MetricAggregation).Field, ShouldEqual, "@value") + }) + + Convey("With term agg and order by metric agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "type": "terms", + "field": "@host", + "id": "2", + "settings": { "size": "5", "order": "asc", "orderBy": "5" } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + {"type": "count", "id": "1" }, + {"type": "avg", "field": "@value", "id": "5" } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + avgAggOrderBy := sr.Aggs[0].Aggregation.Aggs[0] + So(avgAggOrderBy.Key, ShouldEqual, "5") + So(avgAggOrderBy.Aggregation.Type, ShouldEqual, "avg") + + avgAgg := sr.Aggs[0].Aggregation.Aggs[1].Aggregation.Aggs[0] + So(avgAgg.Key, ShouldEqual, "5") + So(avgAgg.Aggregation.Type, ShouldEqual, "avg") + }) + + Convey("With term agg and order by term", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "type": "terms", + "field": "@host", + "id": "2", + "settings": { "size": "5", "order": "asc", "orderBy": "_term" } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + {"type": "count", "id": "1" }, + {"type": "avg", "field": "@value", "id": "5" } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + termsAgg := firstLevel.Aggregation.Aggregation.(*es.TermsAggregation) + So(termsAgg.Order["_term"], ShouldEqual, "asc") + }) + + Convey("With term agg and order by term with es6.x", func() { + c := newFakeClient(60) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "type": "terms", + "field": "@host", + "id": "2", + "settings": { "size": "5", "order": "asc", "orderBy": "_term" } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + {"type": "count", "id": "1" }, + {"type": "avg", "field": "@value", "id": "5" } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + termsAgg := firstLevel.Aggregation.Aggregation.(*es.TermsAggregation) + So(termsAgg.Order["_key"], ShouldEqual, "asc") + }) + + Convey("With metric percentiles", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "3" } + ], + "metrics": [ + { + "id": "1", + "type": "percentiles", + "field": "@load_time", + "settings": { + "percents": [ "1", "2", "3", "4" ] + } + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + percentilesAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(percentilesAgg.Key, ShouldEqual, "1") + So(percentilesAgg.Aggregation.Type, ShouldEqual, "percentiles") + metricAgg := percentilesAgg.Aggregation.Aggregation.(*es.MetricAggregation) + percents := metricAgg.Settings["percents"].([]interface{}) + So(percents, ShouldHaveLength, 4) + So(percents[0], ShouldEqual, "1") + So(percents[1], ShouldEqual, "2") + So(percents[2], ShouldEqual, "3") + So(percents[3], ShouldEqual, "4") + }) + + Convey("With filters aggs on es 2", func() { + c := newFakeClient(2) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "filters", + "settings": { + "filters": [ { "query": "@metric:cpu" }, { "query": "@metric:logins.count" } ] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + filtersAgg := sr.Aggs[0] + So(filtersAgg.Key, ShouldEqual, "2") + So(filtersAgg.Aggregation.Type, ShouldEqual, "filters") + fAgg := filtersAgg.Aggregation.Aggregation.(*es.FiltersAggregation) + So(fAgg.Filters["@metric:cpu"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:cpu") + So(fAgg.Filters["@metric:logins.count"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:logins.count") + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(dateHistogramAgg.Key, ShouldEqual, "4") + So(dateHistogramAgg.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With filters aggs on es 5", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "filters", + "settings": { + "filters": [ { "query": "@metric:cpu" }, { "query": "@metric:logins.count" } ] + } + }, + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + filtersAgg := sr.Aggs[0] + So(filtersAgg.Key, ShouldEqual, "2") + So(filtersAgg.Aggregation.Type, ShouldEqual, "filters") + fAgg := filtersAgg.Aggregation.Aggregation.(*es.FiltersAggregation) + So(fAgg.Filters["@metric:cpu"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:cpu") + So(fAgg.Filters["@metric:logins.count"].(*es.QueryStringFilter).Query, ShouldEqual, "@metric:logins.count") + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggs[0] + So(dateHistogramAgg.Key, ShouldEqual, "4") + So(dateHistogramAgg.Aggregation.Aggregation.(*es.DateHistogramAgg).Field, ShouldEqual, "@timestamp") + }) + + Convey("With raw document metric", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [], + "metrics": [{ "id": "1", "type": "raw_document", "settings": {} }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + So(sr.Size, ShouldEqual, 500) + }) + + Convey("With raw document metric size set", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [], + "metrics": [{ "id": "1", "type": "raw_document", "settings": { "size": 1337 } }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + So(sr.Size, ShouldEqual, 1337) + }) + + Convey("With date histogram agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "2", + "type": "date_histogram", + "field": "@timestamp", + "settings": { "interval": "auto", "min_doc_count": 2 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "2") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + hAgg := firstLevel.Aggregation.Aggregation.(*es.DateHistogramAgg) + So(hAgg.Field, ShouldEqual, "@timestamp") + So(hAgg.Interval, ShouldEqual, "$__interval") + So(hAgg.MinDocCount, ShouldEqual, 2) + }) + + Convey("With histogram agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "3", + "type": "histogram", + "field": "bytes", + "settings": { "interval": 10, "min_doc_count": 2, "missing": 5 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "3") + So(firstLevel.Aggregation.Type, ShouldEqual, "histogram") + hAgg := firstLevel.Aggregation.Aggregation.(*es.HistogramAgg) + So(hAgg.Field, ShouldEqual, "bytes") + So(hAgg.Interval, ShouldEqual, 10) + So(hAgg.MinDocCount, ShouldEqual, 2) + So(*hAgg.Missing, ShouldEqual, 5) + }) + + Convey("With geo hash grid agg", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "id": "3", + "type": "geohash_grid", + "field": "@location", + "settings": { "precision": 3 } + } + ], + "metrics": [{"type": "count", "id": "1" }] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "3") + So(firstLevel.Aggregation.Type, ShouldEqual, "geohash_grid") + ghGridAgg := firstLevel.Aggregation.Aggregation.(*es.GeoHashGridAggregation) + So(ghGridAgg.Field, ShouldEqual, "@location") + So(ghGridAgg.Precision, ShouldEqual, 3) + }) + + Convey("With moving average", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "moving_avg", + "field": "3", + "pipelineAgg": "3" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + So(firstLevel.Aggregation.Aggs, ShouldHaveLength, 2) + + sumAgg := firstLevel.Aggregation.Aggs[0] + So(sumAgg.Key, ShouldEqual, "3") + So(sumAgg.Aggregation.Type, ShouldEqual, "sum") + mAgg := sumAgg.Aggregation.Aggregation.(*es.MetricAggregation) + So(mAgg.Field, ShouldEqual, "@value") + + movingAvgAgg := firstLevel.Aggregation.Aggs[1] + So(movingAvgAgg.Key, ShouldEqual, "2") + So(movingAvgAgg.Aggregation.Type, ShouldEqual, "moving_avg") + pl := movingAvgAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(pl.BucketPath, ShouldEqual, "3") + }) + + Convey("With broken moving average", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "5" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "moving_avg", + "pipelineAgg": "3" + }, + { + "id": "4", + "type": "moving_avg", + "pipelineAgg": "Metric to apply moving average" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "5") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + So(firstLevel.Aggregation.Aggs, ShouldHaveLength, 2) + + movingAvgAgg := firstLevel.Aggregation.Aggs[1] + So(movingAvgAgg.Key, ShouldEqual, "2") + plAgg := movingAvgAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath, ShouldEqual, "3") + }) + + Convey("With derivative", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { + "id": "2", + "type": "derivative", + "pipelineAgg": "3" + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + derivativeAgg := firstLevel.Aggregation.Aggs[1] + So(derivativeAgg.Key, ShouldEqual, "2") + plAgg := derivativeAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath, ShouldEqual, "3") + }) + + }) +} + +type fakeClient struct { + version int + timeField string + multiSearchResponse *es.MultiSearchResponse + multiSearchError error + builder *es.MultiSearchRequestBuilder + multisearchRequests []*es.MultiSearchRequest +} + +func newFakeClient(version int) *fakeClient { + return &fakeClient{ + version: version, + timeField: "@timestamp", + multisearchRequests: make([]*es.MultiSearchRequest, 0), + multiSearchResponse: &es.MultiSearchResponse{}, + } +} + +func (c *fakeClient) GetVersion() int { + return c.version +} + +func (c *fakeClient) GetTimeField() string { + return c.timeField +} + +func (c *fakeClient) GetMinInterval(queryInterval string) (time.Duration, error) { + return 15 * time.Second, nil +} + +func (c *fakeClient) ExecuteMultisearch(r *es.MultiSearchRequest) (*es.MultiSearchResponse, error) { + c.multisearchRequests = append(c.multisearchRequests, r) + return c.multiSearchResponse, c.multiSearchError +} + +func (c *fakeClient) MultiSearch() *es.MultiSearchRequestBuilder { + c.builder = es.NewMultiSearchRequestBuilder(c.version) + return c.builder +} + +func newTsdbQuery(body string) (*tsdb.TsdbQuery, error) { + json, err := simplejson.NewJson([]byte(body)) + if err != nil { + return nil, err + } + return &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: json, + }, + }, + }, nil +} + +func executeTsdbQuery(c es.Client, body string, from, to time.Time, minInterval time.Duration) (*tsdb.Response, error) { + json, err := simplejson.NewJson([]byte(body)) + if err != nil { + return nil, err + } + fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond)) + toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) + tsdbQuery := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: json, + }, + }, + TimeRange: tsdb.NewTimeRange(fromStr, toStr), + } + query := newTimeSeriesQuery(c, tsdbQuery, tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: minInterval})) + return query.execute() +} + +func TestTimeSeriesQueryParser(t *testing.T) { + Convey("Test time series query parser", t, func() { + p := newTimeSeriesQueryParser() + + Convey("Should be able to parse query", func() { + body := `{ + "timeField": "@timestamp", + "query": "@metric:cpu", + "alias": "{{@hostname}} {{metric}}", + "metrics": [ + { + "field": "@value", + "id": "1", + "meta": {}, + "settings": { + "percents": [ + "90" + ] + }, + "type": "percentiles" + }, + { + "type": "count", + "field": "select field", + "id": "4", + "settings": {}, + "meta": {} + } + ], + "bucketAggs": [ + { + "fake": true, + "field": "@hostname", + "id": "3", + "settings": { + "min_doc_count": 1, + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "5m", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ] + }` + tsdbQuery, err := newTsdbQuery(body) + So(err, ShouldBeNil) + queries, err := p.parse(tsdbQuery) + So(err, ShouldBeNil) + So(queries, ShouldHaveLength, 1) + + q := queries[0] + + So(q.TimeField, ShouldEqual, "@timestamp") + So(q.RawQuery, ShouldEqual, "@metric:cpu") + So(q.Alias, ShouldEqual, "{{@hostname}} {{metric}}") + + So(q.Metrics, ShouldHaveLength, 2) + So(q.Metrics[0].Field, ShouldEqual, "@value") + So(q.Metrics[0].ID, ShouldEqual, "1") + So(q.Metrics[0].Type, ShouldEqual, "percentiles") + So(q.Metrics[0].Hide, ShouldBeFalse) + So(q.Metrics[0].PipelineAggregate, ShouldEqual, "") + So(q.Metrics[0].Settings.Get("percents").MustStringArray()[0], ShouldEqual, "90") + + So(q.Metrics[1].Field, ShouldEqual, "select field") + So(q.Metrics[1].ID, ShouldEqual, "4") + So(q.Metrics[1].Type, ShouldEqual, "count") + So(q.Metrics[1].Hide, ShouldBeFalse) + So(q.Metrics[1].PipelineAggregate, ShouldEqual, "") + So(q.Metrics[1].Settings.MustMap(), ShouldBeEmpty) + + So(q.BucketAggs, ShouldHaveLength, 2) + So(q.BucketAggs[0].Field, ShouldEqual, "@hostname") + So(q.BucketAggs[0].ID, ShouldEqual, "3") + So(q.BucketAggs[0].Type, ShouldEqual, "terms") + So(q.BucketAggs[0].Settings.Get("min_doc_count").MustInt64(), ShouldEqual, 1) + So(q.BucketAggs[0].Settings.Get("order").MustString(), ShouldEqual, "desc") + So(q.BucketAggs[0].Settings.Get("orderBy").MustString(), ShouldEqual, "_term") + So(q.BucketAggs[0].Settings.Get("size").MustString(), ShouldEqual, "10") + + So(q.BucketAggs[1].Field, ShouldEqual, "@timestamp") + So(q.BucketAggs[1].ID, ShouldEqual, "2") + So(q.BucketAggs[1].Type, ShouldEqual, "date_histogram") + So(q.BucketAggs[1].Settings.Get("interval").MustString(), ShouldEqual, "5m") + So(q.BucketAggs[1].Settings.Get("min_doc_count").MustInt64(), ShouldEqual, 0) + So(q.BucketAggs[1].Settings.Get("trimEdges").MustInt64(), ShouldEqual, 0) + }) + }) +} diff --git a/pkg/tsdb/graphite/graphite.go b/pkg/tsdb/graphite/graphite.go index 73b173813af..ff0ed8d0620 100644 --- a/pkg/tsdb/graphite/graphite.go +++ b/pkg/tsdb/graphite/graphite.go @@ -28,12 +28,9 @@ func NewGraphiteExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, return &GraphiteExecutor{}, nil } -var ( - glog log.Logger -) +var glog = log.New("tsdb.graphite") func init() { - glog = log.New("tsdb.graphite") tsdb.RegisterTsdbQueryEndpoint("graphite", NewGraphiteExecutor) } @@ -52,6 +49,7 @@ func (e *GraphiteExecutor) Query(ctx context.Context, dsInfo *models.DataSource, } for _, query := range tsdbQuery.Queries { + glog.Info("graphite", "query", query.Model) if fullTarget, err := query.Model.Get("targetFull").String(); err == nil { target = fixIntervalFormat(fullTarget) } else { @@ -79,6 +77,9 @@ func (e *GraphiteExecutor) Query(ctx context.Context, dsInfo *models.DataSource, span.SetTag("target", target) span.SetTag("from", from) span.SetTag("until", until) + span.SetTag("datasource_id", dsInfo.Id) + span.SetTag("org_id", dsInfo.OrgId) + defer span.Finish() opentracing.GlobalTracer().Inject( @@ -163,14 +164,12 @@ func formatTimeRange(input string) string { func fixIntervalFormat(target string) string { rMinute := regexp.MustCompile(`'(\d+)m'`) - rMin := regexp.MustCompile("m") target = rMinute.ReplaceAllStringFunc(target, func(m string) string { - return rMin.ReplaceAllString(m, "min") + return strings.Replace(m, "m", "min", -1) }) rMonth := regexp.MustCompile(`'(\d+)M'`) - rMon := regexp.MustCompile("M") target = rMonth.ReplaceAllStringFunc(target, func(M string) string { - return rMon.ReplaceAllString(M, "mon") + return strings.Replace(M, "M", "mon", -1) }) return target } diff --git a/pkg/tsdb/influxdb/influxdb.go b/pkg/tsdb/influxdb/influxdb.go index 6100d3b401e..ec1e9ff01bd 100644 --- a/pkg/tsdb/influxdb/influxdb.go +++ b/pkg/tsdb/influxdb/influxdb.go @@ -96,16 +96,15 @@ func (e *InfluxDBExecutor) Query(ctx context.Context, dsInfo *models.DataSource, } func (e *InfluxDBExecutor) getQuery(dsInfo *models.DataSource, queries []*tsdb.Query, context *tsdb.TsdbQuery) (*Query, error) { - for _, v := range queries { - - query, err := e.QueryParser.Parse(v.Model, dsInfo) + // The model supports multiple queries, but right now this is only used from + // alerting so we only needed to support batch executing 1 query at a time. + if len(queries) > 0 { + query, err := e.QueryParser.Parse(queries[0].Model, dsInfo) if err != nil { return nil, err } - return query, nil } - return nil, fmt.Errorf("query request contains no queries") } diff --git a/pkg/tsdb/influxdb/model_parser.go b/pkg/tsdb/influxdb/model_parser.go index deb2f15e3ce..f1113511bae 100644 --- a/pkg/tsdb/influxdb/model_parser.go +++ b/pkg/tsdb/influxdb/model_parser.go @@ -40,6 +40,9 @@ func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.Data } parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond*1) + if err != nil { + return nil, err + } return &Query{ Measurement: measurement, diff --git a/pkg/tsdb/influxdb/query.go b/pkg/tsdb/influxdb/query.go index 0a16a507877..7cb8f0ecd82 100644 --- a/pkg/tsdb/influxdb/query.go +++ b/pkg/tsdb/influxdb/query.go @@ -4,7 +4,6 @@ import ( "fmt" "strconv" "strings" - "time" "regexp" @@ -12,8 +11,8 @@ import ( ) var ( - regexpOperatorPattern *regexp.Regexp = regexp.MustCompile(`^\/.*\/$`) - regexpMeasurementPattern *regexp.Regexp = regexp.MustCompile(`^\/.*\/$`) + regexpOperatorPattern = regexp.MustCompile(`^\/.*\/$`) + regexpMeasurementPattern = regexp.MustCompile(`^\/.*\/$`) ) func (query *Query) Build(queryContext *tsdb.TsdbQuery) (string, error) { @@ -34,7 +33,7 @@ func (query *Query) Build(queryContext *tsdb.TsdbQuery) (string, error) { res = strings.Replace(res, "$timeFilter", query.renderTimeFilter(queryContext), -1) res = strings.Replace(res, "$interval", interval.Text, -1) - res = strings.Replace(res, "$__interval_ms", strconv.FormatInt(interval.Value.Nanoseconds()/int64(time.Millisecond), 10), -1) + res = strings.Replace(res, "$__interval_ms", strconv.FormatInt(interval.Milliseconds(), 10), -1) res = strings.Replace(res, "$__interval", interval.Text, -1) return res, nil } @@ -62,9 +61,8 @@ func (query *Query) renderTags() []string { } } - textValue := "" - // quote value unless regex or number + var textValue string if tag.Operator == "=~" || tag.Operator == "!~" { textValue = tag.Value } else if tag.Operator == "<" || tag.Operator == ">" { @@ -107,7 +105,7 @@ func (query *Query) renderSelectors(queryContext *tsdb.TsdbQuery) string { } func (query *Query) renderMeasurement() string { - policy := "" + var policy string if query.Policy == "" || query.Policy == "default" { policy = "" } else { diff --git a/pkg/tsdb/influxdb/query_part.go b/pkg/tsdb/influxdb/query_part.go index 981aea40526..77f565a8597 100644 --- a/pkg/tsdb/influxdb/query_part.go +++ b/pkg/tsdb/influxdb/query_part.go @@ -31,6 +31,7 @@ func init() { renders["mean"] = QueryDefinition{Renderer: functionRenderer} renders["median"] = QueryDefinition{Renderer: functionRenderer} renders["sum"] = QueryDefinition{Renderer: functionRenderer} + renders["mode"] = QueryDefinition{Renderer: functionRenderer} renders["holt_winters"] = QueryDefinition{ Renderer: functionRenderer, diff --git a/pkg/tsdb/influxdb/query_part_test.go b/pkg/tsdb/influxdb/query_part_test.go index d23865174c8..08bcff9b727 100644 --- a/pkg/tsdb/influxdb/query_part_test.go +++ b/pkg/tsdb/influxdb/query_part_test.go @@ -4,77 +4,39 @@ import ( "testing" "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" ) func TestInfluxdbQueryPart(t *testing.T) { - Convey("Influxdb query parts", t, func() { + tcs := []struct { + mode string + input string + params []string + expected string + }{ + {mode: "field", params: []string{"value"}, input: "value", expected: `"value"`}, + {mode: "derivative", params: []string{"10s"}, input: "mean(value)", expected: `derivative(mean(value), 10s)`}, + {mode: "bottom", params: []string{"3"}, input: "value", expected: `bottom(value, 3)`}, + {mode: "time", params: []string{"$interval"}, input: "", expected: `time($interval)`}, + {mode: "time", params: []string{"auto"}, input: "", expected: `time($__interval)`}, + {mode: "spread", params: []string{}, input: "value", expected: `spread(value)`}, + {mode: "math", params: []string{"/ 100"}, input: "mean(value)", expected: `mean(value) / 100`}, + {mode: "alias", params: []string{"test"}, input: "mean(value)", expected: `mean(value) AS "test"`}, + {mode: "count", params: []string{}, input: "distinct(value)", expected: `count(distinct(value))`}, + {mode: "mode", params: []string{}, input: "value", expected: `mode(value)`}, + } - queryContext := &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("5m", "now")} - query := &Query{} + queryContext := &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("5m", "now")} + query := &Query{} - Convey("render field ", func() { - part, err := NewQueryPart("field", []string{"value"}) - So(err, ShouldBeNil) + for _, tc := range tcs { + part, err := NewQueryPart(tc.mode, tc.params) + if err != nil { + t.Errorf("Expected NewQueryPart to not return an error. error: %v", err) + } - res := part.Render(query, queryContext, "value") - So(res, ShouldEqual, `"value"`) - }) - - Convey("render nested part", func() { - part, err := NewQueryPart("derivative", []string{"10s"}) - So(err, ShouldBeNil) - - res := part.Render(query, queryContext, "mean(value)") - So(res, ShouldEqual, "derivative(mean(value), 10s)") - }) - - Convey("render bottom", func() { - part, err := NewQueryPart("bottom", []string{"3"}) - So(err, ShouldBeNil) - - res := part.Render(query, queryContext, "value") - So(res, ShouldEqual, "bottom(value, 3)") - }) - - Convey("render time with $interval", func() { - part, err := NewQueryPart("time", []string{"$interval"}) - So(err, ShouldBeNil) - - res := part.Render(query, queryContext, "") - So(res, ShouldEqual, "time($interval)") - }) - - Convey("render time with auto", func() { - part, err := NewQueryPart("time", []string{"auto"}) - So(err, ShouldBeNil) - - res := part.Render(query, queryContext, "") - So(res, ShouldEqual, "time($__interval)") - }) - - Convey("render spread", func() { - part, err := NewQueryPart("spread", []string{}) - So(err, ShouldBeNil) - - res := part.Render(query, queryContext, "value") - So(res, ShouldEqual, `spread(value)`) - }) - - Convey("render suffix", func() { - part, err := NewQueryPart("math", []string{"/ 100"}) - So(err, ShouldBeNil) - - res := part.Render(query, queryContext, "mean(value)") - So(res, ShouldEqual, "mean(value) / 100") - }) - - Convey("render alias", func() { - part, err := NewQueryPart("alias", []string{"test"}) - So(err, ShouldBeNil) - - res := part.Render(query, queryContext, "mean(value)") - So(res, ShouldEqual, `mean(value) AS "test"`) - }) - }) + res := part.Render(query, queryContext, tc.input) + if res != tc.expected { + t.Errorf("expected %v to render into %s", tc, tc.expected) + } + } } diff --git a/pkg/tsdb/influxdb/query_test.go b/pkg/tsdb/influxdb/query_test.go index f1270560269..cc1358a72d7 100644 --- a/pkg/tsdb/influxdb/query_test.go +++ b/pkg/tsdb/influxdb/query_test.go @@ -158,7 +158,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { So(strings.Join(query.renderTags(), ""), ShouldEqual, `"key" < 10001`) }) - Convey("can render number greather then condition tags", func() { + Convey("can render number greater then condition tags", func() { query := &Query{Tags: []*Tag{{Operator: ">", Value: "10001", Key: "key"}}} So(strings.Join(query.renderTags(), ""), ShouldEqual, `"key" > 10001`) diff --git a/pkg/tsdb/influxdb/response_parser_test.go b/pkg/tsdb/influxdb/response_parser_test.go index a517cf4d71f..d8ec6e145c7 100644 --- a/pkg/tsdb/influxdb/response_parser_test.go +++ b/pkg/tsdb/influxdb/response_parser_test.go @@ -13,7 +13,8 @@ func TestInfluxdbResponseParser(t *testing.T) { Convey("Response parser", func() { parser := &ResponseParser{} - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../../", }) diff --git a/pkg/tsdb/interval.go b/pkg/tsdb/interval.go index e26d39f3986..fd6adee39d7 100644 --- a/pkg/tsdb/interval.go +++ b/pkg/tsdb/interval.go @@ -10,10 +10,10 @@ import ( ) var ( - defaultRes int64 = 1500 - defaultMinInterval time.Duration = 1 * time.Millisecond - year time.Duration = time.Hour * 24 * 365 - day time.Duration = time.Hour * 24 + defaultRes int64 = 1500 + defaultMinInterval = time.Millisecond * 1 + year = time.Hour * 24 * 365 + day = time.Hour * 24 ) type Interval struct { @@ -49,6 +49,10 @@ func NewIntervalCalculator(opt *IntervalOptions) *intervalCalculator { return calc } +func (i *Interval) Milliseconds() int64 { + return i.Value.Nanoseconds() / int64(time.Millisecond) +} + func (ic *intervalCalculator) Calculate(timerange *TimeRange, minInterval time.Duration) Interval { to := timerange.MustGetTo().UnixNano() from := timerange.MustGetFrom().UnixNano() diff --git a/pkg/tsdb/interval_test.go b/pkg/tsdb/interval_test.go index 1e36e5428fe..941b08dd554 100644 --- a/pkg/tsdb/interval_test.go +++ b/pkg/tsdb/interval_test.go @@ -10,7 +10,8 @@ import ( func TestInterval(t *testing.T) { Convey("Default interval ", t, func() { - setting.NewConfigContext(&setting.CommandLineArgs{ + cfg := setting.NewCfg() + cfg.Load(&setting.CommandLineArgs{ HomePath: "../../", }) diff --git a/pkg/tsdb/mssql/macros.go b/pkg/tsdb/mssql/macros.go index 92c6ede148e..0a260f7ad70 100644 --- a/pkg/tsdb/mssql/macros.go +++ b/pkg/tsdb/mssql/macros.go @@ -6,30 +6,29 @@ import ( "strings" "time" - "strconv" - "github.com/grafana/grafana/pkg/tsdb" ) const rsIdentifier = `([_a-zA-Z0-9]+)` const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)` -type MsSqlMacroEngine struct { - TimeRange *tsdb.TimeRange - Query *tsdb.Query +type msSqlMacroEngine struct { + *tsdb.SqlMacroEngineBase + timeRange *tsdb.TimeRange + query *tsdb.Query } -func NewMssqlMacroEngine() tsdb.SqlMacroEngine { - return &MsSqlMacroEngine{} +func newMssqlMacroEngine() tsdb.SqlMacroEngine { + return &msSqlMacroEngine{SqlMacroEngineBase: tsdb.NewSqlMacroEngineBase()} } -func (m *MsSqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { - m.TimeRange = timeRange - m.Query = query +func (m *msSqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { + m.timeRange = timeRange + m.query = query rExp, _ := regexp.Compile(sExpr) var macroError error - sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { + sql = m.ReplaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { args := strings.Split(groups[2], ",") for i, arg := range args { args[i] = strings.Trim(arg, " ") @@ -49,49 +48,24 @@ func (m *MsSqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRa return sql, nil } -func replaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string { - result := "" - lastIndex := 0 - - for _, v := range re.FindAllSubmatchIndex([]byte(str), -1) { - groups := []string{} - for i := 0; i < len(v); i += 2 { - groups = append(groups, str[v[i]:v[i+1]]) - } - - result += str[lastIndex:v[0]] + repl(groups) - lastIndex = v[1] - } - - return result + str[lastIndex:] -} - -func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, error) { +func (m *msSqlMacroEngine) evaluateMacro(name string, args []string) (string, error) { switch name { case "__time": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } return fmt.Sprintf("%s AS time", args[0]), nil - case "__utcTime": - if len(args) == 0 { - return "", fmt.Errorf("missing time column argument for macro %v", name) - } - return fmt.Sprintf("DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), %s) AS time", args[0]), nil case "__timeEpoch": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), %s) ) AS time", args[0]), nil + return fmt.Sprintf("DATEDIFF(second, '1970-01-01', %s) AS time", args[0]), nil case "__timeFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= DATEADD(s, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01') AND %s <= DATEADD(s, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil - case "__timeFrom": - return fmt.Sprintf("DATEADD(second, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil - case "__timeTo": - return fmt.Sprintf("DATEADD(second, %d+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + + return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], m.timeRange.GetFromAsTimeUTC().Format(time.RFC3339), m.timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -101,28 +75,44 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er return "", fmt.Errorf("error parsing interval %v", args[1]) } if len(args) == 3 { - m.Query.Model.Set("fill", true) - m.Query.Model.Set("fillInterval", interval.Seconds()) - if args[2] == "NULL" { - m.Query.Model.Set("fillNull", true) - } else { - floatVal, err := strconv.ParseFloat(args[2], 64) - if err != nil { - return "", fmt.Errorf("error parsing fill value %v", args[2]) - } - m.Query.Model.Set("fillValue", floatVal) + err := tsdb.SetupFillmode(m.query, interval, args[2]) + if err != nil { + return "", err } } - return fmt.Sprintf("cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), %s))/%.0f as int)*%.0f as int)", args[0], interval.Seconds(), interval.Seconds()), nil + return fmt.Sprintf("FLOOR(DATEDIFF(second, '1970-01-01', %s)/%.0f)*%.0f", args[0], interval.Seconds(), interval.Seconds()), nil + case "__timeGroupAlias": + tg, err := m.evaluateMacro("__timeGroup", args) + if err == nil { + return tg + " AS [time]", err + } + return "", err case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil - case "__unixEpochFrom": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil - case "__unixEpochTo": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], m.timeRange.GetFromAsSecondsEpoch(), args[0], m.timeRange.GetToAsSecondsEpoch()), nil + case "__unixEpochGroup": + if len(args) < 2 { + return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name) + } + interval, err := time.ParseDuration(strings.Trim(args[1], `'`)) + if err != nil { + return "", fmt.Errorf("error parsing interval %v", args[1]) + } + if len(args) == 3 { + err := tsdb.SetupFillmode(m.query, interval, args[2]) + if err != nil { + return "", err + } + } + return fmt.Sprintf("FLOOR(%s/%v)*%v", args[0], interval.Seconds(), interval.Seconds()), nil + case "__unixEpochGroupAlias": + tg, err := m.evaluateMacro("__unixEpochGroup", args) + if err == nil { + return tg + " AS [time]", err + } + return "", err default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/mssql/macros_test.go b/pkg/tsdb/mssql/macros_test.go index db1f5670924..7456238efa4 100644 --- a/pkg/tsdb/mssql/macros_test.go +++ b/pkg/tsdb/mssql/macros_test.go @@ -1,6 +1,8 @@ package mssql import ( + "fmt" + "strconv" "testing" "time" @@ -12,120 +14,161 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { - engine := &MsSqlMacroEngine{} - timeRange := &tsdb.TimeRange{From: "5m", To: "now"} + engine := &msSqlMacroEngine{} query := &tsdb.Query{ Model: simplejson.New(), } - Convey("interpolate __time function", func() { - sql, err := engine.Interpolate(query, nil, "select $__time(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { + from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) + to := from.Add(5 * time.Minute) + timeRange := tsdb.NewFakeTimeRange("5m", "now", to) + + Convey("interpolate __time function", func() { + sql, err := engine.Interpolate(query, nil, "select $__time(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select time_column AS time") + }) + + Convey("interpolate __timeEpoch function", func() { + sql, err := engine.Interpolate(query, nil, "select $__timeEpoch(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select DATEDIFF(second, '1970-01-01', time_column) AS time") + }) + + Convey("interpolate __timeEpoch function wrapped in aggregation", func() { + sql, err := engine.Interpolate(query, nil, "select min($__timeEpoch(time_column))") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select min(DATEDIFF(second, '1970-01-01', time_column) AS time)") + }) + + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN '%s' AND '%s'", from.Format(time.RFC3339), to.Format(time.RFC3339))) + }) + + Convey("interpolate __timeGroup function", func() { + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroupAlias(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY FLOOR(DATEDIFF(second, '1970-01-01', time_column)/300)*300") + So(sql2, ShouldEqual, sql+" AS [time]") + }) + + Convey("interpolate __timeGroup function with spaces around arguments", func() { + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroupAlias(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY FLOOR(DATEDIFF(second, '1970-01-01', time_column)/300)*300") + So(sql2, ShouldEqual, sql+" AS [time]") + }) + + Convey("interpolate __timeGroup function with fill (value = NULL)", func() { + _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', NULL)") + + fill := query.Model.Get("fill").MustBool() + fillMode := query.Model.Get("fillMode").MustString() + fillInterval := query.Model.Get("fillInterval").MustInt() + + So(err, ShouldBeNil) + So(fill, ShouldBeTrue) + So(fillMode, ShouldEqual, "null") + So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) + }) + + Convey("interpolate __timeGroup function with fill (value = previous)", func() { + _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', previous)") + + fill := query.Model.Get("fill").MustBool() + fillMode := query.Model.Get("fillMode").MustString() + fillInterval := query.Model.Get("fillInterval").MustInt() + + So(err, ShouldBeNil) + So(fill, ShouldBeTrue) + So(fillMode, ShouldEqual, "previous") + So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) + }) + + Convey("interpolate __timeGroup function with fill (value = float)", func() { + _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', 1.5)") + + fill := query.Model.Get("fill").MustBool() + fillValue := query.Model.Get("fillValue").MustFloat64() + fillInterval := query.Model.Get("fillInterval").MustInt() + + So(err, ShouldBeNil) + So(fill, ShouldBeTrue) + So(fillValue, ShouldEqual, 1.5) + So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time_column >= %d AND time_column <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochGroup function", func() { + + sql, err := engine.Interpolate(query, timeRange, "SELECT $__unixEpochGroup(time_column,'5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "SELECT $__unixEpochGroupAlias(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "SELECT FLOOR(time_column/300)*300") + So(sql2, ShouldEqual, sql+" AS [time]") + }) - So(sql, ShouldEqual, "select time_column AS time") }) - Convey("interpolate __utcTime function", func() { - sql, err := engine.Interpolate(query, nil, "select $__utcTime(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column) AS time") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN '%s' AND '%s'", from.Format(time.RFC3339), to.Format(time.RFC3339))) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time_column >= %d AND time_column <= %d", from.Unix(), to.Unix())) + }) }) - Convey("interpolate __timeEpoch function", func() { - sql, err := engine.Interpolate(query, nil, "select $__timeEpoch(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column) ) AS time") - }) + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeEpoch function wrapped in aggregation", func() { - sql, err := engine.Interpolate(query, nil, "select min($__timeEpoch(time_column))") - So(err, ShouldBeNil) + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN '%s' AND '%s'", from.Format(time.RFC3339), to.Format(time.RFC3339))) + }) - So(sql, ShouldEqual, "select min(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column) ) AS time)") - }) + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "WHERE time_column >= DATEADD(s, 18446744066914186738+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01') AND time_column <= DATEADD(s, 18446744066914187038+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')") - }) - - Convey("interpolate __timeGroup function", func() { - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column))/300 as int)*300 as int)") - }) - - Convey("interpolate __timeGroup function with spaces around arguments", func() { - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time_column))/300 as int)*300 as int)") - }) - - Convey("interpolate __timeGroup function with fill (value = NULL)", func() { - _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', NULL)") - - fill := query.Model.Get("fill").MustBool() - fillNull := query.Model.Get("fillNull").MustBool() - fillInterval := query.Model.Get("fillInterval").MustInt() - - So(err, ShouldBeNil) - So(fill, ShouldBeTrue) - So(fillNull, ShouldBeTrue) - So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) - }) - - Convey("interpolate __timeGroup function with fill (value = float)", func() { - _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', 1.5)") - - fill := query.Model.Get("fill").MustBool() - fillValue := query.Model.Get("fillValue").MustFloat64() - fillInterval := query.Model.Get("fillInterval").MustInt() - - So(err, ShouldBeNil) - So(fill, ShouldBeTrue) - So(fillValue, ShouldEqual, 1.5) - So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) - }) - - Convey("interpolate __timeFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select DATEADD(second, 18446744066914186738+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')") - }) - - Convey("interpolate __timeTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select DATEADD(second, 18446744066914187038+DATEDIFF(second,GETUTCDATE(),GETDATE()), '1970-01-01')") - }) - - Convey("interpolate __unixEpochFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(18446744066914186738)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038") - }) - - Convey("interpolate __unixEpochFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738") - }) - - Convey("interpolate __unixEpochTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914187038") + So(sql, ShouldEqual, fmt.Sprintf("select time_column >= %d AND time_column <= %d", from.Unix(), to.Unix())) + }) }) }) } diff --git a/pkg/tsdb/mssql/mssql.go b/pkg/tsdb/mssql/mssql.go index af68ca0424e..469d6baa5de 100644 --- a/pkg/tsdb/mssql/mssql.go +++ b/pkg/tsdb/mssql/mssql.go @@ -1,51 +1,40 @@ package mssql import ( - "container/list" - "context" "database/sql" "fmt" "strconv" "strings" - "time" - - "math" - _ "github.com/denisenkom/go-mssqldb" "github.com/go-xorm/core" - "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" ) -type MssqlQueryEndpoint struct { - sqlEngine tsdb.SqlEngine - log log.Logger -} - func init() { - tsdb.RegisterTsdbQueryEndpoint("mssql", NewMssqlQueryEndpoint) + tsdb.RegisterTsdbQueryEndpoint("mssql", newMssqlQueryEndpoint) } -func NewMssqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { - endpoint := &MssqlQueryEndpoint{ - log: log.New("tsdb.mssql"), - } - - endpoint.sqlEngine = &tsdb.DefaultSqlEngine{ - MacroEngine: NewMssqlMacroEngine(), - } +func newMssqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + logger := log.New("tsdb.mssql") cnnstr := generateConnectionString(datasource) - endpoint.log.Debug("getEngine", "connection", cnnstr) + logger.Debug("getEngine", "connection", cnnstr) - if err := endpoint.sqlEngine.InitEngine("mssql", datasource, cnnstr); err != nil { - return nil, err + config := tsdb.SqlQueryEndpointConfiguration{ + DriverName: "mssql", + ConnectionString: cnnstr, + Datasource: datasource, + MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"}, } - return endpoint, nil + rowTransformer := mssqlRowTransformer{ + log: logger, + } + + return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newMssqlMacroEngine(), logger) } func generateConnectionString(datasource *models.DataSource) string { @@ -63,85 +52,30 @@ func generateConnectionString(datasource *models.DataSource) string { } server, port := hostParts[0], hostParts[1] - return fmt.Sprintf("server=%s;port=%s;database=%s;user id=%s;password=%s;", + encrypt := datasource.JsonData.Get("encrypt").MustString("false") + connStr := fmt.Sprintf("server=%s;port=%s;database=%s;user id=%s;password=%s;", server, port, datasource.Database, datasource.User, password, ) + if encrypt != "false" { + connStr += fmt.Sprintf("encrypt=%s;", encrypt) + } + return connStr } -// Query is the main function for the MssqlQueryEndpoint -func (e *MssqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable) +type mssqlRowTransformer struct { + log log.Logger } -func (e MssqlQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error { - columnNames, err := rows.Columns() - columnCount := len(columnNames) +func (t *mssqlRowTransformer) Transform(columnTypes []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) { + values := make([]interface{}, len(columnTypes)) + valuePtrs := make([]interface{}, len(columnTypes)) - if err != nil { - return err - } - - rowLimit := 1000000 - rowCount := 0 - timeIndex := -1 - - table := &tsdb.Table{ - Columns: make([]tsdb.TableColumn, columnCount), - Rows: make([]tsdb.RowValues, 0), - } - - for i, name := range columnNames { - table.Columns[i].Text = name - - // check if there is a column named time - switch name { - case "time": - timeIndex = i - } - } - - columnTypes, err := rows.ColumnTypes() - if err != nil { - return err - } - - for ; rows.Next(); rowCount++ { - if rowCount > rowLimit { - return fmt.Errorf("MsSQL query row limit exceeded, limit %d", rowLimit) - } - - values, err := e.getTypedRowData(columnTypes, rows) - if err != nil { - return err - } - - // convert column named time to unix timestamp to make - // native datetime mssql types work in annotation queries - if timeIndex != -1 { - switch value := values[timeIndex].(type) { - case time.Time: - values[timeIndex] = float64(value.Unix()) - } - } - - table.Rows = append(table.Rows, values) - } - - result.Tables = append(result.Tables, table) - result.Meta.Set("rowCount", rowCount) - return nil -} - -func (e MssqlQueryEndpoint) getTypedRowData(types []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) { - values := make([]interface{}, len(types)) - valuePtrs := make([]interface{}, len(types)) - - for i, stype := range types { - e.log.Debug("type", "type", stype) + for i, stype := range columnTypes { + t.log.Debug("type", "type", stype) valuePtrs[i] = &values[i] } @@ -151,17 +85,17 @@ func (e MssqlQueryEndpoint) getTypedRowData(types []*sql.ColumnType, rows *core. // convert types not handled by denisenkom/go-mssqldb // unhandled types are returned as []byte - for i := 0; i < len(types); i++ { - if value, ok := values[i].([]byte); ok == true { - switch types[i].DatabaseTypeName() { + for i := 0; i < len(columnTypes); i++ { + if value, ok := values[i].([]byte); ok { + switch columnTypes[i].DatabaseTypeName() { case "MONEY", "SMALLMONEY", "DECIMAL": if v, err := strconv.ParseFloat(string(value), 64); err == nil { values[i] = v } else { - e.log.Debug("Rows", "Error converting numeric to float", value) + t.log.Debug("Rows", "Error converting numeric to float", value) } default: - e.log.Debug("Rows", "Unknown database type", types[i].DatabaseTypeName(), "value", value) + t.log.Debug("Rows", "Unknown database type", columnTypes[i].DatabaseTypeName(), "value", value) values[i] = string(value) } } @@ -169,160 +103,3 @@ func (e MssqlQueryEndpoint) getTypedRowData(types []*sql.ColumnType, rows *core. return values, nil } - -func (e MssqlQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error { - pointsBySeries := make(map[string]*tsdb.TimeSeries) - seriesByQueryOrder := list.New() - - columnNames, err := rows.Columns() - if err != nil { - return err - } - - columnTypes, err := rows.ColumnTypes() - if err != nil { - return err - } - - rowLimit := 1000000 - rowCount := 0 - timeIndex := -1 - metricIndex := -1 - - // check columns of resultset: a column named time is mandatory - // the first text column is treated as metric name unless a column named metric is present - for i, col := range columnNames { - switch col { - case "time": - timeIndex = i - case "metric": - metricIndex = i - default: - if metricIndex == -1 { - switch columnTypes[i].DatabaseTypeName() { - case "VARCHAR", "CHAR", "NVARCHAR", "NCHAR": - metricIndex = i - } - } - } - } - - if timeIndex == -1 { - return fmt.Errorf("Found no column named time") - } - - fillMissing := query.Model.Get("fill").MustBool(false) - var fillInterval float64 - fillValue := null.Float{} - if fillMissing { - fillInterval = query.Model.Get("fillInterval").MustFloat64() * 1000 - if query.Model.Get("fillNull").MustBool(false) == false { - fillValue.Float64 = query.Model.Get("fillValue").MustFloat64() - fillValue.Valid = true - } - } - - for rows.Next() { - var timestamp float64 - var value null.Float - var metric string - - if rowCount > rowLimit { - return fmt.Errorf("MSSQL query row limit exceeded, limit %d", rowLimit) - } - - values, err := e.getTypedRowData(columnTypes, rows) - if err != nil { - return err - } - - switch columnValue := values[timeIndex].(type) { - case int64: - timestamp = float64(columnValue * 1000) - case float64: - timestamp = columnValue * 1000 - case time.Time: - timestamp = (float64(columnValue.Unix()) * 1000) + float64(columnValue.Nanosecond()/1e6) // in case someone is trying to map times beyond 2262 :D - default: - return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp") - } - - if metricIndex >= 0 { - if columnValue, ok := values[metricIndex].(string); ok == true { - metric = columnValue - } else { - return fmt.Errorf("Column metric must be of type CHAR, VARCHAR, NCHAR or NVARCHAR. metric column name: %s type: %s but datatype is %T", columnNames[metricIndex], columnTypes[metricIndex].DatabaseTypeName(), values[metricIndex]) - } - } - - for i, col := range columnNames { - if i == timeIndex || i == metricIndex { - continue - } - - switch columnValue := values[i].(type) { - case int64: - value = null.FloatFrom(float64(columnValue)) - case float64: - value = null.FloatFrom(columnValue) - case nil: - value.Valid = false - default: - return fmt.Errorf("Value column must have numeric datatype, column: %s type: %T value: %v", col, columnValue, columnValue) - } - if metricIndex == -1 { - metric = col - } - - series, exist := pointsBySeries[metric] - if exist == false { - series = &tsdb.TimeSeries{Name: metric} - pointsBySeries[metric] = series - seriesByQueryOrder.PushBack(metric) - } - - if fillMissing { - var intervalStart float64 - if exist == false { - intervalStart = float64(tsdbQuery.TimeRange.MustGetFrom().UnixNano() / 1e6) - } else { - intervalStart = series.Points[len(series.Points)-1][1].Float64 + fillInterval - } - - // align interval start - intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval - - for i := intervalStart; i < timestamp; i += fillInterval { - series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)}) - rowCount++ - } - } - - series.Points = append(series.Points, tsdb.TimePoint{value, null.FloatFrom(timestamp)}) - - e.log.Debug("Rows", "metric", metric, "time", timestamp, "value", value) - } - } - - for elem := seriesByQueryOrder.Front(); elem != nil; elem = elem.Next() { - key := elem.Value.(string) - result.Series = append(result.Series, pointsBySeries[key]) - - if fillMissing { - series := pointsBySeries[key] - // fill in values from last fetched value till interval end - intervalStart := series.Points[len(series.Points)-1][1].Float64 - intervalEnd := float64(tsdbQuery.TimeRange.MustGetTo().UnixNano() / 1e6) - - // align interval start - intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval - for i := intervalStart + fillInterval; i < intervalEnd; i += fillInterval { - series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)}) - rowCount++ - } - } - } - - result.Meta.Set("rowCount", rowCount) - return nil -} diff --git a/pkg/tsdb/mssql/mssql_test.go b/pkg/tsdb/mssql/mssql_test.go index 88b35b1aa2c..c3d4470603d 100644 --- a/pkg/tsdb/mssql/mssql_test.go +++ b/pkg/tsdb/mssql/mssql_test.go @@ -1,6 +1,7 @@ package mssql import ( + "context" "fmt" "math/rand" "strings" @@ -8,73 +9,89 @@ import ( "time" "github.com/go-xorm/xorm" + "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) // To run this test, remove the Skip from SkipConvey -// and set up a MSSQL db named grafanatest and a user/password grafana/Password! +// The tests require a MSSQL db named grafanatest and a user/password grafana/Password! // Use the docker/blocks/mssql_tests/docker-compose.yaml to spin up a // preconfigured MSSQL server suitable for running these tests. +// There is also a datasource and dashboard provisioned by devenv scripts that you can +// use to verify that the generated data are vizualized as expected, see +// devenv/README.md for setup instructions. // If needed, change the variable below to the IP address of the database. -var serverIP string = "localhost" +var serverIP = "localhost" func TestMSSQL(t *testing.T) { SkipConvey("MSSQL", t, func() { x := InitMSSQLTestDB(t) - endpoint := &MssqlQueryEndpoint{ - sqlEngine: &tsdb.DefaultSqlEngine{ - MacroEngine: NewMssqlMacroEngine(), - XormEngine: x, - }, - log: log.New("tsdb.mssql"), + origXormEngine := tsdb.NewXormEngine + tsdb.NewXormEngine = func(d, c string) (*xorm.Engine, error) { + return x, nil } - sess := x.NewSession() - defer sess.Close() + origInterpolate := tsdb.Interpolate + tsdb.Interpolate = func(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { + return sql, nil + } - fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC) + endpoint, err := newMssqlQueryEndpoint(&models.DataSource{ + JsonData: simplejson.New(), + SecureJsonData: securejsondata.SecureJsonData{}, + }) + So(err, ShouldBeNil) + + sess := x.NewSession() + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) + + Reset(func() { + sess.Close() + tsdb.NewXormEngine = origXormEngine + tsdb.Interpolate = origInterpolate + }) Convey("Given a table with different native data types", func() { sql := ` - IF OBJECT_ID('dbo.[mssql_types]', 'U') IS NOT NULL - DROP TABLE dbo.[mssql_types] + IF OBJECT_ID('dbo.[mssql_types]', 'U') IS NOT NULL + DROP TABLE dbo.[mssql_types] - CREATE TABLE [mssql_types] ( - c_bit bit, + CREATE TABLE [mssql_types] ( + c_bit bit, - c_tinyint tinyint, - c_smallint smallint, - c_int int, - c_bigint bigint, + c_tinyint tinyint, + c_smallint smallint, + c_int int, + c_bigint bigint, - c_money money, - c_smallmoney smallmoney, - c_numeric numeric(10,5), - c_real real, - c_decimal decimal(10,2), - c_float float, + c_money money, + c_smallmoney smallmoney, + c_numeric numeric(10,5), + c_real real, + c_decimal decimal(10,2), + c_float float, - c_char char(10), - c_varchar varchar(10), - c_text text, + c_char char(10), + c_varchar varchar(10), + c_text text, - c_nchar nchar(12), - c_nvarchar nvarchar(12), - c_ntext ntext, + c_nchar nchar(12), + c_nvarchar nvarchar(12), + c_ntext ntext, - c_datetime datetime, - c_datetime2 datetime2, - c_smalldatetime smalldatetime, - c_date date, - c_time time, - c_datetimeoffset datetimeoffset - ) - ` + c_datetime datetime, + c_datetime2 datetime2, + c_smalldatetime smalldatetime, + c_date date, + c_time time, + c_datetimeoffset datetimeoffset + ) + ` _, err := sess.Exec(sql) So(err, ShouldBeNil) @@ -87,14 +104,14 @@ func TestMSSQL(t *testing.T) { d2 := dt2.Format(dt2Format) sql = fmt.Sprintf(` - INSERT INTO [mssql_types] - SELECT - 1, 5, 20020, 980300, 1420070400, '$20000.15', '£2.15', 12345.12, - 1.11, 2.22, 3.33, - 'char10', 'varchar10', 'text', - N'☺nchar12☺', N'☺nvarchar12☺', N'☺text☺', - CAST('%s' AS DATETIME), CAST('%s' AS DATETIME2), CAST('%s' AS SMALLDATETIME), CAST('%s' AS DATE), CAST('%s' AS TIME), SWITCHOFFSET(CAST('%s' AS DATETIMEOFFSET), '-07:00') - `, d, d2, d, d, d, d2) + INSERT INTO [mssql_types] + SELECT + 1, 5, 20020, 980300, 1420070400, '$20000.15', '£2.15', 12345.12, + 1.11, 2.22, 3.33, + 'char10', 'varchar10', 'text', + N'☺nchar12☺', N'☺nvarchar12☺', N'☺text☺', + CAST('%s' AS DATETIME), CAST('%s' AS DATETIME2), CAST('%s' AS SMALLDATETIME), CAST('%s' AS DATE), CAST('%s' AS TIME), SWITCHOFFSET(CAST('%s' AS DATETIMEOFFSET), '-07:00') + `, d, d2, d, d, d, d2) _, err = sess.Exec(sql) So(err, ShouldBeNil) @@ -112,7 +129,7 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) + resp, err := endpoint.Query(context.Background(), nil, query) queryResult := resp.Results["A"] So(err, ShouldBeNil) @@ -151,14 +168,14 @@ func TestMSSQL(t *testing.T) { Convey("Given a table with metrics that lacks data for some series ", func() { sql := ` - IF OBJECT_ID('dbo.[metric]', 'U') IS NOT NULL - DROP TABLE dbo.[metric] + IF OBJECT_ID('dbo.[metric]', 'U') IS NOT NULL + DROP TABLE dbo.[metric] - CREATE TABLE [metric] ( - time datetime, - value int - ) - ` + CREATE TABLE [metric] ( + time datetime, + value int + ) + ` _, err := sess.Exec(sql) So(err, ShouldBeNil) @@ -186,16 +203,8 @@ func TestMSSQL(t *testing.T) { }) } - dtFormat := "2006-01-02 15:04:05.999999999" - for _, s := range series { - sql = fmt.Sprintf(` - INSERT INTO metric (time, value) - VALUES(CAST('%s' AS DATETIME), %d) - `, s.Time.Format(dtFormat), s.Value) - - _, err = sess.Exec(sql) - So(err, ShouldBeNil) - } + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) Convey("When doing a metric query using timeGroup", func() { query := &tsdb.TsdbQuery{ @@ -210,23 +219,34 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["A"] + resp, err := endpoint.Query(context.Background(), nil, query) So(err, ShouldBeNil) + queryResult := resp.Results["A"] So(queryResult.Error, ShouldBeNil) points := queryResult.Series[0].Points - + // without fill this should result in 4 buckets So(len(points), ShouldEqual, 4) - actualValueFirst := points[0][0].Float64 - actualTimeFirst := time.Unix(int64(points[0][1].Float64)/1000, 0) - So(actualValueFirst, ShouldEqual, 15) - So(actualTimeFirst, ShouldEqual, fromStart) - actualValueLast := points[3][0].Float64 - actualTimeLast := time.Unix(int64(points[3][1].Float64)/1000, 0) - So(actualValueLast, ShouldEqual, 20) - So(actualTimeLast, ShouldEqual, fromStart.Add(25*time.Minute)) + dt := fromStart + + for i := 0; i < 2; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 15) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + + // adjust for 10 minute gap between first and second set of points + dt = dt.Add(10 * time.Minute) + for i := 2; i < 4; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 20) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } }) Convey("When doing a metric query using timeGroup with NULL fill enabled", func() { @@ -246,34 +266,74 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["A"] + resp, err := endpoint.Query(context.Background(), nil, query) So(err, ShouldBeNil) + queryResult := resp.Results["A"] So(queryResult.Error, ShouldBeNil) points := queryResult.Series[0].Points - So(len(points), ShouldEqual, 7) - actualValueFirst := points[0][0].Float64 - actualTimeFirst := time.Unix(int64(points[0][1].Float64)/1000, 0) - So(actualValueFirst, ShouldEqual, 15) - So(actualTimeFirst, ShouldEqual, fromStart) - actualNullPoint := points[3][0] - actualNullTime := time.Unix(int64(points[3][1].Float64)/1000, 0) - So(actualNullPoint.Valid, ShouldBeFalse) - So(actualNullTime, ShouldEqual, fromStart.Add(15*time.Minute)) + dt := fromStart - actualValueLast := points[5][0].Float64 - actualTimeLast := time.Unix(int64(points[5][1].Float64)/1000, 0) - So(actualValueLast, ShouldEqual, 20) - So(actualTimeLast, ShouldEqual, fromStart.Add(25*time.Minute)) + for i := 0; i < 2; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 15) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } - actualLastNullPoint := points[6][0] - actualLastNullTime := time.Unix(int64(points[6][1].Float64)/1000, 0) - So(actualLastNullPoint.Valid, ShouldBeFalse) - So(actualLastNullTime, ShouldEqual, fromStart.Add(30*time.Minute)) + // check for NULL values inserted by fill + So(points[2][0].Valid, ShouldBeFalse) + So(points[3][0].Valid, ShouldBeFalse) + // adjust for 10 minute gap between first and second set of points + dt = dt.Add(10 * time.Minute) + for i := 4; i < 6; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 20) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + + So(points[6][0].Valid, ShouldBeFalse) + + }) + + Convey("When doing a metric query using timeGroup and $__interval", func() { + mockInterpolate := tsdb.Interpolate + tsdb.Interpolate = origInterpolate + + Reset(func() { + tsdb.Interpolate = mockInterpolate + }) + + Convey("Should replace $__interval", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + DataSource: &models.DataSource{}, + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, $__interval) AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, $__interval) ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(30*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(queryResult.Meta.Get("sql").MustString(), ShouldEqual, "SELECT FLOOR(DATEDIFF(second, '1970-01-01', time)/60)*60 AS time, avg(value) as value FROM metric GROUP BY FLOOR(DATEDIFF(second, '1970-01-01', time)/60)*60 ORDER BY 1") + }) }) Convey("When doing a metric query using timeGroup with float fill enabled", func() { @@ -293,71 +353,257 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["A"] + resp, err := endpoint.Query(context.Background(), nil, query) So(err, ShouldBeNil) + queryResult := resp.Results["A"] So(queryResult.Error, ShouldBeNil) points := queryResult.Series[0].Points - - So(points[6][0].Float64, ShouldEqual, 1.5) + So(points[3][0].Float64, ShouldEqual, 1.5) }) }) Convey("Given a table with metrics having multiple values and measurements", func() { - sql := ` - IF OBJECT_ID('dbo.[metric_values]', 'U') IS NOT NULL - DROP TABLE dbo.[metric_values] - - CREATE TABLE [metric_values] ( - time datetime, - measurement nvarchar(100), - valueOne int, - valueTwo int, - ) - ` - - _, err := sess.Exec(sql) - So(err, ShouldBeNil) - - type metricValues struct { - Time time.Time - Measurement string - ValueOne int64 - ValueTwo int64 + type metric_values struct { + Time time.Time + TimeInt64 int64 `xorm:"bigint 'timeInt64' not null"` + TimeInt64Nullable *int64 `xorm:"bigint 'timeInt64Nullable' null"` + TimeFloat64 float64 `xorm:"float 'timeFloat64' not null"` + TimeFloat64Nullable *float64 `xorm:"float 'timeFloat64Nullable' null"` + TimeInt32 int32 `xorm:"int(11) 'timeInt32' not null"` + TimeInt32Nullable *int32 `xorm:"int(11) 'timeInt32Nullable' null"` + TimeFloat32 float32 `xorm:"float(11) 'timeFloat32' not null"` + TimeFloat32Nullable *float32 `xorm:"float(11) 'timeFloat32Nullable' null"` + Measurement string + ValueOne int64 `xorm:"integer 'valueOne'"` + ValueTwo int64 `xorm:"integer 'valueTwo'"` } + if exist, err := sess.IsTableExist(metric_values{}); err != nil || exist { + So(err, ShouldBeNil) + sess.DropTable(metric_values{}) + } + err := sess.CreateTable(metric_values{}) + So(err, ShouldBeNil) + rand.Seed(time.Now().Unix()) rnd := func(min, max int64) int64 { return rand.Int63n(max-min) + min } - series := []*metricValues{} - for _, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) { - series = append(series, &metricValues{ - Time: t, - Measurement: "Metric A", - ValueOne: rnd(0, 100), - ValueTwo: rnd(0, 100), - }) - series = append(series, &metricValues{ - Time: t, - Measurement: "Metric B", - ValueOne: rnd(0, 100), - ValueTwo: rnd(0, 100), - }) + var tInitial time.Time + + series := []*metric_values{} + for i, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) { + if i == 0 { + tInitial = t + } + tSeconds := t.Unix() + tSecondsInt32 := int32(tSeconds) + tSecondsFloat32 := float32(tSeconds) + tMilliseconds := tSeconds * 1e3 + tMillisecondsFloat := float64(tMilliseconds) + first := metric_values{ + Time: t, + TimeInt64: tMilliseconds, + TimeInt64Nullable: &(tMilliseconds), + TimeFloat64: tMillisecondsFloat, + TimeFloat64Nullable: &tMillisecondsFloat, + TimeInt32: tSecondsInt32, + TimeInt32Nullable: &tSecondsInt32, + TimeFloat32: tSecondsFloat32, + TimeFloat32Nullable: &tSecondsFloat32, + Measurement: "Metric A", + ValueOne: rnd(0, 100), + ValueTwo: rnd(0, 100), + } + second := first + second.Measurement = "Metric B" + second.ValueOne = rnd(0, 100) + second.ValueTwo = rnd(0, 100) + + series = append(series, &first) + series = append(series, &second) } - dtFormat := "2006-01-02 15:04:05" - for _, s := range series { - sql = fmt.Sprintf(` - INSERT metric_values (time, measurement, valueOne, valueTwo) - VALUES(CAST('%s' AS DATETIME), '%s', %d, %d) - `, s.Time.Format(dtFormat), s.Measurement, s.ValueOne, s.ValueTwo) + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) - _, err = sess.Exec(sql) + Convey("When doing a metric query using epoch (int64) as time column and value column (int64) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeInt64 as time, timeInt64 FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) So(err, ShouldBeNil) - } + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int64 nullable) as time column and value column (int64 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeInt64Nullable as time, timeInt64Nullable FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64) as time column and value column (float64) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeFloat64 as time, timeFloat64 FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64 nullable) as time column and value column (float64 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeFloat64Nullable as time, timeFloat64Nullable FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int32) as time column and value column (int32) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeInt32 as time, timeInt32 FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int32 nullable) as time column and value column (int32 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeInt32Nullable as time, timeInt32Nullable FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float32) as time column and value column (float32) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeFloat32 as time, timeFloat32 FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float32(tInitial.Unix()))*1e3) + }) + + Convey("When doing a metric query using epoch (float32 nullable) as time column and value column (float32 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT TOP 1 timeFloat32Nullable as time, timeFloat32Nullable FROM metric_values ORDER BY time`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float32(tInitial.Unix()))*1e3) + }) Convey("When doing a metric query grouping by time and select metric column should return correct series", func() { query := &tsdb.TsdbQuery{ @@ -372,9 +618,9 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["A"] + resp, err := endpoint.Query(context.Background(), nil, query) So(err, ShouldBeNil) + queryResult := resp.Results["A"] So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 2) @@ -395,9 +641,9 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["A"] + resp, err := endpoint.Query(context.Background(), nil, query) So(err, ShouldBeNil) + queryResult := resp.Results["A"] So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 2) @@ -405,61 +651,110 @@ func TestMSSQL(t *testing.T) { So(queryResult.Series[1].Name, ShouldEqual, "valueTwo") }) + Convey("When doing a metric query with metric column and multiple value columns", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeEpoch(time), measurement, valueOne, valueTwo FROM metric_values ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 4) + So(queryResult.Series[0].Name, ShouldEqual, "Metric A valueOne") + So(queryResult.Series[1].Name, ShouldEqual, "Metric A valueTwo") + So(queryResult.Series[2].Name, ShouldEqual, "Metric B valueOne") + So(queryResult.Series[3].Name, ShouldEqual, "Metric B valueTwo") + }) + + Convey("When doing a query with timeFrom,timeTo,unixEpochFrom,unixEpochTo macros", func() { + tsdb.Interpolate = origInterpolate + query := &tsdb.TsdbQuery{ + TimeRange: tsdb.NewFakeTimeRange("5m", "now", fromStart), + Queries: []*tsdb.Query{ + { + DataSource: &models.DataSource{JsonData: simplejson.New()}, + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeFrom() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(queryResult.Meta.Get("sql").MustString(), ShouldEqual, "SELECT time FROM metric_values WHERE time > '2018-03-15T12:55:00Z' OR time < '2018-03-15T12:55:00Z' OR 1 < 1521118500 OR 1521118800 > 1 ORDER BY 1") + + }) + Convey("Given a stored procedure that takes @from and @to in epoch time", func() { sql := ` - IF object_id('sp_test_epoch') IS NOT NULL - DROP PROCEDURE sp_test_epoch - ` + IF object_id('sp_test_epoch') IS NOT NULL + DROP PROCEDURE sp_test_epoch + ` _, err := sess.Exec(sql) So(err, ShouldBeNil) sql = ` - CREATE PROCEDURE sp_test_epoch( - @from int, - @to int - ) AS - BEGIN - SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time, - measurement + ' - value one' as metric, - avg(valueOne) as value - FROM - metric_values - WHERE - time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01') - GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int), - measurement - UNION ALL - SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time, - measurement + ' - value two' as metric, - avg(valueTwo) as value - FROM - metric_values - WHERE - time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01') - GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int), - measurement - ORDER BY 1 - END - ` + CREATE PROCEDURE sp_test_epoch( + @from int, + @to int, + @interval nvarchar(50) = '5m', + @metric nvarchar(200) = 'ALL' + ) AS + BEGIN + DECLARE @dInterval int + SELECT @dInterval = 300 + + IF @interval = '10m' + SELECT @dInterval = 600 + + SELECT + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval as time, + measurement as metric, + avg(valueOne) as valueOne, + avg(valueTwo) as valueTwo + FROM + metric_values + WHERE + time BETWEEN DATEADD(s, @from, '1970-01-01') AND DATEADD(s, @to, '1970-01-01') AND + (@metric = 'ALL' OR measurement = @metric) + GROUP BY + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval, + measurement + ORDER BY 1 + END + ` _, err = sess.Exec(sql) So(err, ShouldBeNil) Convey("When doing a metric query using stored procedure should return correct result", func() { + tsdb.Interpolate = origInterpolate query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `DECLARE - @from int = $__unixEpochFrom(), - @to int = $__unixEpochTo() + @from int = $__unixEpochFrom(), + @to int = $__unixEpochTo() - EXEC dbo.sp_test_epoch @from, @to`, + EXEC dbo.sp_test_epoch @from, @to`, "format": "time_series", }), RefId: "A", @@ -471,74 +766,74 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) + resp, err := endpoint.Query(context.Background(), nil, query) queryResult := resp.Results["A"] So(err, ShouldBeNil) So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 4) - So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one") - So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one") - So(queryResult.Series[2].Name, ShouldEqual, "Metric A - value two") - So(queryResult.Series[3].Name, ShouldEqual, "Metric B - value two") + So(queryResult.Series[0].Name, ShouldEqual, "Metric A valueOne") + So(queryResult.Series[1].Name, ShouldEqual, "Metric A valueTwo") + So(queryResult.Series[2].Name, ShouldEqual, "Metric B valueOne") + So(queryResult.Series[3].Name, ShouldEqual, "Metric B valueTwo") }) }) Convey("Given a stored procedure that takes @from and @to in datetime", func() { sql := ` - IF object_id('sp_test_datetime') IS NOT NULL - DROP PROCEDURE sp_test_datetime - ` + IF object_id('sp_test_datetime') IS NOT NULL + DROP PROCEDURE sp_test_datetime + ` _, err := sess.Exec(sql) So(err, ShouldBeNil) sql = ` - CREATE PROCEDURE sp_test_datetime( - @from datetime, - @to datetime - ) AS - BEGIN - SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time, - measurement + ' - value one' as metric, - avg(valueOne) as value - FROM - metric_values - WHERE - time >= @from AND time <= @to - GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int), - measurement - UNION ALL - SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time, - measurement + ' - value two' as metric, - avg(valueTwo) as value - FROM - metric_values - WHERE - time >= @from AND time <= @to - GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int), - measurement - ORDER BY 1 - END - ` + CREATE PROCEDURE sp_test_datetime( + @from datetime, + @to datetime, + @interval nvarchar(50) = '5m', + @metric nvarchar(200) = 'ALL' + ) AS + BEGIN + DECLARE @dInterval int + SELECT @dInterval = 300 + + IF @interval = '10m' + SELECT @dInterval = 600 + + SELECT + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval as time, + measurement as metric, + avg(valueOne) as valueOne, + avg(valueTwo) as valueTwo + FROM + metric_values + WHERE + time BETWEEN @from AND @to AND + (@metric = 'ALL' OR measurement = @metric) + GROUP BY + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval, + measurement + ORDER BY 1 + END + ` _, err = sess.Exec(sql) So(err, ShouldBeNil) Convey("When doing a metric query using stored procedure should return correct result", func() { + tsdb.Interpolate = origInterpolate query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `DECLARE - @from int = $__unixEpochFrom(), - @to int = $__unixEpochTo() + @from int = $__unixEpochFrom(), + @to int = $__unixEpochTo() - EXEC dbo.sp_test_epoch @from, @to`, + EXEC dbo.sp_test_epoch @from, @to`, "format": "time_series", }), RefId: "A", @@ -550,16 +845,16 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) + resp, err := endpoint.Query(context.Background(), nil, query) queryResult := resp.Results["A"] So(err, ShouldBeNil) So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 4) - So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one") - So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one") - So(queryResult.Series[2].Name, ShouldEqual, "Metric A - value two") - So(queryResult.Series[3].Name, ShouldEqual, "Metric B - value two") + So(queryResult.Series[0].Name, ShouldEqual, "Metric A valueOne") + So(queryResult.Series[1].Name, ShouldEqual, "Metric A valueTwo") + So(queryResult.Series[2].Name, ShouldEqual, "Metric B valueOne") + So(queryResult.Series[3].Name, ShouldEqual, "Metric B valueTwo") }) }) }) @@ -570,7 +865,7 @@ func TestMSSQL(t *testing.T) { DROP TABLE dbo.[event] CREATE TABLE [event] ( - time_sec bigint, + time_sec int, description nvarchar(100), tags nvarchar(100), ) @@ -626,7 +921,7 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) + resp, err := endpoint.Query(context.Background(), nil, query) queryResult := resp.Results["Deploys"] So(err, ShouldBeNil) So(len(queryResult.Tables[0].Rows), ShouldEqual, 3) @@ -649,25 +944,202 @@ func TestMSSQL(t *testing.T) { }, } - resp, err := endpoint.Query(nil, nil, query) + resp, err := endpoint.Query(context.Background(), nil, query) queryResult := resp.Results["Tickets"] So(err, ShouldBeNil) So(len(queryResult.Tables[0].Rows), ShouldEqual, 3) }) + + Convey("When doing an annotation query with a time column in datetime format", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + dtFormat := "2006-01-02 15:04:05.999999999" + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + CAST('%s' AS DATETIME) as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Format(dtFormat)), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(float64), ShouldEqual, float64(dt.UnixNano()/1e6)) + }) + + Convey("When doing an annotation query with a time column in epoch second format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column in epoch second format (int) should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + cast(%d as int) as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column in epoch millisecond format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()*1000), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(float64), ShouldEqual, float64(dt.Unix()*1000)) + }) + + Convey("When doing an annotation query with a time column holding a bigint null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as bigint) as time, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) + }) + + Convey("When doing an annotation query with a time column holding a datetime null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as datetime) as time, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) + }) }) }) } func InitMSSQLTestDB(t *testing.T) *xorm.Engine { x, err := xorm.NewEngine(sqlutil.TestDB_Mssql.DriverName, strings.Replace(sqlutil.TestDB_Mssql.ConnStr, "localhost", serverIP, 1)) - - // x.ShowSQL() - if err != nil { t.Fatalf("Failed to init mssql db %v", err) } - sqlutil.CleanDB(x) + x.DatabaseTZ = time.UTC + x.TZLocation = time.UTC + + // x.ShowSQL() return x } diff --git a/pkg/tsdb/mysql/macros.go b/pkg/tsdb/mysql/macros.go index b0170070dcf..839f805568e 100644 --- a/pkg/tsdb/mysql/macros.go +++ b/pkg/tsdb/mysql/macros.go @@ -3,33 +3,32 @@ package mysql import ( "fmt" "regexp" - "strconv" "strings" "time" "github.com/grafana/grafana/pkg/tsdb" ) -//const rsString = `(?:"([^"]*)")`; const rsIdentifier = `([_a-zA-Z0-9]+)` const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)` -type MySqlMacroEngine struct { - TimeRange *tsdb.TimeRange - Query *tsdb.Query +type mySqlMacroEngine struct { + *tsdb.SqlMacroEngineBase + timeRange *tsdb.TimeRange + query *tsdb.Query } -func NewMysqlMacroEngine() tsdb.SqlMacroEngine { - return &MySqlMacroEngine{} +func newMysqlMacroEngine() tsdb.SqlMacroEngine { + return &mySqlMacroEngine{SqlMacroEngineBase: tsdb.NewSqlMacroEngineBase()} } -func (m *MySqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { - m.TimeRange = timeRange - m.Query = query +func (m *mySqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { + m.timeRange = timeRange + m.query = query rExp, _ := regexp.Compile(sExpr) var macroError error - sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { + sql = m.ReplaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { args := strings.Split(groups[2], ",") for i, arg := range args { args[i] = strings.Trim(arg, " ") @@ -49,26 +48,9 @@ func (m *MySqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRa return sql, nil } -func replaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string { - result := "" - lastIndex := 0 - - for _, v := range re.FindAllSubmatchIndex([]byte(str), -1) { - groups := []string{} - for i := 0; i < len(v); i += 2 { - groups = append(groups, str[v[i]:v[i+1]]) - } - - result += str[lastIndex:v[0]] + repl(groups) - lastIndex = v[1] - } - - return result + str[lastIndex:] -} - -func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, error) { +func (m *mySqlMacroEngine) evaluateMacro(name string, args []string) (string, error) { switch name { - case "__time": + case "__timeEpoch", "__time": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } @@ -77,11 +59,8 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= FROM_UNIXTIME(%d) AND %s <= FROM_UNIXTIME(%d)", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil - case "__timeFrom": - return fmt.Sprintf("FROM_UNIXTIME(%d)", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil - case "__timeTo": - return fmt.Sprintf("FROM_UNIXTIME(%d)", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + + return fmt.Sprintf("%s BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", args[0], m.timeRange.GetFromAsSecondsEpoch(), m.timeRange.GetToAsSecondsEpoch()), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -91,28 +70,44 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er return "", fmt.Errorf("error parsing interval %v", args[1]) } if len(args) == 3 { - m.Query.Model.Set("fill", true) - m.Query.Model.Set("fillInterval", interval.Seconds()) - if args[2] == "NULL" { - m.Query.Model.Set("fillNull", true) - } else { - floatVal, err := strconv.ParseFloat(args[2], 64) - if err != nil { - return "", fmt.Errorf("error parsing fill value %v", args[2]) - } - m.Query.Model.Set("fillValue", floatVal) + err := tsdb.SetupFillmode(m.query, interval, args[2]) + if err != nil { + return "", err } } - return fmt.Sprintf("cast(cast(UNIX_TIMESTAMP(%s)/(%.0f) as signed)*%.0f as signed)", args[0], interval.Seconds(), interval.Seconds()), nil + return fmt.Sprintf("UNIX_TIMESTAMP(%s) DIV %.0f * %.0f", args[0], interval.Seconds(), interval.Seconds()), nil + case "__timeGroupAlias": + tg, err := m.evaluateMacro("__timeGroup", args) + if err == nil { + return tg + " AS \"time\"", err + } + return "", err case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil - case "__unixEpochFrom": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil - case "__unixEpochTo": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], m.timeRange.GetFromAsSecondsEpoch(), args[0], m.timeRange.GetToAsSecondsEpoch()), nil + case "__unixEpochGroup": + if len(args) < 2 { + return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name) + } + interval, err := time.ParseDuration(strings.Trim(args[1], `'`)) + if err != nil { + return "", fmt.Errorf("error parsing interval %v", args[1]) + } + if len(args) == 3 { + err := tsdb.SetupFillmode(m.query, interval, args[2]) + if err != nil { + return "", err + } + } + return fmt.Sprintf("%s DIV %v * %v", args[0], interval.Seconds(), interval.Seconds()), nil + case "__unixEpochGroupAlias": + tg, err := m.evaluateMacro("__unixEpochGroup", args) + if err == nil { + return tg + " AS \"time\"", err + } + return "", err default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/mysql/macros_test.go b/pkg/tsdb/mysql/macros_test.go index a89ba16ab78..24bf18873d5 100644 --- a/pkg/tsdb/mysql/macros_test.go +++ b/pkg/tsdb/mysql/macros_test.go @@ -1,7 +1,10 @@ package mysql import ( + "fmt" + "strconv" "testing" + "time" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" @@ -9,81 +12,115 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { - engine := &MySqlMacroEngine{} + engine := &mySqlMacroEngine{} query := &tsdb.Query{} - timeRange := &tsdb.TimeRange{From: "5m", To: "now"} - Convey("interpolate __time function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { + from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) + to := from.Add(5 * time.Minute) + timeRange := tsdb.NewFakeTimeRange("5m", "now", to) + + Convey("interpolate __time function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select UNIX_TIMESTAMP(time_column) as time_sec") + }) + + Convey("interpolate __time function wrapped in aggregation", func() { + sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select min(UNIX_TIMESTAMP(time_column) as time_sec)") + }) + + Convey("interpolate __timeGroup function", func() { + + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroupAlias(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY UNIX_TIMESTAMP(time_column) DIV 300 * 300") + So(sql2, ShouldEqual, sql+" AS \"time\"") + }) + + Convey("interpolate __timeGroup function with spaces around arguments", func() { + + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroupAlias(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY UNIX_TIMESTAMP(time_column) DIV 300 * 300") + So(sql2, ShouldEqual, sql+" AS \"time\"") + }) + + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochGroup function", func() { + + sql, err := engine.Interpolate(query, timeRange, "SELECT $__unixEpochGroup(time_column,'5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "SELECT $__unixEpochGroupAlias(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "SELECT time_column DIV 300 * 300") + So(sql2, ShouldEqual, sql+" AS \"time\"") + }) - So(sql, ShouldEqual, "select UNIX_TIMESTAMP(time_column) as time_sec") }) - Convey("interpolate __time function wrapped in aggregation", func() { - sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select min(UNIX_TIMESTAMP(time_column) as time_sec)") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) }) - Convey("interpolate __timeFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "WHERE time_column >= FROM_UNIXTIME(18446744066914186738) AND time_column <= FROM_UNIXTIME(18446744066914187038)") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) }) - - Convey("interpolate __timeFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914186738)") - }) - - Convey("interpolate __timeGroup function", func() { - - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY cast(cast(UNIX_TIMESTAMP(time_column)/(300) as signed)*300 as signed)") - }) - - Convey("interpolate __timeGroup function with spaces around arguments", func() { - - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY cast(cast(UNIX_TIMESTAMP(time_column)/(300) as signed)*300 as signed)") - }) - - Convey("interpolate __timeTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914187038)") - }) - - Convey("interpolate __unixEpochFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(18446744066914186738)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038") - }) - - Convey("interpolate __unixEpochFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738") - }) - - Convey("interpolate __unixEpochTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914187038") - }) - }) } diff --git a/pkg/tsdb/mysql/mysql.go b/pkg/tsdb/mysql/mysql.go index f3060e235e5..35b03e489a0 100644 --- a/pkg/tsdb/mysql/mysql.go +++ b/pkg/tsdb/mysql/mysql.go @@ -1,130 +1,66 @@ package mysql import ( - "container/list" - "context" "database/sql" "fmt" - "math" "reflect" "strconv" - "time" + "strings" "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" - "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" ) -type MysqlQueryEndpoint struct { - sqlEngine tsdb.SqlEngine - log log.Logger -} - func init() { - tsdb.RegisterTsdbQueryEndpoint("mysql", NewMysqlQueryEndpoint) + tsdb.RegisterTsdbQueryEndpoint("mysql", newMysqlQueryEndpoint) } -func NewMysqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { - endpoint := &MysqlQueryEndpoint{ - log: log.New("tsdb.mysql"), - } +func newMysqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + logger := log.New("tsdb.mysql") - endpoint.sqlEngine = &tsdb.DefaultSqlEngine{ - MacroEngine: NewMysqlMacroEngine(), + protocol := "tcp" + if strings.HasPrefix(datasource.Url, "/") { + protocol = "unix" } - cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true", datasource.User, datasource.Password, - "tcp", + protocol, datasource.Url, datasource.Database, ) - endpoint.log.Debug("getEngine", "connection", cnnstr) + logger.Debug("getEngine", "connection", cnnstr) - if err := endpoint.sqlEngine.InitEngine("mysql", datasource, cnnstr); err != nil { - return nil, err + config := tsdb.SqlQueryEndpointConfiguration{ + DriverName: "mysql", + ConnectionString: cnnstr, + Datasource: datasource, + TimeColumnNames: []string{"time", "time_sec"}, + MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"}, } - return endpoint, nil + rowTransformer := mysqlRowTransformer{ + log: logger, + } + + return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newMysqlMacroEngine(), logger) } -// Query is the main function for the MysqlExecutor -func (e *MysqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable) +type mysqlRowTransformer struct { + log log.Logger } -func (e MysqlQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error { - columnNames, err := rows.Columns() - columnCount := len(columnNames) - - if err != nil { - return err - } - - table := &tsdb.Table{ - Columns: make([]tsdb.TableColumn, columnCount), - Rows: make([]tsdb.RowValues, 0), - } - - for i, name := range columnNames { - table.Columns[i].Text = name - } - - rowLimit := 1000000 - rowCount := 0 - timeIndex := -1 - - // check if there is a column named time - for i, col := range columnNames { - switch col { - case "time_sec": - timeIndex = i - } - } - - for ; rows.Next(); rowCount++ { - if rowCount > rowLimit { - return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit) - } - - values, err := e.getTypedRowData(rows) - if err != nil { - return err - } - - // for annotations, convert to epoch - if timeIndex != -1 { - switch value := values[timeIndex].(type) { - case time.Time: - values[timeIndex] = float64(value.UnixNano() / 1e9) - } - } - - table.Rows = append(table.Rows, values) - } - - result.Tables = append(result.Tables, table) - result.Meta.Set("rowCount", rowCount) - return nil -} - -func (e MysqlQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, error) { - types, err := rows.ColumnTypes() - if err != nil { - return nil, err - } - - values := make([]interface{}, len(types)) +func (t *mysqlRowTransformer) Transform(columnTypes []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) { + values := make([]interface{}, len(columnTypes)) for i := range values { - scanType := types[i].ScanType() + scanType := columnTypes[i].ScanType() values[i] = reflect.New(scanType).Interface() - if types[i].DatabaseTypeName() == "BIT" { + if columnTypes[i].DatabaseTypeName() == "BIT" { values[i] = new([]byte) } } @@ -133,7 +69,7 @@ func (e MysqlQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, er return nil, err } - for i := 0; i < len(types); i++ { + for i := 0; i < len(columnTypes); i++ { typeName := reflect.ValueOf(values[i]).Type().String() switch typeName { @@ -162,7 +98,7 @@ func (e MysqlQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, er } } - if types[i].DatabaseTypeName() == "DECIMAL" { + if columnTypes[i].DatabaseTypeName() == "DECIMAL" { f, err := strconv.ParseFloat(values[i].(string), 64) if err == nil { @@ -175,156 +111,3 @@ func (e MysqlQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, er return values, nil } - -func (e MysqlQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error { - pointsBySeries := make(map[string]*tsdb.TimeSeries) - seriesByQueryOrder := list.New() - - columnNames, err := rows.Columns() - if err != nil { - return err - } - - rowData := NewStringStringScan(columnNames) - rowLimit := 1000000 - rowCount := 0 - - fillMissing := query.Model.Get("fill").MustBool(false) - var fillInterval float64 - fillValue := null.Float{} - if fillMissing { - fillInterval = query.Model.Get("fillInterval").MustFloat64() * 1000 - if query.Model.Get("fillNull").MustBool(false) == false { - fillValue.Float64 = query.Model.Get("fillValue").MustFloat64() - fillValue.Valid = true - } - - } - - for ; rows.Next(); rowCount++ { - if rowCount > rowLimit { - return fmt.Errorf("MySQL query row limit exceeded, limit %d", rowLimit) - } - - err := rowData.Update(rows.Rows) - if err != nil { - e.log.Error("MySQL response parsing", "error", err) - return fmt.Errorf("MySQL response parsing error %v", err) - } - - if rowData.metric == "" { - rowData.metric = "Unknown" - } - - if !rowData.time.Valid { - return fmt.Errorf("Found row with no time value") - } - - series, exist := pointsBySeries[rowData.metric] - if exist == false { - series = &tsdb.TimeSeries{Name: rowData.metric} - pointsBySeries[rowData.metric] = series - seriesByQueryOrder.PushBack(rowData.metric) - } - - if fillMissing { - var intervalStart float64 - if exist == false { - intervalStart = float64(tsdbQuery.TimeRange.MustGetFrom().UnixNano() / 1e6) - } else { - intervalStart = series.Points[len(series.Points)-1][1].Float64 + fillInterval - } - - // align interval start - intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval - - for i := intervalStart; i < rowData.time.Float64; i += fillInterval { - series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)}) - rowCount++ - } - } - - series.Points = append(series.Points, tsdb.TimePoint{rowData.value, rowData.time}) - } - - for elem := seriesByQueryOrder.Front(); elem != nil; elem = elem.Next() { - key := elem.Value.(string) - result.Series = append(result.Series, pointsBySeries[key]) - - if fillMissing { - series := pointsBySeries[key] - // fill in values from last fetched value till interval end - intervalStart := series.Points[len(series.Points)-1][1].Float64 - intervalEnd := float64(tsdbQuery.TimeRange.MustGetTo().UnixNano() / 1e6) - - // align interval start - intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval - for i := intervalStart + fillInterval; i < intervalEnd; i += fillInterval { - series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)}) - rowCount++ - } - } - } - - result.Meta.Set("rowCount", rowCount) - return nil -} - -type stringStringScan struct { - rowPtrs []interface{} - rowValues []string - columnNames []string - columnCount int - - time null.Float - value null.Float - metric string -} - -func NewStringStringScan(columnNames []string) *stringStringScan { - s := &stringStringScan{ - columnCount: len(columnNames), - columnNames: columnNames, - rowPtrs: make([]interface{}, len(columnNames)), - rowValues: make([]string, len(columnNames)), - } - - for i := 0; i < s.columnCount; i++ { - s.rowPtrs[i] = new(sql.RawBytes) - } - - return s -} - -func (s *stringStringScan) Update(rows *sql.Rows) error { - if err := rows.Scan(s.rowPtrs...); err != nil { - return err - } - - s.time = null.FloatFromPtr(nil) - s.value = null.FloatFromPtr(nil) - - for i := 0; i < s.columnCount; i++ { - if rb, ok := s.rowPtrs[i].(*sql.RawBytes); ok { - s.rowValues[i] = string(*rb) - - switch s.columnNames[i] { - case "time_sec": - if sec, err := strconv.ParseInt(s.rowValues[i], 10, 64); err == nil { - s.time = null.FloatFrom(float64(sec * 1000)) - } - case "value": - if value, err := strconv.ParseFloat(s.rowValues[i], 64); err == nil { - s.value = null.FloatFrom(value) - } - case "metric": - s.metric = s.rowValues[i] - } - - *rb = nil // reset pointer to discard current value to avoid a bug - } else { - return fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, s.columnNames[i]) - } - } - return nil -} diff --git a/pkg/tsdb/mysql/mysql_test.go b/pkg/tsdb/mysql/mysql_test.go index fe2c82223d2..476e3ba6586 100644 --- a/pkg/tsdb/mysql/mysql_test.go +++ b/pkg/tsdb/mysql/mysql_test.go @@ -1,146 +1,1064 @@ package mysql import ( + "context" + "fmt" + "math/rand" + "strings" "testing" "time" "github.com/go-xorm/xorm" + "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) -// To run this test, remove the Skip from SkipConvey -// and set up a MySQL db named grafana_tests and a user/password grafana/password +// To run this test, set runMySqlTests=true +// Or from the commandline: GRAFANA_TEST_DB=mysql go test -v ./pkg/tsdb/mysql +// The tests require a MySQL db named grafana_ds_tests and a user/password grafana/password +// Use the docker/blocks/mysql_tests/docker-compose.yaml to spin up a +// preconfigured MySQL server suitable for running these tests. +// There is also a datasource and dashboard provisioned by devenv scripts that you can +// use to verify that the generated data are vizualized as expected, see +// devenv/README.md for setup instructions. func TestMySQL(t *testing.T) { - SkipConvey("MySQL", t, func() { + // change to true to run the MySQL tests + runMySqlTests := false + // runMySqlTests := true + + if !(sqlstore.IsTestDbMySql() || runMySqlTests) { + t.Skip() + } + + Convey("MySQL", t, func() { x := InitMySQLTestDB(t) - endpoint := &MysqlQueryEndpoint{ - sqlEngine: &tsdb.DefaultSqlEngine{ - MacroEngine: NewMysqlMacroEngine(), - XormEngine: x, - }, - log: log.New("tsdb.mysql"), + origXormEngine := tsdb.NewXormEngine + tsdb.NewXormEngine = func(d, c string) (*xorm.Engine, error) { + return x, nil } + origInterpolate := tsdb.Interpolate + tsdb.Interpolate = func(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { + return sql, nil + } + + endpoint, err := newMysqlQueryEndpoint(&models.DataSource{ + JsonData: simplejson.New(), + SecureJsonData: securejsondata.SecureJsonData{}, + }) + So(err, ShouldBeNil) + sess := x.NewSession() - defer sess.Close() + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC) - sql := "CREATE TABLE `mysql_types` (" - sql += "`atinyint` tinyint(1) NOT NULL," - sql += "`avarchar` varchar(3) NOT NULL," - sql += "`achar` char(3)," - sql += "`amediumint` mediumint NOT NULL," - sql += "`asmallint` smallint NOT NULL," - sql += "`abigint` bigint NOT NULL," - sql += "`aint` int(11) NOT NULL," - sql += "`adouble` double(10,2)," - sql += "`anewdecimal` decimal(10,2)," - sql += "`afloat` float(10,2) NOT NULL," - sql += "`atimestamp` timestamp NOT NULL," - sql += "`adatetime` datetime NOT NULL," - sql += "`atime` time NOT NULL," - // sql += "`ayear` year," // Crashes xorm when running cleandb - sql += "`abit` bit(1)," - sql += "`atinytext` tinytext," - sql += "`atinyblob` tinyblob," - sql += "`atext` text," - sql += "`ablob` blob," - sql += "`amediumtext` mediumtext," - sql += "`amediumblob` mediumblob," - sql += "`alongtext` longtext," - sql += "`alongblob` longblob," - sql += "`aenum` enum('val1', 'val2')," - sql += "`aset` set('a', 'b', 'c', 'd')," - sql += "`adate` date," - sql += "`time_sec` datetime(6)," - sql += "`aintnull` int(11)," - sql += "`afloatnull` float(10,2)," - sql += "`avarcharnull` varchar(3)," - sql += "`adecimalnull` decimal(10,2)" - sql += ") ENGINE=InnoDB DEFAULT CHARSET=latin1;" - _, err := sess.Exec(sql) - So(err, ShouldBeNil) + Reset(func() { + sess.Close() + tsdb.NewXormEngine = origXormEngine + tsdb.Interpolate = origInterpolate + }) - sql = "INSERT INTO `mysql_types` " - sql += "(`atinyint`, `avarchar`, `achar`, `amediumint`, `asmallint`, `abigint`, `aint`, `adouble`, " - sql += "`anewdecimal`, `afloat`, `adatetime`, `atimestamp`, `atime`, `abit`, `atinytext`, " - sql += "`atinyblob`, `atext`, `ablob`, `amediumtext`, `amediumblob`, `alongtext`, `alongblob`, " - sql += "`aenum`, `aset`, `adate`, `time_sec`) " - sql += "VALUES(1, 'abc', 'def', 1, 10, 100, 1420070400, 1.11, " - sql += "2.22, 3.33, now(), current_timestamp(), '11:11:11', 1, 'tinytext', " - sql += "'tinyblob', 'text', 'blob', 'mediumtext', 'mediumblob', 'longtext', 'longblob', " - sql += "'val2', 'a,b', curdate(), '2018-01-01 00:01:01.123456');" - _, err = sess.Exec(sql) - So(err, ShouldBeNil) + Convey("Given a table with different native data types", func() { + if exists, err := sess.IsTableExist("mysql_types"); err != nil || exists { + So(err, ShouldBeNil) + sess.DropTable("mysql_types") + } - Convey("Query with Table format should map MySQL column types to Go types", func() { + sql := "CREATE TABLE `mysql_types` (" + sql += "`atinyint` tinyint(1) NOT NULL," + sql += "`avarchar` varchar(3) NOT NULL," + sql += "`achar` char(3)," + sql += "`amediumint` mediumint NOT NULL," + sql += "`asmallint` smallint NOT NULL," + sql += "`abigint` bigint NOT NULL," + sql += "`aint` int(11) NOT NULL," + sql += "`adouble` double(10,2)," + sql += "`anewdecimal` decimal(10,2)," + sql += "`afloat` float(10,2) NOT NULL," + sql += "`atimestamp` timestamp NOT NULL," + sql += "`adatetime` datetime NOT NULL," + sql += "`atime` time NOT NULL," + sql += "`ayear` year," // Crashes xorm when running cleandb + sql += "`abit` bit(1)," + sql += "`atinytext` tinytext," + sql += "`atinyblob` tinyblob," + sql += "`atext` text," + sql += "`ablob` blob," + sql += "`amediumtext` mediumtext," + sql += "`amediumblob` mediumblob," + sql += "`alongtext` longtext," + sql += "`alongblob` longblob," + sql += "`aenum` enum('val1', 'val2')," + sql += "`aset` set('a', 'b', 'c', 'd')," + sql += "`adate` date," + sql += "`time_sec` datetime(6)," + sql += "`aintnull` int(11)," + sql += "`afloatnull` float(10,2)," + sql += "`avarcharnull` varchar(3)," + sql += "`adecimalnull` decimal(10,2)" + sql += ") ENGINE=InnoDB DEFAULT CHARSET=latin1;" + _, err := sess.Exec(sql) + So(err, ShouldBeNil) + + sql = "INSERT INTO `mysql_types` " + sql += "(`atinyint`, `avarchar`, `achar`, `amediumint`, `asmallint`, `abigint`, `aint`, `adouble`, " + sql += "`anewdecimal`, `afloat`, `adatetime`, `atimestamp`, `atime`, `ayear`, `abit`, `atinytext`, " + sql += "`atinyblob`, `atext`, `ablob`, `amediumtext`, `amediumblob`, `alongtext`, `alongblob`, " + sql += "`aenum`, `aset`, `adate`, `time_sec`) " + sql += "VALUES(1, 'abc', 'def', 1, 10, 100, 1420070400, 1.11, " + sql += "2.22, 3.33, now(), current_timestamp(), '11:11:11', '2018', 1, 'tinytext', " + sql += "'tinyblob', 'text', 'blob', 'mediumtext', 'mediumblob', 'longtext', 'longblob', " + sql += "'val2', 'a,b', curdate(), '2018-01-01 00:01:01.123456');" + _, err = sess.Exec(sql) + So(err, ShouldBeNil) + + Convey("Query with Table format should map MySQL column types to Go types", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT * FROM mysql_types", + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + column := queryResult.Tables[0].Rows[0] + + So(*column[0].(*int8), ShouldEqual, 1) + So(column[1].(string), ShouldEqual, "abc") + So(column[2].(string), ShouldEqual, "def") + So(*column[3].(*int32), ShouldEqual, 1) + So(*column[4].(*int16), ShouldEqual, 10) + So(*column[5].(*int64), ShouldEqual, 100) + So(*column[6].(*int32), ShouldEqual, 1420070400) + So(column[7].(float64), ShouldEqual, 1.11) + So(column[8].(float64), ShouldEqual, 2.22) + So(*column[9].(*float32), ShouldEqual, 3.33) + So(column[10].(time.Time), ShouldHappenWithin, 10*time.Second, time.Now()) + So(column[11].(time.Time), ShouldHappenWithin, 10*time.Second, time.Now()) + So(column[12].(string), ShouldEqual, "11:11:11") + So(column[13].(int64), ShouldEqual, 2018) + So(*column[14].(*[]byte), ShouldHaveSameTypeAs, []byte{1}) + So(column[15].(string), ShouldEqual, "tinytext") + So(column[16].(string), ShouldEqual, "tinyblob") + So(column[17].(string), ShouldEqual, "text") + So(column[18].(string), ShouldEqual, "blob") + So(column[19].(string), ShouldEqual, "mediumtext") + So(column[20].(string), ShouldEqual, "mediumblob") + So(column[21].(string), ShouldEqual, "longtext") + So(column[22].(string), ShouldEqual, "longblob") + So(column[23].(string), ShouldEqual, "val2") + So(column[24].(string), ShouldEqual, "a,b") + So(column[25].(time.Time).Format("2006-01-02T00:00:00Z"), ShouldEqual, time.Now().UTC().Format("2006-01-02T00:00:00Z")) + So(column[26].(float64), ShouldEqual, float64(1.514764861123456*1e12)) + So(column[27], ShouldEqual, nil) + So(column[28], ShouldEqual, nil) + So(column[29], ShouldEqual, "") + So(column[30], ShouldEqual, nil) + }) + }) + + Convey("Given a table with metrics that lacks data for some series ", func() { + type metric struct { + Time time.Time + Value int64 + } + + if exist, err := sess.IsTableExist(metric{}); err != nil || exist { + So(err, ShouldBeNil) + sess.DropTable(metric{}) + } + err := sess.CreateTable(metric{}) + So(err, ShouldBeNil) + + series := []*metric{} + firstRange := genTimeRangeByInterval(fromStart, 10*time.Minute, 10*time.Second) + secondRange := genTimeRangeByInterval(fromStart.Add(20*time.Minute), 10*time.Minute, 10*time.Second) + + for _, t := range firstRange { + series = append(series, &metric{ + Time: t, + Value: 15, + }) + } + + for _, t := range secondRange { + series = append(series, &metric{ + Time: t, + Value: 20, + }) + } + + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) + + Convey("When doing a metric query using timeGroup", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, '5m') as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + // without fill this should result in 4 buckets + So(len(points), ShouldEqual, 4) + + dt := fromStart + + for i := 0; i < 2; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 15) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + + // adjust for 10 minute gap between first and second set of points + dt = dt.Add(10 * time.Minute) + for i := 2; i < 4; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 20) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + }) + + Convey("When doing a metric query using timeGroup with NULL fill enabled", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, '5m', NULL) as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + So(len(points), ShouldEqual, 7) + + dt := fromStart + + for i := 0; i < 2; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 15) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + + // check for NULL values inserted by fill + So(points[2][0].Valid, ShouldBeFalse) + So(points[3][0].Valid, ShouldBeFalse) + + // adjust for 10 minute gap between first and second set of points + dt = dt.Add(10 * time.Minute) + for i := 4; i < 6; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 20) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + + // check for NULL values inserted by fill + So(points[6][0].Valid, ShouldBeFalse) + + }) + + Convey("When doing a metric query using timeGroup and $__interval", func() { + mockInterpolate := tsdb.Interpolate + tsdb.Interpolate = origInterpolate + + Reset(func() { + tsdb.Interpolate = mockInterpolate + }) + + Convey("Should replace $__interval", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + DataSource: &models.DataSource{JsonData: simplejson.New()}, + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, $__interval) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(30*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(queryResult.Meta.Get("sql").MustString(), ShouldEqual, "SELECT UNIX_TIMESTAMP(time) DIV 60 * 60 AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1") + }) + }) + + Convey("When doing a metric query using timeGroup with value fill enabled", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, '5m', 1.5) as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + So(points[3][0].Float64, ShouldEqual, 1.5) + }) + + Convey("When doing a metric query using timeGroup with previous fill enabled", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, '5m', previous) as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + So(points[2][0].Float64, ShouldEqual, 15.0) + So(points[3][0].Float64, ShouldEqual, 15.0) + So(points[6][0].Float64, ShouldEqual, 20.0) + }) + + }) + + Convey("Given a table with metrics having multiple values and measurements", func() { + type metric_values struct { + Time time.Time `xorm:"datetime 'time' not null"` + TimeNullable *time.Time `xorm:"datetime(6) 'timeNullable' null"` + TimeInt64 int64 `xorm:"bigint(20) 'timeInt64' not null"` + TimeInt64Nullable *int64 `xorm:"bigint(20) 'timeInt64Nullable' null"` + TimeFloat64 float64 `xorm:"double 'timeFloat64' not null"` + TimeFloat64Nullable *float64 `xorm:"double 'timeFloat64Nullable' null"` + TimeInt32 int32 `xorm:"int(11) 'timeInt32' not null"` + TimeInt32Nullable *int32 `xorm:"int(11) 'timeInt32Nullable' null"` + TimeFloat32 float32 `xorm:"double 'timeFloat32' not null"` + TimeFloat32Nullable *float32 `xorm:"double 'timeFloat32Nullable' null"` + Measurement string + ValueOne int64 `xorm:"integer 'valueOne'"` + ValueTwo int64 `xorm:"integer 'valueTwo'"` + } + + if exist, err := sess.IsTableExist(metric_values{}); err != nil || exist { + So(err, ShouldBeNil) + sess.DropTable(metric_values{}) + } + err := sess.CreateTable(metric_values{}) + So(err, ShouldBeNil) + + rand.Seed(time.Now().Unix()) + rnd := func(min, max int64) int64 { + return rand.Int63n(max-min) + min + } + + var tInitial time.Time + + series := []*metric_values{} + for i, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) { + if i == 0 { + tInitial = t + } + tSeconds := t.Unix() + tSecondsInt32 := int32(tSeconds) + tSecondsFloat32 := float32(tSeconds) + tMilliseconds := tSeconds * 1e3 + tMillisecondsFloat := float64(tMilliseconds) + t2 := t + first := metric_values{ + Time: t, + TimeNullable: &t2, + TimeInt64: tMilliseconds, + TimeInt64Nullable: &(tMilliseconds), + TimeFloat64: tMillisecondsFloat, + TimeFloat64Nullable: &tMillisecondsFloat, + TimeInt32: tSecondsInt32, + TimeInt32Nullable: &tSecondsInt32, + TimeFloat32: tSecondsFloat32, + TimeFloat32Nullable: &tSecondsFloat32, + Measurement: "Metric A", + ValueOne: rnd(0, 100), + ValueTwo: rnd(0, 100), + } + second := first + second.Measurement = "Metric B" + second.ValueOne = rnd(0, 100) + second.ValueTwo = rnd(0, 100) + + series = append(series, &first) + series = append(series, &second) + } + + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) + + Convey("When doing a metric query using time as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using time (nullable) as time column should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeNullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int64) as time column and value column (int64) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt64 as time, timeInt64 FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int64 nullable) as time column and value column (int64 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt64Nullable as time, timeInt64Nullable FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64) as time column and value column (float64) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat64 as time, timeFloat64 FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64 nullable) as time column and value column (float64 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat64Nullable as time, timeFloat64Nullable FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + FocusConvey("When doing a metric query using epoch (int32) as time column and value column (int32) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt32 as time, timeInt32 FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int32 nullable) as time column and value column (int32 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeInt32Nullable as time, timeInt32Nullable FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float32) as time column and value column (float32) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat32 as time, timeFloat32 FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float32(tInitial.Unix()))*1e3) + }) + + Convey("When doing a metric query using epoch (float32 nullable) as time column and value column (float32 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT timeFloat32Nullable as time, timeFloat32Nullable FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float32(tInitial.Unix()))*1e3) + }) + + Convey("When doing a metric query grouping by time and select metric column should return correct series", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT $__time(time), CONCAT(measurement, ' - value one') as metric, valueOne FROM metric_values ORDER BY 1,2`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 2) + So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one") + So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one") + }) + + Convey("When doing a metric query with metric column and multiple value columns", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values ORDER BY 1,2`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 4) + So(queryResult.Series[0].Name, ShouldEqual, "Metric A valueOne") + So(queryResult.Series[1].Name, ShouldEqual, "Metric A valueTwo") + So(queryResult.Series[2].Name, ShouldEqual, "Metric B valueOne") + So(queryResult.Series[3].Name, ShouldEqual, "Metric B valueTwo") + }) + + Convey("When doing a metric query grouping by time should return correct series", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT $__time(time), valueOne, valueTwo FROM metric_values ORDER BY 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 2) + So(queryResult.Series[0].Name, ShouldEqual, "valueOne") + So(queryResult.Series[1].Name, ShouldEqual, "valueTwo") + }) + }) + + Convey("When doing a query with timeFrom,timeTo,unixEpochFrom,unixEpochTo macros", func() { + tsdb.Interpolate = origInterpolate query := &tsdb.TsdbQuery{ + TimeRange: tsdb.NewFakeTimeRange("5m", "now", fromStart), Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ - "rawSql": "SELECT * FROM mysql_types", - "format": "table", + "rawSql": `SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeFrom() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1`, + "format": "time_series", }), RefId: "A", }, }, } - resp, err := endpoint.Query(nil, nil, query) + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(queryResult.Meta.Get("sql").MustString(), ShouldEqual, "SELECT time FROM metric_values WHERE time > '2018-03-15T12:55:00Z' OR time < '2018-03-15T12:55:00Z' OR 1 < 1521118500 OR 1521118800 > 1 ORDER BY 1") + + }) + + Convey("Given a table with event data", func() { + type event struct { + TimeSec int64 + Description string + Tags string + } + + if exist, err := sess.IsTableExist(event{}); err != nil || exist { + So(err, ShouldBeNil) + sess.DropTable(event{}) + } + err := sess.CreateTable(event{}) So(err, ShouldBeNil) - column := queryResult.Tables[0].Rows[0] + events := []*event{} + for _, t := range genTimeRangeByInterval(fromStart.Add(-20*time.Minute), 60*time.Minute, 25*time.Minute) { + events = append(events, &event{ + TimeSec: t.Unix(), + Description: "Someone deployed something", + Tags: "deploy", + }) + events = append(events, &event{ + TimeSec: t.Add(5 * time.Minute).Unix(), + Description: "New support ticket registered", + Tags: "ticket", + }) + } - So(*column[0].(*int8), ShouldEqual, 1) - So(column[1].(string), ShouldEqual, "abc") - So(column[2].(string), ShouldEqual, "def") - So(*column[3].(*int32), ShouldEqual, 1) - So(*column[4].(*int16), ShouldEqual, 10) - So(*column[5].(*int64), ShouldEqual, 100) - So(*column[6].(*int32), ShouldEqual, 1420070400) - So(column[7].(float64), ShouldEqual, 1.11) - So(column[8].(float64), ShouldEqual, 2.22) - So(*column[9].(*float32), ShouldEqual, 3.33) - _, offset := time.Now().Zone() - So(column[10].(time.Time), ShouldHappenWithin, time.Duration(10*time.Second), time.Now().Add(time.Duration(offset)*time.Second)) - So(column[11].(time.Time), ShouldHappenWithin, time.Duration(10*time.Second), time.Now().Add(time.Duration(offset)*time.Second)) - So(column[12].(string), ShouldEqual, "11:11:11") - So(*column[13].(*[]byte), ShouldHaveSameTypeAs, []byte{1}) - So(column[14].(string), ShouldEqual, "tinytext") - So(column[15].(string), ShouldEqual, "tinyblob") - So(column[16].(string), ShouldEqual, "text") - So(column[17].(string), ShouldEqual, "blob") - So(column[18].(string), ShouldEqual, "mediumtext") - So(column[19].(string), ShouldEqual, "mediumblob") - So(column[20].(string), ShouldEqual, "longtext") - So(column[21].(string), ShouldEqual, "longblob") - So(column[22].(string), ShouldEqual, "val2") - So(column[23].(string), ShouldEqual, "a,b") - So(column[24].(time.Time).Format("2006-01-02T00:00:00Z"), ShouldEqual, time.Now().Format("2006-01-02T00:00:00Z")) - So(column[25].(float64), ShouldEqual, 1514764861) - So(column[26], ShouldEqual, nil) - So(column[27], ShouldEqual, nil) - So(column[28], ShouldEqual, "") - So(column[29], ShouldEqual, nil) + for _, e := range events { + _, err = sess.Insert(e) + So(err, ShouldBeNil) + } + + Convey("When doing an annotation query of deploy events should return expected result", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT time_sec, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC`, + "format": "table", + }), + RefId: "Deploys", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + queryResult := resp.Results["Deploys"] + So(err, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 3) + }) + + Convey("When doing an annotation query of ticket events should return expected result", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT time_sec, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC`, + "format": "table", + }), + RefId: "Tickets", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + queryResult := resp.Results["Tickets"] + So(err, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 3) + }) + + Convey("When doing an annotation query with a time column in datetime format", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 0, time.UTC) + dtFormat := "2006-01-02 15:04:05.999999999" + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + CAST('%s' as datetime) as time_sec, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Format(dtFormat)), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(float64), ShouldEqual, float64(dt.Unix()*1000)) + }) + + Convey("When doing an annotation query with a time column in epoch second format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time_sec, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column in epoch second format (signed integer) should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 0, time.Local) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + CAST('%d' as signed integer) as time_sec, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column in epoch millisecond format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time_sec, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()*1000), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column holding a unsigned integer null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as unsigned integer) as time_sec, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) + }) + + Convey("When doing an annotation query with a time column holding a DATETIME null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as DATETIME) as time_sec, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) + }) }) }) } func InitMySQLTestDB(t *testing.T) *xorm.Engine { - x, err := xorm.NewEngine(sqlutil.TestDB_Mysql.DriverName, sqlutil.TestDB_Mysql.ConnStr+"&parseTime=true") - - // x.ShowSQL() - + x, err := xorm.NewEngine(sqlutil.TestDB_Mysql.DriverName, strings.Replace(sqlutil.TestDB_Mysql.ConnStr, "/grafana_tests", "/grafana_ds_tests", 1)) if err != nil { t.Fatalf("Failed to init mysql db %v", err) } - sqlutil.CleanDB(x) + x.DatabaseTZ = time.UTC + x.TZLocation = time.UTC + + // x.ShowSQL() return x } + +func genTimeRangeByInterval(from time.Time, duration time.Duration, interval time.Duration) []time.Time { + durationSec := int64(duration.Seconds()) + intervalSec := int64(interval.Seconds()) + timeRange := []time.Time{} + + for i := int64(0); i < durationSec; i += intervalSec { + timeRange = append(timeRange, from) + from = from.Add(time.Duration(int64(time.Second) * intervalSec)) + } + + return timeRange +} diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index 692b891eddd..16da764de54 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -83,6 +83,10 @@ func (e *OpenTsdbExecutor) createRequest(dsInfo *models.DataSource, data OpenTsd u.Path = path.Join(u.Path, "api/query") postData, err := json.Marshal(data) + if err != nil { + plog.Info("Failed marshalling data", "error", err) + return nil, fmt.Errorf("Failed to create request. error: %v", err) + } req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(string(postData))) if err != nil { diff --git a/pkg/tsdb/opentsdb/opentsdb_test.go b/pkg/tsdb/opentsdb/opentsdb_test.go index 094deb9e8ec..fe03599f54d 100644 --- a/pkg/tsdb/opentsdb/opentsdb_test.go +++ b/pkg/tsdb/opentsdb/opentsdb_test.go @@ -35,7 +35,7 @@ func TestOpenTsdbExecutor(t *testing.T) { }) - Convey("Build metric with downsampling diabled", func() { + Convey("Build metric with downsampling disabled", func() { query := &tsdb.Query{ Model: simplejson.New(), diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index 23daeebec5a..0fa5d8077e1 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -3,33 +3,53 @@ package postgres import ( "fmt" "regexp" - "strconv" "strings" "time" "github.com/grafana/grafana/pkg/tsdb" ) -//const rsString = `(?:"([^"]*)")`; const rsIdentifier = `([_a-zA-Z0-9]+)` const sExpr = `\$` + rsIdentifier + `\(([^\)]*)\)` -type PostgresMacroEngine struct { - TimeRange *tsdb.TimeRange - Query *tsdb.Query +type postgresMacroEngine struct { + *tsdb.SqlMacroEngineBase + timeRange *tsdb.TimeRange + query *tsdb.Query + timescaledb bool } -func NewPostgresMacroEngine() tsdb.SqlMacroEngine { - return &PostgresMacroEngine{} +func newPostgresMacroEngine(timescaledb bool) tsdb.SqlMacroEngine { + return &postgresMacroEngine{ + SqlMacroEngineBase: tsdb.NewSqlMacroEngineBase(), + timescaledb: timescaledb, + } } -func (m *PostgresMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { - m.TimeRange = timeRange - m.Query = query +func (m *postgresMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { + m.timeRange = timeRange + m.query = query rExp, _ := regexp.Compile(sExpr) var macroError error - sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { + sql = m.ReplaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { + + // detect if $__timeGroup is supposed to add AS time for pre 5.3 compatibility + // if there is a ',' directly after the macro call $__timeGroup is probably used + // in the old way. Inside window function ORDER BY $__timeGroup will be followed + // by ')' + if groups[1] == "__timeGroup" { + if index := strings.Index(sql, groups[0]); index >= 0 { + index += len(groups[0]) + if len(sql) > index { + // check for character after macro expression + if sql[index] == ',' { + groups[1] = "__timeGroupAlias" + } + } + } + } + args := strings.Split(groups[2], ",") for i, arg := range args { args[i] = strings.Trim(arg, " ") @@ -49,24 +69,7 @@ func (m *PostgresMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.Tim return sql, nil } -func replaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string { - result := "" - lastIndex := 0 - - for _, v := range re.FindAllSubmatchIndex([]byte(str), -1) { - groups := []string{} - for i := 0; i < len(v); i += 2 { - groups = append(groups, str[v[i]:v[i+1]]) - } - - result += str[lastIndex:v[0]] + repl(groups) - lastIndex = v[1] - } - - return result + str[lastIndex:] -} - -func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, error) { +func (m *postgresMacroEngine) evaluateMacro(name string, args []string) (string, error) { switch name { case "__time": if len(args) == 0 { @@ -79,15 +82,11 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, } return fmt.Sprintf("extract(epoch from %s) as \"time\"", args[0]), nil case "__timeFilter": - // dont use to_timestamp in this macro for redshift compatibility #9566 if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("extract(epoch from %s) BETWEEN %d AND %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil - case "__timeFrom": - return fmt.Sprintf("to_timestamp(%d)", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil - case "__timeTo": - return fmt.Sprintf("to_timestamp(%d)", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + + return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], m.timeRange.GetFromAsTimeUTC().Format(time.RFC3339), m.timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name) @@ -97,28 +96,49 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, return "", fmt.Errorf("error parsing interval %v", args[1]) } if len(args) == 3 { - m.Query.Model.Set("fill", true) - m.Query.Model.Set("fillInterval", interval.Seconds()) - if args[2] == "NULL" { - m.Query.Model.Set("fillNull", true) - } else { - floatVal, err := strconv.ParseFloat(args[2], 64) - if err != nil { - return "", fmt.Errorf("error parsing fill value %v", args[2]) - } - m.Query.Model.Set("fillValue", floatVal) + err := tsdb.SetupFillmode(m.query, interval, args[2]) + if err != nil { + return "", err } } - return fmt.Sprintf("(extract(epoch from %s)/%v)::bigint*%v AS time", args[0], interval.Seconds(), interval.Seconds()), nil + + if m.timescaledb { + return fmt.Sprintf("time_bucket('%vs',%s)", interval.Seconds(), args[0]), nil + } else { + return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v", args[0], interval.Seconds(), interval.Seconds()), nil + } + case "__timeGroupAlias": + tg, err := m.evaluateMacro("__timeGroup", args) + if err == nil { + return tg + " AS \"time\"", err + } + return "", err case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil - case "__unixEpochFrom": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil - case "__unixEpochTo": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], m.timeRange.GetFromAsSecondsEpoch(), args[0], m.timeRange.GetToAsSecondsEpoch()), nil + case "__unixEpochGroup": + if len(args) < 2 { + return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name) + } + interval, err := time.ParseDuration(strings.Trim(args[1], `'`)) + if err != nil { + return "", fmt.Errorf("error parsing interval %v", args[1]) + } + if len(args) == 3 { + err := tsdb.SetupFillmode(m.query, interval, args[2]) + if err != nil { + return "", err + } + } + return fmt.Sprintf("floor(%s/%v)*%v", args[0], interval.Seconds(), interval.Seconds()), nil + case "__unixEpochGroupAlias": + tg, err := m.evaluateMacro("__unixEpochGroup", args) + if err == nil { + return tg + " AS \"time\"", err + } + return "", err default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index b18acced963..8a3699f82b2 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -1,7 +1,10 @@ package postgres import ( + "fmt" + "strconv" "testing" + "time" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" @@ -9,81 +12,147 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { - engine := &PostgresMacroEngine{} + timescaledbEnabled := false + engine := newPostgresMacroEngine(timescaledbEnabled) + timescaledbEnabled = true + engineTS := newPostgresMacroEngine(timescaledbEnabled) query := &tsdb.Query{} - timeRange := &tsdb.TimeRange{From: "5m", To: "now"} - Convey("interpolate __time function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { + from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) + to := from.Add(5 * time.Minute) + timeRange := tsdb.NewFakeTimeRange("5m", "now", to) + + Convey("interpolate __time function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select time_column AS \"time\"") + }) + + Convey("interpolate __time function wrapped in aggregation", func() { + sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select min(time_column AS \"time\")") + }) + + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN '%s' AND '%s'", from.Format(time.RFC3339), to.Format(time.RFC3339))) + }) + + Convey("interpolate __timeGroup function pre 5.3 compatibility", func() { + + sql, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m'), value") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "SELECT floor(extract(epoch from time_column)/300)*300 AS \"time\", value") + + sql, err = engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m') as time, value") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "SELECT floor(extract(epoch from time_column)/300)*300 as time, value") + }) + + Convey("interpolate __timeGroup function", func() { + + sql, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroupAlias(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "SELECT floor(extract(epoch from time_column)/300)*300") + So(sql2, ShouldEqual, sql+" AS \"time\"") + }) + + Convey("interpolate __timeGroup function with spaces between args", func() { + + sql, err := engine.Interpolate(query, timeRange, "$__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "$__timeGroupAlias(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "floor(extract(epoch from time_column)/300)*300") + So(sql2, ShouldEqual, sql+" AS \"time\"") + }) + + Convey("interpolate __timeGroup function with TimescaleDB enabled", func() { + + sql, err := engineTS.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column)") + }) + + Convey("interpolate __timeGroup function with spaces between args and TimescaleDB enabled", func() { + + sql, err := engineTS.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column)") + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochGroup function", func() { + + sql, err := engine.Interpolate(query, timeRange, "SELECT $__unixEpochGroup(time_column,'5m')") + So(err, ShouldBeNil) + sql2, err := engine.Interpolate(query, timeRange, "SELECT $__unixEpochGroupAlias(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "SELECT floor(time_column/300)*300") + So(sql2, ShouldEqual, sql+" AS \"time\"") + }) - So(sql, ShouldEqual, "select time_column AS \"time\"") }) - Convey("interpolate __time function wrapped in aggregation", func() { - sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select min(time_column AS \"time\")") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN '%s' AND '%s'", from.Format(time.RFC3339), to.Format(time.RFC3339))) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) }) - Convey("interpolate __timeFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "WHERE extract(epoch from time_column) BETWEEN 18446744066914186738 AND 18446744066914187038") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column BETWEEN '%s' AND '%s'", from.Format(time.RFC3339), to.Format(time.RFC3339))) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) }) - - Convey("interpolate __timeFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select to_timestamp(18446744066914186738)") - }) - - Convey("interpolate __timeGroup function", func() { - - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time") - }) - - Convey("interpolate __timeGroup function with spaces between args", func() { - - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time") - }) - - Convey("interpolate __timeTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select to_timestamp(18446744066914187038)") - }) - - Convey("interpolate __unixEpochFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(18446744066914186738)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038") - }) - - Convey("interpolate __unixEpochFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738") - }) - - Convey("interpolate __unixEpochTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914187038") - }) - }) } diff --git a/pkg/tsdb/postgres/postgres.go b/pkg/tsdb/postgres/postgres.go index 6a084ad1237..4bcf06638f4 100644 --- a/pkg/tsdb/postgres/postgres.go +++ b/pkg/tsdb/postgres/postgres.go @@ -1,47 +1,40 @@ package postgres import ( - "container/list" - "context" - "fmt" - "math" + "database/sql" "net/url" "strconv" - "time" "github.com/go-xorm/core" - "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" ) -type PostgresQueryEndpoint struct { - sqlEngine tsdb.SqlEngine - log log.Logger -} - func init() { - tsdb.RegisterTsdbQueryEndpoint("postgres", NewPostgresQueryEndpoint) + tsdb.RegisterTsdbQueryEndpoint("postgres", newPostgresQueryEndpoint) } -func NewPostgresQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { - endpoint := &PostgresQueryEndpoint{ - log: log.New("tsdb.postgres"), - } - - endpoint.sqlEngine = &tsdb.DefaultSqlEngine{ - MacroEngine: NewPostgresMacroEngine(), - } +func newPostgresQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + logger := log.New("tsdb.postgres") cnnstr := generateConnectionString(datasource) - endpoint.log.Debug("getEngine", "connection", cnnstr) + logger.Debug("getEngine", "connection", cnnstr) - if err := endpoint.sqlEngine.InitEngine("postgres", datasource, cnnstr); err != nil { - return nil, err + config := tsdb.SqlQueryEndpointConfiguration{ + DriverName: "postgres", + ConnectionString: cnnstr, + Datasource: datasource, + MetricColumnTypes: []string{"UNKNOWN", "TEXT", "VARCHAR", "CHAR"}, } - return endpoint, nil + rowTransformer := postgresRowTransformer{ + log: logger, + } + + timescaledb := datasource.JsonData.Get("timescaledb").MustBool(false) + + return tsdb.NewSqlQueryEndpoint(&config, &rowTransformer, newPostgresMacroEngine(timescaledb), logger) } func generateConnectionString(datasource *models.DataSource) string { @@ -54,80 +47,25 @@ func generateConnectionString(datasource *models.DataSource) string { } sslmode := datasource.JsonData.Get("sslmode").MustString("verify-full") - u := &url.URL{Scheme: "postgres", User: url.UserPassword(datasource.User, password), Host: datasource.Url, Path: datasource.Database, RawQuery: "sslmode=" + sslmode} + u := &url.URL{ + Scheme: "postgres", + User: url.UserPassword(datasource.User, password), + Host: datasource.Url, Path: datasource.Database, + RawQuery: "sslmode=" + url.QueryEscape(sslmode), + } + return u.String() } -func (e *PostgresQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable) +type postgresRowTransformer struct { + log log.Logger } -func (e PostgresQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error { +func (t *postgresRowTransformer) Transform(columnTypes []*sql.ColumnType, rows *core.Rows) (tsdb.RowValues, error) { + values := make([]interface{}, len(columnTypes)) + valuePtrs := make([]interface{}, len(columnTypes)) - columnNames, err := rows.Columns() - if err != nil { - return err - } - - table := &tsdb.Table{ - Columns: make([]tsdb.TableColumn, len(columnNames)), - Rows: make([]tsdb.RowValues, 0), - } - - for i, name := range columnNames { - table.Columns[i].Text = name - } - - rowLimit := 1000000 - rowCount := 0 - timeIndex := -1 - - // check if there is a column named time - for i, col := range columnNames { - switch col { - case "time": - timeIndex = i - } - } - - for ; rows.Next(); rowCount++ { - if rowCount > rowLimit { - return fmt.Errorf("PostgreSQL query row limit exceeded, limit %d", rowLimit) - } - - values, err := e.getTypedRowData(rows) - if err != nil { - return err - } - - // convert column named time to unix timestamp to make - // native datetime postgres types work in annotation queries - if timeIndex != -1 { - switch value := values[timeIndex].(type) { - case time.Time: - values[timeIndex] = float64(value.UnixNano() / 1e9) - } - } - - table.Rows = append(table.Rows, values) - } - - result.Tables = append(result.Tables, table) - result.Meta.Set("rowCount", rowCount) - return nil -} - -func (e PostgresQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, error) { - - types, err := rows.ColumnTypes() - if err != nil { - return nil, err - } - - values := make([]interface{}, len(types)) - valuePtrs := make([]interface{}, len(types)) - - for i := 0; i < len(types); i++ { + for i := 0; i < len(columnTypes); i++ { valuePtrs[i] = &values[i] } @@ -137,20 +75,20 @@ func (e PostgresQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, // convert types not handled by lib/pq // unhandled types are returned as []byte - for i := 0; i < len(types); i++ { - if value, ok := values[i].([]byte); ok == true { - switch types[i].DatabaseTypeName() { + for i := 0; i < len(columnTypes); i++ { + if value, ok := values[i].([]byte); ok { + switch columnTypes[i].DatabaseTypeName() { case "NUMERIC": if v, err := strconv.ParseFloat(string(value), 64); err == nil { values[i] = v } else { - e.log.Debug("Rows", "Error converting numeric to float", value) + t.log.Debug("Rows", "Error converting numeric to float", value) } case "UNKNOWN", "CIDR", "INET", "MACADDR": // char literals have type UNKNOWN values[i] = string(value) default: - e.log.Debug("Rows", "Unknown database type", types[i].DatabaseTypeName(), "value", value) + t.log.Debug("Rows", "Unknown database type", columnTypes[i].DatabaseTypeName(), "value", value) values[i] = string(value) } } @@ -158,163 +96,3 @@ func (e PostgresQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, return values, nil } - -func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult, tsdbQuery *tsdb.TsdbQuery) error { - pointsBySeries := make(map[string]*tsdb.TimeSeries) - seriesByQueryOrder := list.New() - - columnNames, err := rows.Columns() - if err != nil { - return err - } - - columnTypes, err := rows.ColumnTypes() - if err != nil { - return err - } - - rowLimit := 1000000 - rowCount := 0 - timeIndex := -1 - metricIndex := -1 - - // check columns of resultset: a column named time is mandatory - // the first text column is treated as metric name unless a column named metric is present - for i, col := range columnNames { - switch col { - case "time": - timeIndex = i - case "metric": - metricIndex = i - default: - if metricIndex == -1 { - switch columnTypes[i].DatabaseTypeName() { - case "UNKNOWN", "TEXT", "VARCHAR", "CHAR": - metricIndex = i - } - } - } - } - - if timeIndex == -1 { - return fmt.Errorf("Found no column named time") - } - - fillMissing := query.Model.Get("fill").MustBool(false) - var fillInterval float64 - fillValue := null.Float{} - if fillMissing { - fillInterval = query.Model.Get("fillInterval").MustFloat64() * 1000 - if query.Model.Get("fillNull").MustBool(false) == false { - fillValue.Float64 = query.Model.Get("fillValue").MustFloat64() - fillValue.Valid = true - } - - } - - for rows.Next() { - var timestamp float64 - var value null.Float - var metric string - - if rowCount > rowLimit { - return fmt.Errorf("PostgreSQL query row limit exceeded, limit %d", rowLimit) - } - - values, err := e.getTypedRowData(rows) - if err != nil { - return err - } - - switch columnValue := values[timeIndex].(type) { - case int64: - timestamp = float64(columnValue * 1000) - case float64: - timestamp = columnValue * 1000 - case time.Time: - timestamp = float64(columnValue.UnixNano() / 1e6) - default: - return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp, got: %T %v", columnValue, columnValue) - } - - if metricIndex >= 0 { - if columnValue, ok := values[metricIndex].(string); ok == true { - metric = columnValue - } else { - return fmt.Errorf("Column metric must be of type char,varchar or text, got: %T %v", values[metricIndex], values[metricIndex]) - } - } - - for i, col := range columnNames { - if i == timeIndex || i == metricIndex { - continue - } - - switch columnValue := values[i].(type) { - case int64: - value = null.FloatFrom(float64(columnValue)) - case float64: - value = null.FloatFrom(columnValue) - case nil: - value.Valid = false - default: - return fmt.Errorf("Value column must have numeric datatype, column: %s type: %T value: %v", col, columnValue, columnValue) - } - if metricIndex == -1 { - metric = col - } - - series, exist := pointsBySeries[metric] - if exist == false { - series = &tsdb.TimeSeries{Name: metric} - pointsBySeries[metric] = series - seriesByQueryOrder.PushBack(metric) - } - - if fillMissing { - var intervalStart float64 - if exist == false { - intervalStart = float64(tsdbQuery.TimeRange.MustGetFrom().UnixNano() / 1e6) - } else { - intervalStart = series.Points[len(series.Points)-1][1].Float64 + fillInterval - } - - // align interval start - intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval - - for i := intervalStart; i < timestamp; i += fillInterval { - series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)}) - rowCount++ - } - } - - series.Points = append(series.Points, tsdb.TimePoint{value, null.FloatFrom(timestamp)}) - - e.log.Debug("Rows", "metric", metric, "time", timestamp, "value", value) - rowCount++ - - } - } - - for elem := seriesByQueryOrder.Front(); elem != nil; elem = elem.Next() { - key := elem.Value.(string) - result.Series = append(result.Series, pointsBySeries[key]) - - if fillMissing { - series := pointsBySeries[key] - // fill in values from last fetched value till interval end - intervalStart := series.Points[len(series.Points)-1][1].Float64 - intervalEnd := float64(tsdbQuery.TimeRange.MustGetTo().UnixNano() / 1e6) - - // align interval start - intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval - for i := intervalStart + fillInterval; i < intervalEnd; i += fillInterval { - series.Points = append(series.Points, tsdb.TimePoint{fillValue, null.FloatFrom(i)}) - rowCount++ - } - } - } - - result.Meta.Set("rowCount", rowCount) - return nil -} diff --git a/pkg/tsdb/postgres/postgres_test.go b/pkg/tsdb/postgres/postgres_test.go index 75e8cb77f2e..c381938aead 100644 --- a/pkg/tsdb/postgres/postgres_test.go +++ b/pkg/tsdb/postgres/postgres_test.go @@ -1,125 +1,997 @@ package postgres import ( + "context" + "fmt" + "math/rand" + "strings" "testing" "time" "github.com/go-xorm/xorm" + "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" "github.com/grafana/grafana/pkg/tsdb" _ "github.com/lib/pq" . "github.com/smartystreets/goconvey/convey" ) -// To run this test, remove the Skip from SkipConvey -// and set up a PostgreSQL db named grafanatest and a user/password grafanatest/grafanatest +// To run this test, set runPostgresTests=true +// Or from the commandline: GRAFANA_TEST_DB=postgres go test -v ./pkg/tsdb/postgres +// The tests require a PostgreSQL db named grafanadstest and a user/password grafanatest/grafanatest! +// Use the docker/blocks/postgres_tests/docker-compose.yaml to spin up a +// preconfigured Postgres server suitable for running these tests. +// There is also a datasource and dashboard provisioned by devenv scripts that you can +// use to verify that the generated data are vizualized as expected, see +// devenv/README.md for setup instructions. func TestPostgres(t *testing.T) { - SkipConvey("PostgreSQL", t, func() { + // change to true to run the PostgreSQL tests + runPostgresTests := false + // runPostgresTests := true + + if !(sqlstore.IsTestDbPostgres() || runPostgresTests) { + t.Skip() + } + + Convey("PostgreSQL", t, func() { x := InitPostgresTestDB(t) - endpoint := &PostgresQueryEndpoint{ - sqlEngine: &tsdb.DefaultSqlEngine{ - MacroEngine: NewPostgresMacroEngine(), - XormEngine: x, - }, - log: log.New("tsdb.postgres"), + origXormEngine := tsdb.NewXormEngine + tsdb.NewXormEngine = func(d, c string) (*xorm.Engine, error) { + return x, nil } + origInterpolate := tsdb.Interpolate + tsdb.Interpolate = func(query *tsdb.Query, timeRange *tsdb.TimeRange, sql string) (string, error) { + return sql, nil + } + + endpoint, err := newPostgresQueryEndpoint(&models.DataSource{ + JsonData: simplejson.New(), + SecureJsonData: securejsondata.SecureJsonData{}, + }) + So(err, ShouldBeNil) + sess := x.NewSession() - defer sess.Close() + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) - sql := ` - CREATE TABLE postgres_types( - c00_smallint smallint, - c01_integer integer, - c02_bigint bigint, + Reset(func() { + sess.Close() + tsdb.NewXormEngine = origXormEngine + tsdb.Interpolate = origInterpolate + }) - c03_real real, - c04_double double precision, - c05_decimal decimal(10,2), - c06_numeric numeric(10,2), + Convey("Given a table with different native data types", func() { + sql := ` + DROP TABLE IF EXISTS postgres_types; + CREATE TABLE postgres_types( + c00_smallint smallint, + c01_integer integer, + c02_bigint bigint, - c07_char char(10), - c08_varchar varchar(10), - c09_text text, + c03_real real, + c04_double double precision, + c05_decimal decimal(10,2), + c06_numeric numeric(10,2), - c10_timestamp timestamp without time zone, - c11_timestamptz timestamp with time zone, - c12_date date, - c13_time time without time zone, - c14_timetz time with time zone, - c15_interval interval - ); - ` - _, err := sess.Exec(sql) - So(err, ShouldBeNil) + c07_char char(10), + c08_varchar varchar(10), + c09_text text, - sql = ` - INSERT INTO postgres_types VALUES( - 1,2,3, - 4.5,6.7,1.1,1.2, - 'char10','varchar10','text', + c10_timestamp timestamp without time zone, + c11_timestamptz timestamp with time zone, + c12_date date, + c13_time time without time zone, + c14_timetz time with time zone, - now(),now(),now(),now(),now(),'15m'::interval - ); - ` - _, err = sess.Exec(sql) - So(err, ShouldBeNil) + c15_interval interval + ); + ` + _, err := sess.Exec(sql) + So(err, ShouldBeNil) - Convey("Query with Table format should map PostgreSQL column types to Go types", func() { + sql = ` + INSERT INTO postgres_types VALUES( + 1,2,3, + 4.5,6.7,1.1,1.2, + 'char10','varchar10','text', + + now(),now(),now(),now(),now(),'15m'::interval + ); + ` + _, err = sess.Exec(sql) + So(err, ShouldBeNil) + + Convey("When doing a table query should map Postgres column types to Go types", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT * FROM postgres_types", + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + column := queryResult.Tables[0].Rows[0] + So(column[0].(int64), ShouldEqual, 1) + So(column[1].(int64), ShouldEqual, 2) + So(column[2].(int64), ShouldEqual, 3) + + So(column[3].(float64), ShouldEqual, 4.5) + So(column[4].(float64), ShouldEqual, 6.7) + So(column[5].(float64), ShouldEqual, 1.1) + So(column[6].(float64), ShouldEqual, 1.2) + + So(column[7].(string), ShouldEqual, "char10 ") + So(column[8].(string), ShouldEqual, "varchar10") + So(column[9].(string), ShouldEqual, "text") + + So(column[10].(time.Time), ShouldHaveSameTypeAs, time.Now()) + So(column[11].(time.Time), ShouldHaveSameTypeAs, time.Now()) + So(column[12].(time.Time), ShouldHaveSameTypeAs, time.Now()) + So(column[13].(time.Time), ShouldHaveSameTypeAs, time.Now()) + So(column[14].(time.Time), ShouldHaveSameTypeAs, time.Now()) + + So(column[15].(string), ShouldEqual, "00:15:00") + }) + }) + + Convey("Given a table with metrics that lacks data for some series ", func() { + sql := ` + DROP TABLE IF EXISTS metric; + CREATE TABLE metric ( + time timestamp, + value integer + ) + ` + + _, err := sess.Exec(sql) + So(err, ShouldBeNil) + + type metric struct { + Time time.Time + Value int64 + } + + series := []*metric{} + firstRange := genTimeRangeByInterval(fromStart, 10*time.Minute, 10*time.Second) + secondRange := genTimeRangeByInterval(fromStart.Add(20*time.Minute), 10*time.Minute, 10*time.Second) + + for _, t := range firstRange { + series = append(series, &metric{ + Time: t, + Value: 15, + }) + } + + for _, t := range secondRange { + series = append(series, &metric{ + Time: t, + Value: 20, + }) + } + + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) + + Convey("When doing a metric query using timeGroup", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + // without fill this should result in 4 buckets + So(len(points), ShouldEqual, 4) + + dt := fromStart + + for i := 0; i < 2; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 15) + So(aTime, ShouldEqual, dt) + So(aTime.Unix()%300, ShouldEqual, 0) + dt = dt.Add(5 * time.Minute) + } + + // adjust for 10 minute gap between first and second set of points + dt = dt.Add(10 * time.Minute) + for i := 2; i < 4; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 20) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + }) + + Convey("When doing a metric query using timeGroup and $__interval", func() { + mockInterpolate := tsdb.Interpolate + tsdb.Interpolate = origInterpolate + + Reset(func() { + tsdb.Interpolate = mockInterpolate + }) + + Convey("Should replace $__interval", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + DataSource: &models.DataSource{}, + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, $__interval) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(30*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(queryResult.Meta.Get("sql").MustString(), ShouldEqual, "SELECT floor(extract(epoch from time)/60)*60 AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1") + }) + }) + + Convey("When doing a metric query using timeGroup with NULL fill enabled", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + So(len(points), ShouldEqual, 7) + + dt := fromStart + + for i := 0; i < 2; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 15) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + + // check for NULL values inserted by fill + So(points[2][0].Valid, ShouldBeFalse) + So(points[3][0].Valid, ShouldBeFalse) + + // adjust for 10 minute gap between first and second set of points + dt = dt.Add(10 * time.Minute) + for i := 4; i < 6; i++ { + aValue := points[i][0].Float64 + aTime := time.Unix(int64(points[i][1].Float64)/1000, 0) + So(aValue, ShouldEqual, 20) + So(aTime, ShouldEqual, dt) + dt = dt.Add(5 * time.Minute) + } + + // check for NULL values inserted by fill + So(points[6][0].Valid, ShouldBeFalse) + + }) + + Convey("When doing a metric query using timeGroup with value fill enabled", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": "SELECT $__timeGroup(time, '5m', 1.5) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", + }), + RefId: "A", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + So(points[3][0].Float64, ShouldEqual, 1.5) + }) + }) + + Convey("When doing a metric query using timeGroup with previous fill enabled", func() { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { Model: simplejson.NewFromAny(map[string]interface{}{ - "rawSql": "SELECT * FROM postgres_types", - "format": "table", + "rawSql": "SELECT $__timeGroup(time, '5m', previous), avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", + "format": "time_series", }), RefId: "A", }, }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, } - resp, err := endpoint.Query(nil, nil, query) + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + points := queryResult.Series[0].Points + So(points[2][0].Float64, ShouldEqual, 15.0) + So(points[3][0].Float64, ShouldEqual, 15.0) + So(points[6][0].Float64, ShouldEqual, 20.0) + }) + + Convey("Given a table with metrics having multiple values and measurements", func() { + type metric_values struct { + Time time.Time + TimeInt64 int64 `xorm:"bigint 'timeInt64' not null"` + TimeInt64Nullable *int64 `xorm:"bigint 'timeInt64Nullable' null"` + TimeFloat64 float64 `xorm:"double 'timeFloat64' not null"` + TimeFloat64Nullable *float64 `xorm:"double 'timeFloat64Nullable' null"` + TimeInt32 int32 `xorm:"int(11) 'timeInt32' not null"` + TimeInt32Nullable *int32 `xorm:"int(11) 'timeInt32Nullable' null"` + TimeFloat32 float32 `xorm:"double 'timeFloat32' not null"` + TimeFloat32Nullable *float32 `xorm:"double 'timeFloat32Nullable' null"` + Measurement string + ValueOne int64 `xorm:"integer 'valueOne'"` + ValueTwo int64 `xorm:"integer 'valueTwo'"` + } + + if exist, err := sess.IsTableExist(metric_values{}); err != nil || exist { + So(err, ShouldBeNil) + sess.DropTable(metric_values{}) + } + err := sess.CreateTable(metric_values{}) So(err, ShouldBeNil) - column := queryResult.Tables[0].Rows[0] - So(column[0].(int64), ShouldEqual, 1) - So(column[1].(int64), ShouldEqual, 2) - So(column[2].(int64), ShouldEqual, 3) - So(column[3].(float64), ShouldEqual, 4.5) - So(column[4].(float64), ShouldEqual, 6.7) - // libpq doesnt properly convert decimal, numeric and char to go types but returns []uint8 instead - // So(column[5].(float64), ShouldEqual, 1.1) - // So(column[6].(float64), ShouldEqual, 1.2) - // So(column[7].(string), ShouldEqual, "char") - So(column[8].(string), ShouldEqual, "varchar10") - So(column[9].(string), ShouldEqual, "text") + rand.Seed(time.Now().Unix()) + rnd := func(min, max int64) int64 { + return rand.Int63n(max-min) + min + } - So(column[10].(time.Time), ShouldHaveSameTypeAs, time.Now()) - So(column[11].(time.Time), ShouldHaveSameTypeAs, time.Now()) - So(column[12].(time.Time), ShouldHaveSameTypeAs, time.Now()) - So(column[13].(time.Time), ShouldHaveSameTypeAs, time.Now()) - So(column[14].(time.Time), ShouldHaveSameTypeAs, time.Now()) + var tInitial time.Time - // libpq doesnt properly convert interval to go types but returns []uint8 instead - // So(column[15].(time.Time), ShouldHaveSameTypeAs, time.Now()) + series := []*metric_values{} + for i, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) { + if i == 0 { + tInitial = t + } + tSeconds := t.Unix() + tSecondsInt32 := int32(tSeconds) + tSecondsFloat32 := float32(tSeconds) + tMilliseconds := tSeconds * 1e3 + tMillisecondsFloat := float64(tMilliseconds) + first := metric_values{ + Time: t, + TimeInt64: tMilliseconds, + TimeInt64Nullable: &(tMilliseconds), + TimeFloat64: tMillisecondsFloat, + TimeFloat64Nullable: &tMillisecondsFloat, + TimeInt32: tSecondsInt32, + TimeInt32Nullable: &tSecondsInt32, + TimeFloat32: tSecondsFloat32, + TimeFloat32Nullable: &tSecondsFloat32, + Measurement: "Metric A", + ValueOne: rnd(0, 100), + ValueTwo: rnd(0, 100), + } + second := first + second.Measurement = "Metric B" + second.ValueOne = rnd(0, 100) + second.ValueTwo = rnd(0, 100) + + series = append(series, &first) + series = append(series, &second) + } + + _, err = sess.InsertMulti(series) + So(err, ShouldBeNil) + + Convey("When doing a metric query using epoch (int64) as time column and value column (int64) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeInt64" as time, "timeInt64" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int64 nullable) as time column and value column (int64 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeInt64Nullable" as time, "timeInt64Nullable" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64) as time column and value column (float64) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeFloat64" as time, "timeFloat64" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float64 nullable) as time column and value column (float64 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeFloat64Nullable" as time, "timeFloat64Nullable" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int32) as time column and value column (int32) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeInt32" as time, "timeInt32" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (int32 nullable) as time column and value column (int32 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeInt32Nullable" as time, "timeInt32Nullable" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(tInitial.UnixNano()/1e6)) + }) + + Convey("When doing a metric query using epoch (float32) as time column and value column (float32) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeFloat32" as time, "timeFloat32" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float32(tInitial.Unix()))*1e3) + }) + + Convey("When doing a metric query using epoch (float32 nullable) as time column and value column (float32 nullable) should return metric with time in milliseconds", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "timeFloat32Nullable" as time, "timeFloat32Nullable" FROM metric_values ORDER BY time LIMIT 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 1) + So(queryResult.Series[0].Points[0][1].Float64, ShouldEqual, float64(float32(tInitial.Unix()))*1e3) + }) + + Convey("When doing a metric query grouping by time and select metric column should return correct series", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT $__timeEpoch(time), measurement || ' - value one' as metric, "valueOne" FROM metric_values ORDER BY 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 2) + So(queryResult.Series[0].Name, ShouldEqual, "Metric A - value one") + So(queryResult.Series[1].Name, ShouldEqual, "Metric B - value one") + }) + + Convey("When doing a metric query with metric column and multiple value columns", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT $__timeEpoch(time), measurement as metric, "valueOne", "valueTwo" FROM metric_values ORDER BY 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 4) + So(queryResult.Series[0].Name, ShouldEqual, "Metric A valueOne") + So(queryResult.Series[1].Name, ShouldEqual, "Metric A valueTwo") + So(queryResult.Series[2].Name, ShouldEqual, "Metric B valueOne") + So(queryResult.Series[3].Name, ShouldEqual, "Metric B valueTwo") + }) + + Convey("When doing a metric query grouping by time should return correct series", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT $__timeEpoch(time), "valueOne", "valueTwo" FROM metric_values ORDER BY 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + + So(len(queryResult.Series), ShouldEqual, 2) + So(queryResult.Series[0].Name, ShouldEqual, "valueOne") + So(queryResult.Series[1].Name, ShouldEqual, "valueTwo") + }) + + Convey("When doing a query with timeFrom,timeTo,unixEpochFrom,unixEpochTo macros", func() { + tsdb.Interpolate = origInterpolate + query := &tsdb.TsdbQuery{ + TimeRange: tsdb.NewFakeTimeRange("5m", "now", fromStart), + Queries: []*tsdb.Query{ + { + DataSource: &models.DataSource{JsonData: simplejson.New()}, + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeFrom() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1`, + "format": "time_series", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(queryResult.Meta.Get("sql").MustString(), ShouldEqual, "SELECT time FROM metric_values WHERE time > '2018-03-15T12:55:00Z' OR time < '2018-03-15T12:55:00Z' OR 1 < 1521118500 OR 1521118800 > 1 ORDER BY 1") + + }) + }) + + Convey("Given a table with event data", func() { + type event struct { + TimeSec int64 + Description string + Tags string + } + + if exist, err := sess.IsTableExist(event{}); err != nil || exist { + So(err, ShouldBeNil) + sess.DropTable(event{}) + } + err := sess.CreateTable(event{}) + So(err, ShouldBeNil) + + events := []*event{} + for _, t := range genTimeRangeByInterval(fromStart.Add(-20*time.Minute), 60*time.Minute, 25*time.Minute) { + events = append(events, &event{ + TimeSec: t.Unix(), + Description: "Someone deployed something", + Tags: "deploy", + }) + events = append(events, &event{ + TimeSec: t.Add(5 * time.Minute).Unix(), + Description: "New support ticket registered", + Tags: "ticket", + }) + } + + for _, e := range events { + _, err = sess.Insert(e) + So(err, ShouldBeNil) + } + + Convey("When doing an annotation query of deploy events should return expected result", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC`, + "format": "table", + }), + RefId: "Deploys", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + queryResult := resp.Results["Deploys"] + So(err, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 3) + }) + + Convey("When doing an annotation query of ticket events should return expected result", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC`, + "format": "table", + }), + RefId: "Tickets", + }, + }, + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000), + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + queryResult := resp.Results["Tickets"] + So(err, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 3) + }) + + Convey("When doing an annotation query with a time column in datetime format", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + dtFormat := "2006-01-02 15:04:05.999999999" + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + CAST('%s' AS TIMESTAMP) as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Format(dtFormat)), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(float64), ShouldEqual, float64(dt.UnixNano()/1e6)) + }) + + Convey("When doing an annotation query with a time column in epoch second format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column in epoch second format (int) should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + cast(%d as bigint) as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column in epoch millisecond format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()*1000), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, dt.Unix()*1000) + }) + + Convey("When doing an annotation query with a time column holding a bigint null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as bigint) as time, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) + }) + + Convey("When doing an annotation query with a time column holding a timestamp null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as timestamp) as time, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(context.Background(), nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) + }) }) }) } func InitPostgresTestDB(t *testing.T) *xorm.Engine { - x, err := xorm.NewEngine(sqlutil.TestDB_Postgres.DriverName, sqlutil.TestDB_Postgres.ConnStr) - - // x.ShowSQL() - + x, err := xorm.NewEngine(sqlutil.TestDB_Postgres.DriverName, strings.Replace(sqlutil.TestDB_Postgres.ConnStr, "dbname=grafanatest", "dbname=grafanadstest", 1)) if err != nil { t.Fatalf("Failed to init postgres db %v", err) } - sqlutil.CleanDB(x) + x.DatabaseTZ = time.UTC + x.TZLocation = time.UTC + + // x.ShowSQL() return x } + +func genTimeRangeByInterval(from time.Time, duration time.Duration, interval time.Duration) []time.Time { + durationSec := int64(duration.Seconds()) + intervalSec := int64(interval.Seconds()) + timeRange := []time.Time{} + + for i := int64(0); i < durationSec; i += intervalSec { + timeRange = append(timeRange, from) + from = from.Add(time.Duration(int64(time.Second) * intervalSec)) + } + + return timeRange +} diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go index 1186fccbbf9..83bb683fccf 100644 --- a/pkg/tsdb/prometheus/prometheus.go +++ b/pkg/tsdb/prometheus/prometheus.go @@ -92,12 +92,12 @@ func (e *PrometheusExecutor) Query(ctx context.Context, dsInfo *models.DataSourc return nil, err } - querys, err := parseQuery(dsInfo, tsdbQuery.Queries, tsdbQuery) + queries, err := parseQuery(dsInfo, tsdbQuery.Queries, tsdbQuery) if err != nil { return nil, err } - for _, query := range querys { + for _, query := range queries { timeRange := apiv1.Range{ Start: query.Start, End: query.End, @@ -108,8 +108,8 @@ func (e *PrometheusExecutor) Query(ctx context.Context, dsInfo *models.DataSourc span, ctx := opentracing.StartSpanFromContext(ctx, "alerting.prometheus") span.SetTag("expr", query.Expr) - span.SetTag("start_unixnano", int64(query.Start.UnixNano())) - span.SetTag("stop_unixnano", int64(query.End.UnixNano())) + span.SetTag("start_unixnano", query.Start.UnixNano()) + span.SetTag("stop_unixnano", query.End.UnixNano()) defer span.Finish() value, err := client.QueryRange(ctx, query.Expr, timeRange) diff --git a/pkg/tsdb/sql_engine.go b/pkg/tsdb/sql_engine.go index 7ea0682235f..1a4e2bd3943 100644 --- a/pkg/tsdb/sql_engine.go +++ b/pkg/tsdb/sql_engine.go @@ -1,8 +1,20 @@ package tsdb import ( + "container/list" "context" + "database/sql" + "fmt" + "math" + "regexp" + "strconv" + "strings" "sync" + "time" + + "github.com/grafana/grafana/pkg/log" + + "github.com/grafana/grafana/pkg/components/null" "github.com/go-xorm/core" "github.com/go-xorm/xorm" @@ -10,27 +22,15 @@ import ( "github.com/grafana/grafana/pkg/models" ) -// SqlEngine is a wrapper class around xorm for relational database data sources. -type SqlEngine interface { - InitEngine(driverName string, dsInfo *models.DataSource, cnnstr string) error - Query( - ctx context.Context, - ds *models.DataSource, - query *TsdbQuery, - transformToTimeSeries func(query *Query, rows *core.Rows, result *QueryResult, tsdbQuery *TsdbQuery) error, - transformToTable func(query *Query, rows *core.Rows, result *QueryResult, tsdbQuery *TsdbQuery) error, - ) (*Response, error) -} - // SqlMacroEngine interpolates macros into sql. It takes in the Query to have access to query context and // timeRange to be able to generate queries that use from and to. type SqlMacroEngine interface { Interpolate(query *Query, timeRange *TimeRange, sql string) (string, error) } -type DefaultSqlEngine struct { - MacroEngine SqlMacroEngine - XormEngine *xorm.Engine +// SqlTableRowTransformer transforms a query result row to RowValues with proper types. +type SqlTableRowTransformer interface { + Transform(columnTypes []*sql.ColumnType, rows *core.Rows) (RowValues, error) } type engineCacheType struct { @@ -44,92 +44,612 @@ var engineCache = engineCacheType{ versions: make(map[int64]int), } -// InitEngine creates the db connection and inits the xorm engine or loads it from the engine cache -func (e *DefaultSqlEngine) InitEngine(driverName string, dsInfo *models.DataSource, cnnstr string) error { +var sqlIntervalCalculator = NewIntervalCalculator(nil) + +var NewXormEngine = func(driverName string, connectionString string) (*xorm.Engine, error) { + return xorm.NewEngine(driverName, connectionString) +} + +type sqlQueryEndpoint struct { + macroEngine SqlMacroEngine + rowTransformer SqlTableRowTransformer + engine *xorm.Engine + timeColumnNames []string + metricColumnTypes []string + log log.Logger +} + +type SqlQueryEndpointConfiguration struct { + DriverName string + Datasource *models.DataSource + ConnectionString string + TimeColumnNames []string + MetricColumnTypes []string +} + +var NewSqlQueryEndpoint = func(config *SqlQueryEndpointConfiguration, rowTransformer SqlTableRowTransformer, macroEngine SqlMacroEngine, log log.Logger) (TsdbQueryEndpoint, error) { + queryEndpoint := sqlQueryEndpoint{ + rowTransformer: rowTransformer, + macroEngine: macroEngine, + timeColumnNames: []string{"time"}, + log: log, + } + + if len(config.TimeColumnNames) > 0 { + queryEndpoint.timeColumnNames = config.TimeColumnNames + } + + if len(config.MetricColumnTypes) > 0 { + queryEndpoint.metricColumnTypes = config.MetricColumnTypes + } + engineCache.Lock() defer engineCache.Unlock() - if engine, present := engineCache.cache[dsInfo.Id]; present { - if version, _ := engineCache.versions[dsInfo.Id]; version == dsInfo.Version { - e.XormEngine = engine - return nil + if engine, present := engineCache.cache[config.Datasource.Id]; present { + if version := engineCache.versions[config.Datasource.Id]; version == config.Datasource.Version { + queryEndpoint.engine = engine + return &queryEndpoint, nil } } - engine, err := xorm.NewEngine(driverName, cnnstr) + engine, err := NewXormEngine(config.DriverName, config.ConnectionString) if err != nil { - return err + return nil, err } - engine.SetMaxOpenConns(10) - engine.SetMaxIdleConns(10) + maxOpenConns := config.Datasource.JsonData.Get("maxOpenConns").MustInt(0) + engine.SetMaxOpenConns(maxOpenConns) + maxIdleConns := config.Datasource.JsonData.Get("maxIdleConns").MustInt(2) + engine.SetMaxIdleConns(maxIdleConns) + connMaxLifetime := config.Datasource.JsonData.Get("connMaxLifetime").MustInt(14400) + engine.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) - engineCache.cache[dsInfo.Id] = engine - e.XormEngine = engine + engineCache.versions[config.Datasource.Id] = config.Datasource.Version + engineCache.cache[config.Datasource.Id] = engine + queryEndpoint.engine = engine - return nil + return &queryEndpoint, nil } -// Query is a default implementation of the Query method for an SQL data source. -// The caller of this function must implement transformToTimeSeries and transformToTable and -// pass them in as parameters. -func (e *DefaultSqlEngine) Query( - ctx context.Context, - dsInfo *models.DataSource, - tsdbQuery *TsdbQuery, - transformToTimeSeries func(query *Query, rows *core.Rows, result *QueryResult, tsdbQuery *TsdbQuery) error, - transformToTable func(query *Query, rows *core.Rows, result *QueryResult, tsdbQuery *TsdbQuery) error, -) (*Response, error) { +const rowLimit = 1000000 + +// Query is the main function for the SqlQueryEndpoint +func (e *sqlQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *TsdbQuery) (*Response, error) { result := &Response{ Results: make(map[string]*QueryResult), } - session := e.XormEngine.NewSession() - defer session.Close() - db := session.DB() + var wg sync.WaitGroup for _, query := range tsdbQuery.Queries { - rawSql := query.Model.Get("rawSql").MustString() - if rawSql == "" { + rawSQL := query.Model.Get("rawSql").MustString() + if rawSQL == "" { continue } queryResult := &QueryResult{Meta: simplejson.New(), RefId: query.RefId} result.Results[query.RefId] = queryResult - rawSql, err := e.MacroEngine.Interpolate(query, tsdbQuery.TimeRange, rawSql) + // global substitutions + rawSQL, err := Interpolate(query, tsdbQuery.TimeRange, rawSQL) if err != nil { queryResult.Error = err continue } - queryResult.Meta.Set("sql", rawSql) - - rows, err := db.Query(rawSql) + // datasource specific substitutions + rawSQL, err = e.macroEngine.Interpolate(query, tsdbQuery.TimeRange, rawSQL) if err != nil { queryResult.Error = err continue } - defer rows.Close() + queryResult.Meta.Set("sql", rawSQL) - format := query.Model.Get("format").MustString("time_series") + wg.Add(1) - switch format { - case "time_series": - err := transformToTimeSeries(query, rows, queryResult, tsdbQuery) + go func(rawSQL string, query *Query, queryResult *QueryResult) { + defer wg.Done() + session := e.engine.NewSession() + defer session.Close() + db := session.DB() + + rows, err := db.Query(rawSQL) if err != nil { queryResult.Error = err - continue + return } - case "table": - err := transformToTable(query, rows, queryResult, tsdbQuery) - if err != nil { - queryResult.Error = err - continue + + defer rows.Close() + + format := query.Model.Get("format").MustString("time_series") + + switch format { + case "time_series": + err := e.transformToTimeSeries(query, rows, queryResult, tsdbQuery) + if err != nil { + queryResult.Error = err + return + } + case "table": + err := e.transformToTable(query, rows, queryResult, tsdbQuery) + if err != nil { + queryResult.Error = err + return + } + } + }(rawSQL, query, queryResult) + } + wg.Wait() + + return result, nil +} + +// global macros/substitutions for all sql datasources +var Interpolate = func(query *Query, timeRange *TimeRange, sql string) (string, error) { + minInterval, err := GetIntervalFrom(query.DataSource, query.Model, time.Second*60) + if err != nil { + return sql, nil + } + interval := sqlIntervalCalculator.Calculate(timeRange, minInterval) + + sql = strings.Replace(sql, "$__interval_ms", strconv.FormatInt(interval.Milliseconds(), 10), -1) + sql = strings.Replace(sql, "$__interval", interval.Text, -1) + sql = strings.Replace(sql, "$__timeFrom()", fmt.Sprintf("'%s'", timeRange.GetFromAsTimeUTC().Format(time.RFC3339)), -1) + sql = strings.Replace(sql, "$__timeTo()", fmt.Sprintf("'%s'", timeRange.GetToAsTimeUTC().Format(time.RFC3339)), -1) + sql = strings.Replace(sql, "$__unixEpochFrom()", fmt.Sprintf("%d", timeRange.GetFromAsSecondsEpoch()), -1) + sql = strings.Replace(sql, "$__unixEpochTo()", fmt.Sprintf("%d", timeRange.GetToAsSecondsEpoch()), -1) + + return sql, nil +} + +func (e *sqlQueryEndpoint) transformToTable(query *Query, rows *core.Rows, result *QueryResult, tsdbQuery *TsdbQuery) error { + columnNames, err := rows.Columns() + columnCount := len(columnNames) + + if err != nil { + return err + } + + rowCount := 0 + timeIndex := -1 + + table := &Table{ + Columns: make([]TableColumn, columnCount), + Rows: make([]RowValues, 0), + } + + for i, name := range columnNames { + table.Columns[i].Text = name + + for _, tc := range e.timeColumnNames { + if name == tc { + timeIndex = i + break } } } - return result, nil + columnTypes, err := rows.ColumnTypes() + if err != nil { + return err + } + + for ; rows.Next(); rowCount++ { + if rowCount > rowLimit { + return fmt.Errorf("query row limit exceeded, limit %d", rowLimit) + } + + values, err := e.rowTransformer.Transform(columnTypes, rows) + if err != nil { + return err + } + + // converts column named time to unix timestamp in milliseconds + // to make native mssql datetime types and epoch dates work in + // annotation and table queries. + ConvertSqlTimeColumnToEpochMs(values, timeIndex) + table.Rows = append(table.Rows, values) + } + + result.Tables = append(result.Tables, table) + result.Meta.Set("rowCount", rowCount) + return nil +} + +func (e *sqlQueryEndpoint) transformToTimeSeries(query *Query, rows *core.Rows, result *QueryResult, tsdbQuery *TsdbQuery) error { + pointsBySeries := make(map[string]*TimeSeries) + seriesByQueryOrder := list.New() + + columnNames, err := rows.Columns() + if err != nil { + return err + } + + columnTypes, err := rows.ColumnTypes() + if err != nil { + return err + } + + rowCount := 0 + timeIndex := -1 + metricIndex := -1 + metricPrefix := false + var metricPrefixValue string + + // check columns of resultset: a column named time is mandatory + // the first text column is treated as metric name unless a column named metric is present + for i, col := range columnNames { + for _, tc := range e.timeColumnNames { + if col == tc { + timeIndex = i + continue + } + } + switch col { + case "metric": + metricIndex = i + default: + if metricIndex == -1 { + columnType := columnTypes[i].DatabaseTypeName() + + for _, mct := range e.metricColumnTypes { + if columnType == mct { + metricIndex = i + continue + } + } + } + } + } + + // use metric column as prefix with multiple value columns + if metricIndex != -1 && len(columnNames) > 3 { + metricPrefix = true + } + + if timeIndex == -1 { + return fmt.Errorf("Found no column named %s", strings.Join(e.timeColumnNames, " or ")) + } + + fillMissing := query.Model.Get("fill").MustBool(false) + var fillInterval float64 + fillValue := null.Float{} + fillPrevious := false + + if fillMissing { + fillInterval = query.Model.Get("fillInterval").MustFloat64() * 1000 + switch query.Model.Get("fillMode").MustString() { + case "null": + case "previous": + fillPrevious = true + case "value": + fillValue.Float64 = query.Model.Get("fillValue").MustFloat64() + fillValue.Valid = true + } + } + + for rows.Next() { + var timestamp float64 + var value null.Float + var metric string + + if rowCount > rowLimit { + return fmt.Errorf("query row limit exceeded, limit %d", rowLimit) + } + + values, err := e.rowTransformer.Transform(columnTypes, rows) + if err != nil { + return err + } + + // converts column named time to unix timestamp in milliseconds to make + // native mysql datetime types and epoch dates work in + // annotation and table queries. + ConvertSqlTimeColumnToEpochMs(values, timeIndex) + + switch columnValue := values[timeIndex].(type) { + case int64: + timestamp = float64(columnValue) + case float64: + timestamp = columnValue + default: + return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp, got: %T %v", columnValue, columnValue) + } + + if metricIndex >= 0 { + if columnValue, ok := values[metricIndex].(string); ok { + if metricPrefix { + metricPrefixValue = columnValue + } else { + metric = columnValue + } + } else { + return fmt.Errorf("Column metric must be of type %s. metric column name: %s type: %s but datatype is %T", strings.Join(e.metricColumnTypes, ", "), columnNames[metricIndex], columnTypes[metricIndex].DatabaseTypeName(), values[metricIndex]) + } + } + + for i, col := range columnNames { + if i == timeIndex || i == metricIndex { + continue + } + + if value, err = ConvertSqlValueColumnToFloat(col, values[i]); err != nil { + return err + } + + if metricIndex == -1 { + metric = col + } else if metricPrefix { + metric = metricPrefixValue + " " + col + } + + series, exist := pointsBySeries[metric] + if !exist { + series = &TimeSeries{Name: metric} + pointsBySeries[metric] = series + seriesByQueryOrder.PushBack(metric) + } + + if fillMissing { + var intervalStart float64 + if !exist { + intervalStart = float64(tsdbQuery.TimeRange.MustGetFrom().UnixNano() / 1e6) + } else { + intervalStart = series.Points[len(series.Points)-1][1].Float64 + fillInterval + } + + if fillPrevious { + if len(series.Points) > 0 { + fillValue = series.Points[len(series.Points)-1][0] + } else { + fillValue.Valid = false + } + } + + // align interval start + intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval + + for i := intervalStart; i < timestamp; i += fillInterval { + series.Points = append(series.Points, TimePoint{fillValue, null.FloatFrom(i)}) + rowCount++ + } + } + + series.Points = append(series.Points, TimePoint{value, null.FloatFrom(timestamp)}) + + e.log.Debug("Rows", "metric", metric, "time", timestamp, "value", value) + } + } + + for elem := seriesByQueryOrder.Front(); elem != nil; elem = elem.Next() { + key := elem.Value.(string) + result.Series = append(result.Series, pointsBySeries[key]) + + if fillMissing { + series := pointsBySeries[key] + // fill in values from last fetched value till interval end + intervalStart := series.Points[len(series.Points)-1][1].Float64 + intervalEnd := float64(tsdbQuery.TimeRange.MustGetTo().UnixNano() / 1e6) + + if fillPrevious { + if len(series.Points) > 0 { + fillValue = series.Points[len(series.Points)-1][0] + } else { + fillValue.Valid = false + } + } + + // align interval start + intervalStart = math.Floor(intervalStart/fillInterval) * fillInterval + for i := intervalStart + fillInterval; i < intervalEnd; i += fillInterval { + series.Points = append(series.Points, TimePoint{fillValue, null.FloatFrom(i)}) + rowCount++ + } + } + } + + result.Meta.Set("rowCount", rowCount) + return nil +} + +// ConvertSqlTimeColumnToEpochMs converts column named time to unix timestamp in milliseconds +// to make native datetime types and epoch dates work in annotation and table queries. +func ConvertSqlTimeColumnToEpochMs(values RowValues, timeIndex int) { + if timeIndex >= 0 { + switch value := values[timeIndex].(type) { + case time.Time: + values[timeIndex] = float64(value.UnixNano()) / float64(time.Millisecond) + case *time.Time: + if value != nil { + values[timeIndex] = float64((*value).UnixNano()) / float64(time.Millisecond) + } + case int64: + values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) + case *int64: + if value != nil { + values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) + } + case uint64: + values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) + case *uint64: + if value != nil { + values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) + } + case int32: + values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) + case *int32: + if value != nil { + values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) + } + case uint32: + values[timeIndex] = int64(EpochPrecisionToMs(float64(value))) + case *uint32: + if value != nil { + values[timeIndex] = int64(EpochPrecisionToMs(float64(*value))) + } + case float64: + values[timeIndex] = EpochPrecisionToMs(value) + case *float64: + if value != nil { + values[timeIndex] = EpochPrecisionToMs(*value) + } + case float32: + values[timeIndex] = EpochPrecisionToMs(float64(value)) + case *float32: + if value != nil { + values[timeIndex] = EpochPrecisionToMs(float64(*value)) + } + } + } +} + +// ConvertSqlValueColumnToFloat converts timeseries value column to float. +func ConvertSqlValueColumnToFloat(columnName string, columnValue interface{}) (null.Float, error) { + var value null.Float + + switch typedValue := columnValue.(type) { + case int: + value = null.FloatFrom(float64(typedValue)) + case *int: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case int64: + value = null.FloatFrom(float64(typedValue)) + case *int64: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case int32: + value = null.FloatFrom(float64(typedValue)) + case *int32: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case int16: + value = null.FloatFrom(float64(typedValue)) + case *int16: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case int8: + value = null.FloatFrom(float64(typedValue)) + case *int8: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case uint: + value = null.FloatFrom(float64(typedValue)) + case *uint: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case uint64: + value = null.FloatFrom(float64(typedValue)) + case *uint64: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case uint32: + value = null.FloatFrom(float64(typedValue)) + case *uint32: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case uint16: + value = null.FloatFrom(float64(typedValue)) + case *uint16: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case uint8: + value = null.FloatFrom(float64(typedValue)) + case *uint8: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case float64: + value = null.FloatFrom(typedValue) + case *float64: + value = null.FloatFromPtr(typedValue) + case float32: + value = null.FloatFrom(float64(typedValue)) + case *float32: + if typedValue == nil { + value.Valid = false + } else { + value = null.FloatFrom(float64(*typedValue)) + } + case nil: + value.Valid = false + default: + return null.NewFloat(0, false), fmt.Errorf("Value column must have numeric datatype, column: %s type: %T value: %v", columnName, typedValue, typedValue) + } + + return value, nil +} + +func SetupFillmode(query *Query, interval time.Duration, fillmode string) error { + query.Model.Set("fill", true) + query.Model.Set("fillInterval", interval.Seconds()) + switch fillmode { + case "NULL": + query.Model.Set("fillMode", "null") + case "previous": + query.Model.Set("fillMode", "previous") + default: + query.Model.Set("fillMode", "value") + floatVal, err := strconv.ParseFloat(fillmode, 64) + if err != nil { + return fmt.Errorf("error parsing fill value %v", fillmode) + } + query.Model.Set("fillValue", floatVal) + } + + return nil +} + +type SqlMacroEngineBase struct{} + +func NewSqlMacroEngineBase() *SqlMacroEngineBase { + return &SqlMacroEngineBase{} +} + +func (m *SqlMacroEngineBase) ReplaceAllStringSubmatchFunc(re *regexp.Regexp, str string, repl func([]string) string) string { + result := "" + lastIndex := 0 + + for _, v := range re.FindAllSubmatchIndex([]byte(str), -1) { + groups := []string{} + for i := 0; i < len(v); i += 2 { + groups = append(groups, str[v[i]:v[i+1]]) + } + + result += str[lastIndex:v[0]] + repl(groups) + lastIndex = v[1] + } + + return result + str[lastIndex:] } diff --git a/pkg/tsdb/sql_engine_test.go b/pkg/tsdb/sql_engine_test.go new file mode 100644 index 00000000000..bfcc82aac47 --- /dev/null +++ b/pkg/tsdb/sql_engine_test.go @@ -0,0 +1,349 @@ +package tsdb + +import ( + "fmt" + "testing" + "time" + + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestSqlEngine(t *testing.T) { + Convey("SqlEngine", t, func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, int(527345*time.Microsecond), time.UTC) + earlyDt := time.Date(1970, 3, 14, 21, 20, 6, int(527345*time.Microsecond), time.UTC) + + Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { + from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) + to := from.Add(5 * time.Minute) + timeRange := NewFakeTimeRange("5m", "now", to) + query := &Query{DataSource: &models.DataSource{}, Model: simplejson.New()} + + Convey("interpolate $__interval", func() { + sql, err := Interpolate(query, timeRange, "select $__interval ") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select 1m ") + }) + + Convey("interpolate $__interval in $__timeGroup", func() { + sql, err := Interpolate(query, timeRange, "select $__timeGroupAlias(time,$__interval)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select $__timeGroupAlias(time,1m)") + }) + + Convey("interpolate $__interval_ms", func() { + sql, err := Interpolate(query, timeRange, "select $__interval_ms ") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select 60000 ") + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := Interpolate(query, timeRange, "select $__timeFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select '%s'", from.Format(time.RFC3339))) + }) + + Convey("interpolate __timeTo function", func() { + sql, err := Interpolate(query, timeRange, "select $__timeTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select '%s'", to.Format(time.RFC3339))) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) + + }) + + Convey("Given row values with time.Time as time columns", func() { + var nilPointer *time.Time + + fixtures := make([]interface{}, 5) + fixtures[0] = dt + fixtures[1] = &dt + fixtures[2] = earlyDt + fixtures[3] = &earlyDt + fixtures[4] = nilPointer + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + expected := float64(dt.UnixNano()) / float64(time.Millisecond) + expectedEarly := float64(earlyDt.UnixNano()) / float64(time.Millisecond) + + So(fixtures[0].(float64), ShouldEqual, expected) + So(fixtures[1].(float64), ShouldEqual, expected) + So(fixtures[2].(float64), ShouldEqual, expectedEarly) + So(fixtures[3].(float64), ShouldEqual, expectedEarly) + So(fixtures[4], ShouldBeNil) + }) + }) + + Convey("Given row values with int64 as time columns", func() { + tSeconds := dt.Unix() + tMilliseconds := dt.UnixNano() / 1e6 + tNanoSeconds := dt.UnixNano() + var nilPointer *int64 + + fixtures := make([]interface{}, 7) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = tMilliseconds + fixtures[3] = &tMilliseconds + fixtures[4] = tNanoSeconds + fixtures[5] = &tNanoSeconds + fixtures[6] = nilPointer + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[1].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[2].(int64), ShouldEqual, tMilliseconds) + So(fixtures[3].(int64), ShouldEqual, tMilliseconds) + So(fixtures[4].(int64), ShouldEqual, tMilliseconds) + So(fixtures[5].(int64), ShouldEqual, tMilliseconds) + So(fixtures[6], ShouldBeNil) + }) + }) + + Convey("Given row values with uin64 as time columns", func() { + tSeconds := uint64(dt.Unix()) + tMilliseconds := uint64(dt.UnixNano() / 1e6) + tNanoSeconds := uint64(dt.UnixNano()) + var nilPointer *uint64 + + fixtures := make([]interface{}, 7) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = tMilliseconds + fixtures[3] = &tMilliseconds + fixtures[4] = tNanoSeconds + fixtures[5] = &tNanoSeconds + fixtures[6] = nilPointer + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[1].(int64), ShouldEqual, tSeconds*1e3) + So(fixtures[2].(int64), ShouldEqual, tMilliseconds) + So(fixtures[3].(int64), ShouldEqual, tMilliseconds) + So(fixtures[4].(int64), ShouldEqual, tMilliseconds) + So(fixtures[5].(int64), ShouldEqual, tMilliseconds) + So(fixtures[6], ShouldBeNil) + }) + }) + + Convey("Given row values with int32 as time columns", func() { + tSeconds := int32(dt.Unix()) + var nilInt *int32 + + fixtures := make([]interface{}, 3) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = nilInt + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[1].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[2], ShouldBeNil) + }) + }) + + Convey("Given row values with uint32 as time columns", func() { + tSeconds := uint32(dt.Unix()) + var nilInt *uint32 + + fixtures := make([]interface{}, 3) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = nilInt + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[1].(int64), ShouldEqual, dt.Unix()*1e3) + So(fixtures[2], ShouldBeNil) + }) + }) + + Convey("Given row values with float64 as time columns", func() { + tSeconds := float64(dt.UnixNano()) / float64(time.Second) + tMilliseconds := float64(dt.UnixNano()) / float64(time.Millisecond) + tNanoSeconds := float64(dt.UnixNano()) + var nilPointer *float64 + + fixtures := make([]interface{}, 7) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = tMilliseconds + fixtures[3] = &tMilliseconds + fixtures[4] = tNanoSeconds + fixtures[5] = &tNanoSeconds + fixtures[6] = nilPointer + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(float64), ShouldEqual, tMilliseconds) + So(fixtures[1].(float64), ShouldEqual, tMilliseconds) + So(fixtures[2].(float64), ShouldEqual, tMilliseconds) + So(fixtures[3].(float64), ShouldEqual, tMilliseconds) + So(fixtures[4].(float64), ShouldEqual, tMilliseconds) + So(fixtures[5].(float64), ShouldEqual, tMilliseconds) + So(fixtures[6], ShouldBeNil) + }) + }) + + Convey("Given row values with float32 as time columns", func() { + tSeconds := float32(dt.Unix()) + var nilInt *float32 + + fixtures := make([]interface{}, 3) + fixtures[0] = tSeconds + fixtures[1] = &tSeconds + fixtures[2] = nilInt + + for i := range fixtures { + ConvertSqlTimeColumnToEpochMs(fixtures, i) + } + + Convey("When converting them should return epoch time with millisecond precision ", func() { + So(fixtures[0].(float64), ShouldEqual, float32(dt.Unix()*1e3)) + So(fixtures[1].(float64), ShouldEqual, float32(dt.Unix()*1e3)) + So(fixtures[2], ShouldBeNil) + }) + }) + + Convey("Given row with value columns", func() { + intValue := 1 + int64Value := int64(1) + int32Value := int32(1) + int16Value := int16(1) + int8Value := int8(1) + float64Value := float64(1) + float32Value := float32(1) + uintValue := uint(1) + uint64Value := uint64(1) + uint32Value := uint32(1) + uint16Value := uint16(1) + uint8Value := uint8(1) + + fixtures := make([]interface{}, 24) + fixtures[0] = intValue + fixtures[1] = &intValue + fixtures[2] = int64Value + fixtures[3] = &int64Value + fixtures[4] = int32Value + fixtures[5] = &int32Value + fixtures[6] = int16Value + fixtures[7] = &int16Value + fixtures[8] = int8Value + fixtures[9] = &int8Value + fixtures[10] = float64Value + fixtures[11] = &float64Value + fixtures[12] = float32Value + fixtures[13] = &float32Value + fixtures[14] = uintValue + fixtures[15] = &uintValue + fixtures[16] = uint64Value + fixtures[17] = &uint64Value + fixtures[18] = uint32Value + fixtures[19] = &uint32Value + fixtures[20] = uint16Value + fixtures[21] = &uint16Value + fixtures[22] = uint8Value + fixtures[23] = &uint8Value + + var intNilPointer *int + var int64NilPointer *int64 + var int32NilPointer *int32 + var int16NilPointer *int16 + var int8NilPointer *int8 + var float64NilPointer *float64 + var float32NilPointer *float32 + var uintNilPointer *uint + var uint64NilPointer *uint64 + var uint32NilPointer *uint32 + var uint16NilPointer *uint16 + var uint8NilPointer *uint8 + + nilPointerFixtures := make([]interface{}, 12) + nilPointerFixtures[0] = intNilPointer + nilPointerFixtures[1] = int64NilPointer + nilPointerFixtures[2] = int32NilPointer + nilPointerFixtures[3] = int16NilPointer + nilPointerFixtures[4] = int8NilPointer + nilPointerFixtures[5] = float64NilPointer + nilPointerFixtures[6] = float32NilPointer + nilPointerFixtures[7] = uintNilPointer + nilPointerFixtures[8] = uint64NilPointer + nilPointerFixtures[9] = uint32NilPointer + nilPointerFixtures[10] = uint16NilPointer + nilPointerFixtures[11] = uint8NilPointer + + Convey("When converting values to float should return expected value", func() { + for _, f := range fixtures { + value, _ := ConvertSqlValueColumnToFloat("col", f) + + if !value.Valid { + t.Fatalf("Failed to convert %T value, expected a valid float value", f) + } + + if value.Float64 != null.FloatFrom(1).Float64 { + t.Fatalf("Failed to convert %T value, expected a float value of 1.000, but got %v", f, value) + } + } + }) + + Convey("When converting nil pointer values to float should return expected value", func() { + for _, f := range nilPointerFixtures { + value, err := ConvertSqlValueColumnToFloat("col", f) + + if err != nil { + t.Fatalf("Failed to convert %T value, expected a non nil error, but got %v", f, err) + } + + if value.Valid { + t.Fatalf("Failed to convert %T value, expected an invalid float value", f) + } + } + }) + }) + }) +} diff --git a/pkg/tsdb/stackdriver/annotation_query.go b/pkg/tsdb/stackdriver/annotation_query.go new file mode 100644 index 00000000000..db35171ad70 --- /dev/null +++ b/pkg/tsdb/stackdriver/annotation_query.go @@ -0,0 +1,120 @@ +package stackdriver + +import ( + "context" + "strconv" + "strings" + "time" + + "github.com/grafana/grafana/pkg/tsdb" +) + +func (e *StackdriverExecutor) executeAnnotationQuery(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + + firstQuery := tsdbQuery.Queries[0] + + queries, err := e.buildQueries(tsdbQuery) + if err != nil { + return nil, err + } + + queryRes, resp, err := e.executeQuery(ctx, queries[0], tsdbQuery) + if err != nil { + return nil, err + } + title := firstQuery.Model.Get("title").MustString() + text := firstQuery.Model.Get("text").MustString() + tags := firstQuery.Model.Get("tags").MustString() + err = e.parseToAnnotations(queryRes, resp, queries[0], title, text, tags) + result.Results[firstQuery.RefId] = queryRes + + return result, err +} + +func (e *StackdriverExecutor) parseToAnnotations(queryRes *tsdb.QueryResult, data StackdriverResponse, query *StackdriverQuery, title string, text string, tags string) error { + annotations := make([]map[string]string, 0) + + for _, series := range data.TimeSeries { + // reverse the order to be ascending + for i := len(series.Points) - 1; i >= 0; i-- { + point := series.Points[i] + value := strconv.FormatFloat(point.Value.DoubleValue, 'f', 6, 64) + if series.ValueType == "STRING" { + value = point.Value.StringValue + } + annotation := make(map[string]string) + annotation["time"] = point.Interval.EndTime.UTC().Format(time.RFC3339) + annotation["title"] = formatAnnotationText(title, value, series.Metric.Type, series.Metric.Labels, series.Resource.Labels) + annotation["tags"] = tags + annotation["text"] = formatAnnotationText(text, value, series.Metric.Type, series.Metric.Labels, series.Resource.Labels) + annotations = append(annotations, annotation) + } + } + + transformAnnotationToTable(annotations, queryRes) + return nil +} + +func transformAnnotationToTable(data []map[string]string, result *tsdb.QueryResult) { + table := &tsdb.Table{ + Columns: make([]tsdb.TableColumn, 4), + Rows: make([]tsdb.RowValues, 0), + } + table.Columns[0].Text = "time" + table.Columns[1].Text = "title" + table.Columns[2].Text = "tags" + table.Columns[3].Text = "text" + + for _, r := range data { + values := make([]interface{}, 4) + values[0] = r["time"] + values[1] = r["title"] + values[2] = r["tags"] + values[3] = r["text"] + table.Rows = append(table.Rows, values) + } + result.Tables = append(result.Tables, table) + result.Meta.Set("rowCount", len(data)) + slog.Info("anno", "len", len(data)) +} + +func formatAnnotationText(annotationText string, pointValue string, metricType string, metricLabels map[string]string, resourceLabels map[string]string) string { + result := legendKeyFormat.ReplaceAllFunc([]byte(annotationText), func(in []byte) []byte { + metaPartName := strings.Replace(string(in), "{{", "", 1) + metaPartName = strings.Replace(metaPartName, "}}", "", 1) + metaPartName = strings.TrimSpace(metaPartName) + + if metaPartName == "metric.type" { + return []byte(metricType) + } + + metricPart := replaceWithMetricPart(metaPartName, metricType) + + if metricPart != nil { + return metricPart + } + + if metaPartName == "metric.value" { + return []byte(pointValue) + } + + metaPartName = strings.Replace(metaPartName, "metric.label.", "", 1) + + if val, exists := metricLabels[metaPartName]; exists { + return []byte(val) + } + + metaPartName = strings.Replace(metaPartName, "resource.label.", "", 1) + + if val, exists := resourceLabels[metaPartName]; exists { + return []byte(val) + } + + return in + }) + + return string(result) +} diff --git a/pkg/tsdb/stackdriver/annotation_query_test.go b/pkg/tsdb/stackdriver/annotation_query_test.go new file mode 100644 index 00000000000..8229470d665 --- /dev/null +++ b/pkg/tsdb/stackdriver/annotation_query_test.go @@ -0,0 +1,33 @@ +package stackdriver + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestStackdriverAnnotationQuery(t *testing.T) { + Convey("Stackdriver Annotation Query Executor", t, func() { + executor := &StackdriverExecutor{} + Convey("When parsing the stackdriver api response", func() { + data, err := loadTestFile("./test-data/2-series-response-no-agg.json") + So(err, ShouldBeNil) + So(len(data.TimeSeries), ShouldEqual, 3) + + res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "annotationQuery"} + query := &StackdriverQuery{} + err = executor.parseToAnnotations(res, data, query, "atitle {{metric.label.instance_name}} {{metric.value}}", "atext {{resource.label.zone}}", "atag") + So(err, ShouldBeNil) + + Convey("Should return annotations table", func() { + So(len(res.Tables), ShouldEqual, 1) + So(len(res.Tables[0].Rows), ShouldEqual, 9) + So(res.Tables[0].Rows[0][1], ShouldEqual, "atitle collector-asia-east-1 9.856650") + So(res.Tables[0].Rows[0][3], ShouldEqual, "atext asia-east1-a") + }) + }) + }) +} diff --git a/pkg/tsdb/stackdriver/ensure_default_project.go b/pkg/tsdb/stackdriver/ensure_default_project.go new file mode 100644 index 00000000000..265fdda5151 --- /dev/null +++ b/pkg/tsdb/stackdriver/ensure_default_project.go @@ -0,0 +1,24 @@ +package stackdriver + +import ( + "context" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" +) + +func (e *StackdriverExecutor) ensureDefaultProject(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: tsdbQuery.Queries[0].RefId} + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + defaultProject, err := e.getDefaultProject(ctx) + if err != nil { + return nil, err + } + + e.dsInfo.JsonData.Set("defaultProject", defaultProject) + queryResult.Meta.Set("defaultProject", defaultProject) + result.Results[tsdbQuery.Queries[0].RefId] = queryResult + return result, nil +} diff --git a/pkg/tsdb/stackdriver/stackdriver.go b/pkg/tsdb/stackdriver/stackdriver.go new file mode 100644 index 00000000000..2ad47cc4b83 --- /dev/null +++ b/pkg/tsdb/stackdriver/stackdriver.go @@ -0,0 +1,612 @@ +package stackdriver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "math" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "time" + + "golang.org/x/net/context/ctxhttp" + "golang.org/x/oauth2/google" + + "github.com/grafana/grafana/pkg/api/pluginproxy" + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb" + "github.com/opentracing/opentracing-go" +) + +var ( + slog log.Logger + legendKeyFormat *regexp.Regexp + metricNameFormat *regexp.Regexp +) + +const ( + gceAuthentication string = "gce" + jwtAuthentication string = "jwt" +) + +// StackdriverExecutor executes queries for the Stackdriver datasource +type StackdriverExecutor struct { + httpClient *http.Client + dsInfo *models.DataSource +} + +// NewStackdriverExecutor initializes a http client +func NewStackdriverExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + httpClient, err := dsInfo.GetHttpClient() + if err != nil { + return nil, err + } + + return &StackdriverExecutor{ + httpClient: httpClient, + dsInfo: dsInfo, + }, nil +} + +func init() { + slog = log.New("tsdb.stackdriver") + tsdb.RegisterTsdbQueryEndpoint("stackdriver", NewStackdriverExecutor) + legendKeyFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) + metricNameFormat = regexp.MustCompile(`([\w\d_]+)\.googleapis\.com/(.+)`) +} + +// Query takes in the frontend queries, parses them into the Stackdriver query format +// executes the queries against the Stackdriver API and parses the response into +// the time series or table format +func (e *StackdriverExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + var result *tsdb.Response + var err error + queryType := tsdbQuery.Queries[0].Model.Get("type").MustString("") + + switch queryType { + case "annotationQuery": + result, err = e.executeAnnotationQuery(ctx, tsdbQuery) + case "ensureDefaultProjectQuery": + result, err = e.ensureDefaultProject(ctx, tsdbQuery) + case "timeSeriesQuery": + fallthrough + default: + result, err = e.executeTimeSeriesQuery(ctx, tsdbQuery) + } + + return result, err +} + +func (e *StackdriverExecutor) executeTimeSeriesQuery(ctx context.Context, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + + authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString(jwtAuthentication) + if authenticationType == gceAuthentication { + defaultProject, err := e.getDefaultProject(ctx) + if err != nil { + return nil, fmt.Errorf("Failed to retrieve default project from GCE metadata server. error: %v", err) + } + + e.dsInfo.JsonData.Set("defaultProject", defaultProject) + } + + queries, err := e.buildQueries(tsdbQuery) + if err != nil { + return nil, err + } + + for _, query := range queries { + queryRes, resp, err := e.executeQuery(ctx, query, tsdbQuery) + if err != nil { + return nil, err + } + err = e.parseResponse(queryRes, resp, query) + if err != nil { + queryRes.Error = err + } + result.Results[query.RefID] = queryRes + } + + return result, nil +} + +func (e *StackdriverExecutor) buildQueries(tsdbQuery *tsdb.TsdbQuery) ([]*StackdriverQuery, error) { + stackdriverQueries := []*StackdriverQuery{} + + startTime, err := tsdbQuery.TimeRange.ParseFrom() + if err != nil { + return nil, err + } + + endTime, err := tsdbQuery.TimeRange.ParseTo() + if err != nil { + return nil, err + } + + durationSeconds := int(endTime.Sub(startTime).Seconds()) + + for _, query := range tsdbQuery.Queries { + var target string + + metricType := query.Model.Get("metricType").MustString() + filterParts := query.Model.Get("filters").MustArray() + + params := url.Values{} + params.Add("interval.startTime", startTime.UTC().Format(time.RFC3339)) + params.Add("interval.endTime", endTime.UTC().Format(time.RFC3339)) + params.Add("filter", buildFilterString(metricType, filterParts)) + params.Add("view", query.Model.Get("view").MustString("FULL")) + setAggParams(¶ms, query, durationSeconds) + + target = params.Encode() + + if setting.Env == setting.DEV { + slog.Debug("Stackdriver request", "params", params) + } + + groupBys := query.Model.Get("groupBys").MustArray() + groupBysAsStrings := make([]string, 0) + for _, groupBy := range groupBys { + groupBysAsStrings = append(groupBysAsStrings, groupBy.(string)) + } + + aliasBy := query.Model.Get("aliasBy").MustString() + + stackdriverQueries = append(stackdriverQueries, &StackdriverQuery{ + Target: target, + Params: params, + RefID: query.RefId, + GroupBys: groupBysAsStrings, + AliasBy: aliasBy, + }) + } + + return stackdriverQueries, nil +} + +func reverse(s string) string { + chars := []rune(s) + for i, j := 0, len(chars)-1; i < j; i, j = i+1, j-1 { + chars[i], chars[j] = chars[j], chars[i] + } + return string(chars) +} + +func interpolateFilterWildcards(value string) string { + matches := strings.Count(value, "*") + if matches == 2 && strings.HasSuffix(value, "*") && strings.HasPrefix(value, "*") { + value = strings.Replace(value, "*", "", -1) + value = fmt.Sprintf(`has_substring("%s")`, value) + } else if matches == 1 && strings.HasPrefix(value, "*") { + value = strings.Replace(value, "*", "", 1) + value = fmt.Sprintf(`ends_with("%s")`, value) + } else if matches == 1 && strings.HasSuffix(value, "*") { + value = reverse(strings.Replace(reverse(value), "*", "", 1)) + value = fmt.Sprintf(`starts_with("%s")`, value) + } else if matches != 0 { + re := regexp.MustCompile(`[-\/^$+?.()|[\]{}]`) + value = string(re.ReplaceAllFunc([]byte(value), func(in []byte) []byte { + return []byte(strings.Replace(string(in), string(in), `\\`+string(in), 1)) + })) + value = strings.Replace(value, "*", ".*", -1) + value = strings.Replace(value, `"`, `\\"`, -1) + value = fmt.Sprintf(`monitoring.regex.full_match("^%s$")`, value) + } + + return value +} + +func buildFilterString(metricType string, filterParts []interface{}) string { + filterString := "" + for i, part := range filterParts { + mod := i % 4 + if part == "AND" { + filterString += " " + } else if mod == 2 { + operator := filterParts[i-1] + if operator == "=~" || operator == "!=~" { + filterString = reverse(strings.Replace(reverse(filterString), "~", "", 1)) + filterString += fmt.Sprintf(`monitoring.regex.full_match("%s")`, part) + } else if strings.Contains(part.(string), "*") { + filterString += interpolateFilterWildcards(part.(string)) + } else { + filterString += fmt.Sprintf(`"%s"`, part) + } + } else { + filterString += part.(string) + } + } + return strings.Trim(fmt.Sprintf(`metric.type="%s" %s`, metricType, filterString), " ") +} + +func setAggParams(params *url.Values, query *tsdb.Query, durationSeconds int) { + primaryAggregation := query.Model.Get("primaryAggregation").MustString() + perSeriesAligner := query.Model.Get("perSeriesAligner").MustString() + alignmentPeriod := query.Model.Get("alignmentPeriod").MustString() + + if primaryAggregation == "" { + primaryAggregation = "REDUCE_NONE" + } + + if perSeriesAligner == "" { + perSeriesAligner = "ALIGN_MEAN" + } + + if alignmentPeriod == "grafana-auto" || alignmentPeriod == "" { + alignmentPeriodValue := int(math.Max(float64(query.IntervalMs)/1000, 60.0)) + alignmentPeriod = "+" + strconv.Itoa(alignmentPeriodValue) + "s" + } + + if alignmentPeriod == "stackdriver-auto" { + alignmentPeriodValue := int(math.Max(float64(durationSeconds), 60.0)) + if alignmentPeriodValue < 60*60*23 { + alignmentPeriod = "+60s" + } else if alignmentPeriodValue < 60*60*24*6 { + alignmentPeriod = "+300s" + } else { + alignmentPeriod = "+3600s" + } + } + + re := regexp.MustCompile("[0-9]+") + seconds, err := strconv.ParseInt(re.FindString(alignmentPeriod), 10, 64) + if err != nil || seconds > 3600 { + alignmentPeriod = "+3600s" + } + + params.Add("aggregation.crossSeriesReducer", primaryAggregation) + params.Add("aggregation.perSeriesAligner", perSeriesAligner) + params.Add("aggregation.alignmentPeriod", alignmentPeriod) + + groupBys := query.Model.Get("groupBys").MustArray() + if len(groupBys) > 0 { + for i := 0; i < len(groupBys); i++ { + params.Add("aggregation.groupByFields", groupBys[i].(string)) + } + } +} + +func (e *StackdriverExecutor) executeQuery(ctx context.Context, query *StackdriverQuery, tsdbQuery *tsdb.TsdbQuery) (*tsdb.QueryResult, StackdriverResponse, error) { + queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: query.RefID} + + req, err := e.createRequest(ctx, e.dsInfo) + if err != nil { + queryResult.Error = err + return queryResult, StackdriverResponse{}, nil + } + + req.URL.RawQuery = query.Params.Encode() + queryResult.Meta.Set("rawQuery", req.URL.RawQuery) + alignmentPeriod, ok := req.URL.Query()["aggregation.alignmentPeriod"] + + if ok { + re := regexp.MustCompile("[0-9]+") + seconds, err := strconv.ParseInt(re.FindString(alignmentPeriod[0]), 10, 64) + if err == nil { + queryResult.Meta.Set("alignmentPeriod", seconds) + } + } + + span, ctx := opentracing.StartSpanFromContext(ctx, "stackdriver query") + span.SetTag("target", query.Target) + span.SetTag("from", tsdbQuery.TimeRange.From) + span.SetTag("until", tsdbQuery.TimeRange.To) + span.SetTag("datasource_id", e.dsInfo.Id) + span.SetTag("org_id", e.dsInfo.OrgId) + + defer span.Finish() + + opentracing.GlobalTracer().Inject( + span.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + res, err := ctxhttp.Do(ctx, e.httpClient, req) + if err != nil { + queryResult.Error = err + return queryResult, StackdriverResponse{}, nil + } + + data, err := e.unmarshalResponse(res) + if err != nil { + queryResult.Error = err + return queryResult, StackdriverResponse{}, nil + } + + return queryResult, data, nil +} + +func (e *StackdriverExecutor) unmarshalResponse(res *http.Response) (StackdriverResponse, error) { + body, err := ioutil.ReadAll(res.Body) + defer res.Body.Close() + if err != nil { + return StackdriverResponse{}, err + } + + if res.StatusCode/100 != 2 { + slog.Error("Request failed", "status", res.Status, "body", string(body)) + return StackdriverResponse{}, fmt.Errorf(string(body)) + } + + var data StackdriverResponse + err = json.Unmarshal(body, &data) + if err != nil { + slog.Error("Failed to unmarshal Stackdriver response", "error", err, "status", res.Status, "body", string(body)) + return StackdriverResponse{}, err + } + + return data, nil +} + +func (e *StackdriverExecutor) parseResponse(queryRes *tsdb.QueryResult, data StackdriverResponse, query *StackdriverQuery) error { + metricLabels := make(map[string][]string) + resourceLabels := make(map[string][]string) + var resourceTypes []string + + for _, series := range data.TimeSeries { + if !containsLabel(resourceTypes, series.Resource.Type) { + resourceTypes = append(resourceTypes, series.Resource.Type) + } + } + + for _, series := range data.TimeSeries { + points := make([]tsdb.TimePoint, 0) + + defaultMetricName := series.Metric.Type + if len(resourceTypes) > 1 { + defaultMetricName += " " + series.Resource.Type + } + + for key, value := range series.Metric.Labels { + if !containsLabel(metricLabels[key], value) { + metricLabels[key] = append(metricLabels[key], value) + } + if len(query.GroupBys) == 0 || containsLabel(query.GroupBys, "metric.label."+key) { + defaultMetricName += " " + value + } + } + + for key, value := range series.Resource.Labels { + if !containsLabel(resourceLabels[key], value) { + resourceLabels[key] = append(resourceLabels[key], value) + } + if containsLabel(query.GroupBys, "resource.label."+key) { + defaultMetricName += " " + value + } + } + + // reverse the order to be ascending + if series.ValueType != "DISTRIBUTION" { + for i := len(series.Points) - 1; i >= 0; i-- { + point := series.Points[i] + value := point.Value.DoubleValue + + if series.ValueType == "INT64" { + parsedValue, err := strconv.ParseFloat(point.Value.IntValue, 64) + if err == nil { + value = parsedValue + } + } + + if series.ValueType == "BOOL" { + if point.Value.BoolValue { + value = 1 + } else { + value = 0 + } + } + + points = append(points, tsdb.NewTimePoint(null.FloatFrom(value), float64((point.Interval.EndTime).Unix())*1000)) + } + + metricName := formatLegendKeys(series.Metric.Type, defaultMetricName, series.Resource.Type, series.Metric.Labels, series.Resource.Labels, make(map[string]string), query) + + queryRes.Series = append(queryRes.Series, &tsdb.TimeSeries{ + Name: metricName, + Points: points, + }) + } else { + buckets := make(map[int]*tsdb.TimeSeries) + + for i := len(series.Points) - 1; i >= 0; i-- { + point := series.Points[i] + if len(point.Value.DistributionValue.BucketCounts) == 0 { + continue + } + maxKey := 0 + for i := 0; i < len(point.Value.DistributionValue.BucketCounts); i++ { + value, err := strconv.ParseFloat(point.Value.DistributionValue.BucketCounts[i], 64) + if err != nil { + continue + } + if _, ok := buckets[i]; !ok { + // set lower bounds + // https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries#Distribution + bucketBound := calcBucketBound(point.Value.DistributionValue.BucketOptions, i) + additionalLabels := map[string]string{"bucket": bucketBound} + buckets[i] = &tsdb.TimeSeries{ + Name: formatLegendKeys(series.Metric.Type, defaultMetricName, series.Resource.Type, series.Metric.Labels, series.Resource.Labels, additionalLabels, query), + Points: make([]tsdb.TimePoint, 0), + } + if maxKey < i { + maxKey = i + } + } + buckets[i].Points = append(buckets[i].Points, tsdb.NewTimePoint(null.FloatFrom(value), float64((point.Interval.EndTime).Unix())*1000)) + } + + // fill empty bucket + for i := 0; i < maxKey; i++ { + if _, ok := buckets[i]; !ok { + bucketBound := calcBucketBound(point.Value.DistributionValue.BucketOptions, i) + additionalLabels := map[string]string{"bucket": bucketBound} + buckets[i] = &tsdb.TimeSeries{ + Name: formatLegendKeys(series.Metric.Type, defaultMetricName, series.Resource.Type, series.Metric.Labels, series.Resource.Labels, additionalLabels, query), + Points: make([]tsdb.TimePoint, 0), + } + } + } + } + for i := 0; i < len(buckets); i++ { + queryRes.Series = append(queryRes.Series, buckets[i]) + } + } + } + + queryRes.Meta.Set("resourceLabels", resourceLabels) + queryRes.Meta.Set("metricLabels", metricLabels) + queryRes.Meta.Set("groupBys", query.GroupBys) + queryRes.Meta.Set("resourceTypes", resourceTypes) + + return nil +} + +func containsLabel(labels []string, newLabel string) bool { + for _, val := range labels { + if val == newLabel { + return true + } + } + return false +} + +func formatLegendKeys(metricType string, defaultMetricName string, resourceType string, metricLabels map[string]string, resourceLabels map[string]string, additionalLabels map[string]string, query *StackdriverQuery) string { + if query.AliasBy == "" { + return defaultMetricName + } + + result := legendKeyFormat.ReplaceAllFunc([]byte(query.AliasBy), func(in []byte) []byte { + metaPartName := strings.Replace(string(in), "{{", "", 1) + metaPartName = strings.Replace(metaPartName, "}}", "", 1) + metaPartName = strings.TrimSpace(metaPartName) + + if metaPartName == "metric.type" { + return []byte(metricType) + } + + if metaPartName == "resource.type" && resourceType != "" { + return []byte(resourceType) + } + + metricPart := replaceWithMetricPart(metaPartName, metricType) + + if metricPart != nil { + return metricPart + } + + metaPartName = strings.Replace(metaPartName, "metric.label.", "", 1) + + if val, exists := metricLabels[metaPartName]; exists { + return []byte(val) + } + + metaPartName = strings.Replace(metaPartName, "resource.label.", "", 1) + + if val, exists := resourceLabels[metaPartName]; exists { + return []byte(val) + } + + if val, exists := additionalLabels[metaPartName]; exists { + return []byte(val) + } + + return in + }) + + return string(result) +} + +func replaceWithMetricPart(metaPartName string, metricType string) []byte { + // https://cloud.google.com/monitoring/api/v3/metrics-details#label_names + shortMatches := metricNameFormat.FindStringSubmatch(metricType) + + if metaPartName == "metric.name" { + if len(shortMatches) > 0 { + return []byte(shortMatches[2]) + } + } + + if metaPartName == "metric.service" { + if len(shortMatches) > 0 { + return []byte(shortMatches[1]) + } + } + + return nil +} + +func calcBucketBound(bucketOptions StackdriverBucketOptions, n int) string { + bucketBound := "0" + if n == 0 { + return bucketBound + } + + if bucketOptions.LinearBuckets != nil { + bucketBound = strconv.FormatInt(bucketOptions.LinearBuckets.Offset+(bucketOptions.LinearBuckets.Width*int64(n-1)), 10) + } else if bucketOptions.ExponentialBuckets != nil { + bucketBound = strconv.FormatInt(int64(bucketOptions.ExponentialBuckets.Scale*math.Pow(bucketOptions.ExponentialBuckets.GrowthFactor, float64(n-1))), 10) + } else if bucketOptions.ExplicitBuckets != nil { + bucketBound = strconv.FormatInt(bucketOptions.ExplicitBuckets.Bounds[(n-1)], 10) + } + return bucketBound +} + +func (e *StackdriverExecutor) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) { + u, _ := url.Parse(dsInfo.Url) + u.Path = path.Join(u.Path, "render") + + req, err := http.NewRequest(http.MethodGet, "https://monitoring.googleapis.com/", nil) + if err != nil { + slog.Error("Failed to create request", "error", err) + return nil, fmt.Errorf("Failed to create request. error: %v", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) + + // find plugin + plugin, ok := plugins.DataSources[dsInfo.Type] + if !ok { + return nil, errors.New("Unable to find datasource plugin Stackdriver") + } + + var stackdriverRoute *plugins.AppPluginRoute + for _, route := range plugin.Routes { + if route.Path == "stackdriver" { + stackdriverRoute = route + break + } + } + + projectName := dsInfo.JsonData.Get("defaultProject").MustString() + proxyPass := fmt.Sprintf("stackdriver%s", "v3/projects/"+projectName+"/timeSeries") + + pluginproxy.ApplyRoute(ctx, req, proxyPass, stackdriverRoute, dsInfo) + + return req, nil +} + +func (e *StackdriverExecutor) getDefaultProject(ctx context.Context) (string, error) { + authenticationType := e.dsInfo.JsonData.Get("authenticationType").MustString(jwtAuthentication) + if authenticationType == gceAuthentication { + defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") + if err != nil { + return "", fmt.Errorf("Failed to retrieve default project from GCE metadata server. error: %v", err) + } + return defaultCredentials.ProjectID, nil + } + return e.dsInfo.JsonData.Get("defaultProject").MustString(), nil +} diff --git a/pkg/tsdb/stackdriver/stackdriver_test.go b/pkg/tsdb/stackdriver/stackdriver_test.go new file mode 100644 index 00000000000..784bf4a7fbb --- /dev/null +++ b/pkg/tsdb/stackdriver/stackdriver_test.go @@ -0,0 +1,490 @@ +package stackdriver + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "math" + "strconv" + "testing" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestStackdriver(t *testing.T) { + Convey("Stackdriver", t, func() { + executor := &StackdriverExecutor{} + + Convey("Parse queries from frontend and build Stackdriver API queries", func() { + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) + tsdbQuery := &tsdb.TsdbQuery{ + TimeRange: &tsdb.TimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "metricType": "a/metric/type", + "view": "FULL", + "aliasBy": "testalias", + "type": "timeSeriesQuery", + }), + RefId: "A", + }, + }, + } + + Convey("and query has no aggregation set", func() { + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + + So(len(queries), ShouldEqual, 1) + So(queries[0].RefID, ShouldEqual, "A") + So(queries[0].Target, ShouldEqual, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL") + So(len(queries[0].Params), ShouldEqual, 7) + So(queries[0].Params["interval.startTime"][0], ShouldEqual, "2018-03-15T13:00:00Z") + So(queries[0].Params["interval.endTime"][0], ShouldEqual, "2018-03-15T13:34:00Z") + So(queries[0].Params["aggregation.perSeriesAligner"][0], ShouldEqual, "ALIGN_MEAN") + So(queries[0].Params["filter"][0], ShouldEqual, "metric.type=\"a/metric/type\"") + So(queries[0].Params["view"][0], ShouldEqual, "FULL") + So(queries[0].AliasBy, ShouldEqual, "testalias") + }) + + Convey("and query has filters", func() { + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "metricType": "a/metric/type", + "filters": []interface{}{"key", "=", "value", "AND", "key2", "=", "value2"}, + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(len(queries), ShouldEqual, 1) + So(queries[0].Params["filter"][0], ShouldEqual, `metric.type="a/metric/type" key="value" key2="value2"`) + }) + + Convey("and alignmentPeriod is set to grafana-auto", func() { + Convey("and IntervalMs is larger than 60000", func() { + tsdbQuery.Queries[0].IntervalMs = 1000000 + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "alignmentPeriod": "grafana-auto", + "filters": []interface{}{"key", "=", "value", "AND", "key2", "=", "value2"}, + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+1000s`) + }) + Convey("and IntervalMs is less than 60000", func() { + tsdbQuery.Queries[0].IntervalMs = 30000 + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "alignmentPeriod": "grafana-auto", + "filters": []interface{}{"key", "=", "value", "AND", "key2", "=", "value2"}, + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+60s`) + }) + }) + + Convey("and alignmentPeriod is set to stackdriver-auto", func() { + Convey("and range is two hours", func() { + tsdbQuery.TimeRange.From = "1538033322461" + tsdbQuery.TimeRange.To = "1538040522461" + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "target": "target", + "alignmentPeriod": "stackdriver-auto", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+60s`) + }) + + Convey("and range is 22 hours", func() { + tsdbQuery.TimeRange.From = "1538034524922" + tsdbQuery.TimeRange.To = "1538113724922" + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "target": "target", + "alignmentPeriod": "stackdriver-auto", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+60s`) + }) + + Convey("and range is 23 hours", func() { + tsdbQuery.TimeRange.From = "1538034567985" + tsdbQuery.TimeRange.To = "1538117367985" + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "target": "target", + "alignmentPeriod": "stackdriver-auto", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+300s`) + }) + + Convey("and range is 7 days", func() { + tsdbQuery.TimeRange.From = "1538036324073" + tsdbQuery.TimeRange.To = "1538641124073" + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "target": "target", + "alignmentPeriod": "stackdriver-auto", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+3600s`) + }) + }) + + Convey("and alignmentPeriod is set in frontend", func() { + Convey("and alignment period is too big", func() { + tsdbQuery.Queries[0].IntervalMs = 1000 + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "alignmentPeriod": "+360000s", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+3600s`) + }) + + Convey("and alignment period is within accepted range", func() { + tsdbQuery.Queries[0].IntervalMs = 1000 + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "alignmentPeriod": "+600s", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, `+600s`) + }) + }) + + Convey("and query has aggregation mean set", func() { + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "metricType": "a/metric/type", + "primaryAggregation": "REDUCE_MEAN", + "view": "FULL", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + + So(len(queries), ShouldEqual, 1) + So(queries[0].RefID, ShouldEqual, "A") + So(queries[0].Target, ShouldEqual, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_MEAN&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL") + So(len(queries[0].Params), ShouldEqual, 7) + So(queries[0].Params["interval.startTime"][0], ShouldEqual, "2018-03-15T13:00:00Z") + So(queries[0].Params["interval.endTime"][0], ShouldEqual, "2018-03-15T13:34:00Z") + So(queries[0].Params["aggregation.crossSeriesReducer"][0], ShouldEqual, "REDUCE_MEAN") + So(queries[0].Params["aggregation.perSeriesAligner"][0], ShouldEqual, "ALIGN_MEAN") + So(queries[0].Params["aggregation.alignmentPeriod"][0], ShouldEqual, "+60s") + So(queries[0].Params["filter"][0], ShouldEqual, "metric.type=\"a/metric/type\"") + So(queries[0].Params["view"][0], ShouldEqual, "FULL") + }) + + Convey("and query has group bys", func() { + tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ + "metricType": "a/metric/type", + "primaryAggregation": "REDUCE_NONE", + "groupBys": []interface{}{"metric.label.group1", "metric.label.group2"}, + "view": "FULL", + }) + + queries, err := executor.buildQueries(tsdbQuery) + So(err, ShouldBeNil) + + So(len(queries), ShouldEqual, 1) + So(queries[0].RefID, ShouldEqual, "A") + So(queries[0].Target, ShouldEqual, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.groupByFields=metric.label.group1&aggregation.groupByFields=metric.label.group2&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL") + So(len(queries[0].Params), ShouldEqual, 8) + So(queries[0].Params["interval.startTime"][0], ShouldEqual, "2018-03-15T13:00:00Z") + So(queries[0].Params["interval.endTime"][0], ShouldEqual, "2018-03-15T13:34:00Z") + So(queries[0].Params["aggregation.perSeriesAligner"][0], ShouldEqual, "ALIGN_MEAN") + So(queries[0].Params["aggregation.groupByFields"][0], ShouldEqual, "metric.label.group1") + So(queries[0].Params["aggregation.groupByFields"][1], ShouldEqual, "metric.label.group2") + So(queries[0].Params["filter"][0], ShouldEqual, "metric.type=\"a/metric/type\"") + So(queries[0].Params["view"][0], ShouldEqual, "FULL") + }) + + }) + + Convey("Parse stackdriver response in the time series format", func() { + Convey("when data from query aggregated to one time series", func() { + data, err := loadTestFile("./test-data/1-series-response-agg-one-metric.json") + So(err, ShouldBeNil) + So(len(data.TimeSeries), ShouldEqual, 1) + + res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"} + query := &StackdriverQuery{} + err = executor.parseResponse(res, data, query) + So(err, ShouldBeNil) + + So(len(res.Series), ShouldEqual, 1) + So(res.Series[0].Name, ShouldEqual, "serviceruntime.googleapis.com/api/request_count") + So(len(res.Series[0].Points), ShouldEqual, 3) + + Convey("timestamps should be in ascending order", func() { + So(res.Series[0].Points[0][0].Float64, ShouldEqual, 0.05) + So(res.Series[0].Points[0][1].Float64, ShouldEqual, 1536670020000) + + So(res.Series[0].Points[1][0].Float64, ShouldEqual, 1.05) + So(res.Series[0].Points[1][1].Float64, ShouldEqual, 1536670080000) + + So(res.Series[0].Points[2][0].Float64, ShouldEqual, 1.0666666666667) + So(res.Series[0].Points[2][1].Float64, ShouldEqual, 1536670260000) + }) + }) + + Convey("when data from query with no aggregation", func() { + data, err := loadTestFile("./test-data/2-series-response-no-agg.json") + So(err, ShouldBeNil) + So(len(data.TimeSeries), ShouldEqual, 3) + + res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"} + query := &StackdriverQuery{} + err = executor.parseResponse(res, data, query) + So(err, ShouldBeNil) + + Convey("Should add labels to metric name", func() { + So(len(res.Series), ShouldEqual, 3) + So(res.Series[0].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time collector-asia-east-1") + So(res.Series[1].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time collector-europe-west-1") + So(res.Series[2].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time collector-us-east-1") + }) + + Convey("Should parse to time series", func() { + So(len(res.Series[0].Points), ShouldEqual, 3) + So(res.Series[0].Points[0][0].Float64, ShouldEqual, 9.8566497180145) + So(res.Series[0].Points[1][0].Float64, ShouldEqual, 9.7323568146676) + So(res.Series[0].Points[2][0].Float64, ShouldEqual, 9.7730520330369) + }) + + Convey("Should add meta for labels to the response", func() { + metricLabels := res.Meta.Get("metricLabels").Interface().(map[string][]string) + So(metricLabels, ShouldNotBeNil) + So(len(metricLabels["instance_name"]), ShouldEqual, 3) + So(metricLabels["instance_name"][0], ShouldEqual, "collector-asia-east-1") + So(metricLabels["instance_name"][1], ShouldEqual, "collector-europe-west-1") + So(metricLabels["instance_name"][2], ShouldEqual, "collector-us-east-1") + + resourceLabels := res.Meta.Get("resourceLabels").Interface().(map[string][]string) + So(resourceLabels, ShouldNotBeNil) + So(len(resourceLabels["zone"]), ShouldEqual, 3) + So(resourceLabels["zone"][0], ShouldEqual, "asia-east1-a") + So(resourceLabels["zone"][1], ShouldEqual, "europe-west1-b") + So(resourceLabels["zone"][2], ShouldEqual, "us-east1-b") + + So(len(resourceLabels["project_id"]), ShouldEqual, 1) + So(resourceLabels["project_id"][0], ShouldEqual, "grafana-prod") + }) + }) + + Convey("when data from query with no aggregation and group bys", func() { + data, err := loadTestFile("./test-data/2-series-response-no-agg.json") + So(err, ShouldBeNil) + So(len(data.TimeSeries), ShouldEqual, 3) + + res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"} + query := &StackdriverQuery{GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}} + err = executor.parseResponse(res, data, query) + So(err, ShouldBeNil) + + Convey("Should add instance name and zone labels to metric name", func() { + So(len(res.Series), ShouldEqual, 3) + So(res.Series[0].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time collector-asia-east-1 asia-east1-a") + So(res.Series[1].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time collector-europe-west-1 europe-west1-b") + So(res.Series[2].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time collector-us-east-1 us-east1-b") + }) + }) + + Convey("when data from query with no aggregation and alias by", func() { + data, err := loadTestFile("./test-data/2-series-response-no-agg.json") + So(err, ShouldBeNil) + So(len(data.TimeSeries), ShouldEqual, 3) + + res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"} + + Convey("and the alias pattern is for metric type, a metric label and a resource label", func() { + + query := &StackdriverQuery{AliasBy: "{{metric.type}} - {{metric.label.instance_name}} - {{resource.label.zone}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}} + err = executor.parseResponse(res, data, query) + So(err, ShouldBeNil) + + Convey("Should use alias by formatting and only show instance name", func() { + So(len(res.Series), ShouldEqual, 3) + So(res.Series[0].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time - collector-asia-east-1 - asia-east1-a") + So(res.Series[1].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time - collector-europe-west-1 - europe-west1-b") + So(res.Series[2].Name, ShouldEqual, "compute.googleapis.com/instance/cpu/usage_time - collector-us-east-1 - us-east1-b") + }) + }) + + Convey("and the alias pattern is for metric name", func() { + + query := &StackdriverQuery{AliasBy: "metric {{metric.name}} service {{metric.service}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}} + err = executor.parseResponse(res, data, query) + So(err, ShouldBeNil) + + Convey("Should use alias by formatting and only show instance name", func() { + So(len(res.Series), ShouldEqual, 3) + So(res.Series[0].Name, ShouldEqual, "metric instance/cpu/usage_time service compute") + So(res.Series[1].Name, ShouldEqual, "metric instance/cpu/usage_time service compute") + So(res.Series[2].Name, ShouldEqual, "metric instance/cpu/usage_time service compute") + }) + }) + }) + + Convey("when data from query is distribution", func() { + data, err := loadTestFile("./test-data/3-series-response-distribution.json") + So(err, ShouldBeNil) + So(len(data.TimeSeries), ShouldEqual, 1) + + res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"} + query := &StackdriverQuery{AliasBy: "{{bucket}}"} + err = executor.parseResponse(res, data, query) + So(err, ShouldBeNil) + + So(len(res.Series), ShouldEqual, 11) + for i := 0; i < 11; i++ { + if i == 0 { + So(res.Series[i].Name, ShouldEqual, "0") + } else { + So(res.Series[i].Name, ShouldEqual, strconv.FormatInt(int64(math.Pow(float64(2), float64(i-1))), 10)) + } + So(len(res.Series[i].Points), ShouldEqual, 3) + } + + Convey("timestamps should be in ascending order", func() { + So(res.Series[0].Points[0][1].Float64, ShouldEqual, 1536668940000) + So(res.Series[0].Points[1][1].Float64, ShouldEqual, 1536669000000) + So(res.Series[0].Points[2][1].Float64, ShouldEqual, 1536669060000) + }) + + Convey("value should be correct", func() { + So(res.Series[8].Points[0][0].Float64, ShouldEqual, 1) + So(res.Series[9].Points[0][0].Float64, ShouldEqual, 1) + So(res.Series[10].Points[0][0].Float64, ShouldEqual, 1) + So(res.Series[8].Points[1][0].Float64, ShouldEqual, 0) + So(res.Series[9].Points[1][0].Float64, ShouldEqual, 0) + So(res.Series[10].Points[1][0].Float64, ShouldEqual, 1) + So(res.Series[8].Points[2][0].Float64, ShouldEqual, 0) + So(res.Series[9].Points[2][0].Float64, ShouldEqual, 1) + So(res.Series[10].Points[2][0].Float64, ShouldEqual, 0) + }) + }) + + }) + + Convey("when interpolating filter wildcards", func() { + Convey("and wildcard is used in the beginning and the end of the word", func() { + Convey("and theres no wildcard in the middle of the word", func() { + value := interpolateFilterWildcards("*-central1*") + So(value, ShouldEqual, `has_substring("-central1")`) + }) + Convey("and there is a wildcard in the middle of the word", func() { + value := interpolateFilterWildcards("*-cent*ral1*") + So(value, ShouldNotStartWith, `has_substring`) + }) + }) + + Convey("and wildcard is used in the beginning of the word", func() { + Convey("and there is not a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("*-central1") + So(value, ShouldEqual, `ends_with("-central1")`) + }) + Convey("and there is a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("*-cent*al1") + So(value, ShouldNotStartWith, `ends_with`) + }) + }) + + Convey("and wildcard is used at the end of the word", func() { + Convey("and there is not a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("us-central*") + So(value, ShouldEqual, `starts_with("us-central")`) + }) + Convey("and there is a wildcard elsewhere in the word", func() { + value := interpolateFilterWildcards("*us-central*") + So(value, ShouldNotStartWith, `starts_with`) + }) + }) + + Convey("and wildcard is used in the middle of the word", func() { + Convey("and there is only one wildcard", func() { + value := interpolateFilterWildcards("us-ce*tral1-b") + So(value, ShouldEqual, `monitoring.regex.full_match("^us\\-ce.*tral1\\-b$")`) + }) + + Convey("and there is more than one wildcard", func() { + value := interpolateFilterWildcards("us-ce*tra*1-b") + So(value, ShouldEqual, `monitoring.regex.full_match("^us\\-ce.*tra.*1\\-b$")`) + }) + }) + + Convey("and wildcard is used in the middle of the word and in the beginning of the word", func() { + value := interpolateFilterWildcards("*s-ce*tral1-b") + So(value, ShouldEqual, `monitoring.regex.full_match("^.*s\\-ce.*tral1\\-b$")`) + }) + + Convey("and wildcard is used in the middle of the word and in the ending of the word", func() { + value := interpolateFilterWildcards("us-ce*tral1-*") + So(value, ShouldEqual, `monitoring.regex.full_match("^us\\-ce.*tral1\\-.*$")`) + }) + + Convey("and no wildcard is used", func() { + value := interpolateFilterWildcards("us-central1-a}") + So(value, ShouldEqual, `us-central1-a}`) + }) + }) + + Convey("when building filter string", func() { + Convey("and theres no regex operator", func() { + Convey("and there are wildcards in a filter value", func() { + filterParts := []interface{}{"zone", "=", "*-central1*"} + value := buildFilterString("somemetrictype", filterParts) + So(value, ShouldEqual, `metric.type="somemetrictype" zone=has_substring("-central1")`) + }) + + Convey("and there are no wildcards in any filter value", func() { + filterParts := []interface{}{"zone", "!=", "us-central1-a"} + value := buildFilterString("somemetrictype", filterParts) + So(value, ShouldEqual, `metric.type="somemetrictype" zone!="us-central1-a"`) + }) + }) + + Convey("and there is a regex operator", func() { + filterParts := []interface{}{"zone", "=~", "us-central1-a~"} + value := buildFilterString("somemetrictype", filterParts) + Convey("it should remove the ~ character from the operator that belongs to the value", func() { + So(value, ShouldNotContainSubstring, `=~`) + So(value, ShouldContainSubstring, `zone=`) + }) + + Convey("it should insert monitoring.regex.full_match before filter value", func() { + So(value, ShouldContainSubstring, `zone=monitoring.regex.full_match("us-central1-a~")`) + }) + }) + }) + }) +} + +func loadTestFile(path string) (StackdriverResponse, error) { + var data StackdriverResponse + + jsonBody, err := ioutil.ReadFile(path) + if err != nil { + return data, err + } + err = json.Unmarshal(jsonBody, &data) + return data, err +} diff --git a/pkg/tsdb/stackdriver/test-data/1-series-response-agg-one-metric.json b/pkg/tsdb/stackdriver/test-data/1-series-response-agg-one-metric.json new file mode 100644 index 00000000000..e1a84583cc4 --- /dev/null +++ b/pkg/tsdb/stackdriver/test-data/1-series-response-agg-one-metric.json @@ -0,0 +1,46 @@ +{ + "timeSeries": [ + { + "metric": { + "type": "serviceruntime.googleapis.com\/api\/request_count" + }, + "resource": { + "type": "consumed_api", + "labels": { + "project_id": "grafana-prod" + } + }, + "metricKind": "GAUGE", + "valueType": "DOUBLE", + "points": [ + { + "interval": { + "startTime": "2018-09-11T12:51:00Z", + "endTime": "2018-09-11T12:51:00Z" + }, + "value": { + "doubleValue": 1.0666666666667 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:48:00Z", + "endTime": "2018-09-11T12:48:00Z" + }, + "value": { + "doubleValue": 1.05 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:47:00Z", + "endTime": "2018-09-11T12:47:00Z" + }, + "value": { + "doubleValue": 0.05 + } + } + ] + } + ] +} diff --git a/pkg/tsdb/stackdriver/test-data/2-series-response-no-agg.json b/pkg/tsdb/stackdriver/test-data/2-series-response-no-agg.json new file mode 100644 index 00000000000..da615a168bf --- /dev/null +++ b/pkg/tsdb/stackdriver/test-data/2-series-response-no-agg.json @@ -0,0 +1,145 @@ +{ + "timeSeries": [ + { + "metric": { + "labels": { + "instance_name": "collector-asia-east-1" + }, + "type": "compute.googleapis.com\/instance\/cpu\/usage_time" + }, + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "1119268429530133111", + "zone": "asia-east1-a", + "project_id": "grafana-prod" + } + }, + "metricKind": "DELTA", + "valueType": "DOUBLE", + "points": [ + { + "interval": { + "startTime": "2018-09-11T12:30:00Z", + "endTime": "2018-09-11T12:31:00Z" + }, + "value": { + "doubleValue": 9.7730520330369 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:29:00Z", + "endTime": "2018-09-11T12:30:00Z" + }, + "value": { + "doubleValue": 9.7323568146676 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:28:00Z", + "endTime": "2018-09-11T12:29:00Z" + }, + "value": { + "doubleValue": 9.8566497180145 + } + } + ] + }, + { + "metric": { + "labels": { + "instance_name": "collector-europe-west-1" + }, + "type": "compute.googleapis.com\/instance\/cpu\/usage_time" + }, + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "22241654114540837222", + "zone": "europe-west1-b", + "project_id": "grafana-prod" + } + }, + "metricKind": "DELTA", + "valueType": "DOUBLE", + "points": [ + { + "interval": { + "startTime": "2018-09-11T12:30:00Z", + "endTime": "2018-09-11T12:31:00Z" + }, + "value": { + "doubleValue": 8.8210971239023 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:29:00Z", + "endTime": "2018-09-11T12:30:00Z" + }, + "value": { + "doubleValue": 8.9689492364414 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:28:00Z", + "endTime": "2018-09-11T12:29:00Z" + }, + "value": { + "doubleValue": 9.0238475054502 + } + } + ] + }, + { + "metric": { + "labels": { + "instance_name": "collector-us-east-1" + }, + "type": "compute.googleapis.com\/instance\/cpu\/usage_time" + }, + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "3332264424035095333", + "zone": "us-east1-b", + "project_id": "grafana-prod" + } + }, + "metricKind": "DELTA", + "valueType": "DOUBLE", + "points": [ + { + "interval": { + "startTime": "2018-09-11T12:30:00Z", + "endTime": "2018-09-11T12:31:00Z" + }, + "value": { + "doubleValue": 30.807846801355 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:29:00Z", + "endTime": "2018-09-11T12:30:00Z" + }, + "value": { + "doubleValue": 30.903974115849 + } + }, + { + "interval": { + "startTime": "2018-09-11T12:28:00Z", + "endTime": "2018-09-11T12:29:00Z" + }, + "value": { + "doubleValue": 30.829426143318 + } + } + ] + } + ] +} diff --git a/pkg/tsdb/stackdriver/test-data/3-series-response-distribution.json b/pkg/tsdb/stackdriver/test-data/3-series-response-distribution.json new file mode 100644 index 00000000000..8603f78eab4 --- /dev/null +++ b/pkg/tsdb/stackdriver/test-data/3-series-response-distribution.json @@ -0,0 +1,112 @@ +{ + "timeSeries": [ + { + "metric": { + "type": "loadbalancing.googleapis.com\/https\/backend_latencies" + }, + "resource": { + "type": "https_lb_rule", + "labels": { + "project_id": "grafana-prod" + } + }, + "metricKind": "DELTA", + "valueType": "DISTRIBUTION", + "points": [ + { + "interval": { + "startTime": "2018-09-11T12:30:00Z", + "endTime": "2018-09-11T12:31:00Z" + }, + "value": { + "distributionValue": { + "count": "1", + "bucketOptions": { + "exponentialBuckets": { + "numFiniteBuckets": 10, + "growthFactor": 2, + "scale": 1 + } + }, + "bucketCounts": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0" + ] + } + } + }, + { + "interval": { + "startTime": "2018-09-11T12:29:00Z", + "endTime": "2018-09-11T12:30:00Z" + }, + "value": { + "distributionValue": { + "count": "1", + "bucketOptions": { + "exponentialBuckets": { + "numFiniteBuckets": 10, + "growthFactor": 2, + "scale": 1 + } + }, + "bucketCounts": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1" + ] + } + } + }, + { + "interval": { + "startTime": "2018-09-11T12:28:00Z", + "endTime": "2018-09-11T12:29:00Z" + }, + "value": { + "distributionValue": { + "count": "3", + "bucketOptions": { + "exponentialBuckets": { + "numFiniteBuckets": 10, + "growthFactor": 2, + "scale": 1 + } + }, + "bucketCounts": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "1", + "1" + ] + } + } + } + ] + } + ] +} diff --git a/pkg/tsdb/stackdriver/types.go b/pkg/tsdb/stackdriver/types.go new file mode 100644 index 00000000000..3821ce7ceda --- /dev/null +++ b/pkg/tsdb/stackdriver/types.go @@ -0,0 +1,75 @@ +package stackdriver + +import ( + "net/url" + "time" +) + +// StackdriverQuery is the query that Grafana sends from the frontend +type StackdriverQuery struct { + Target string + Params url.Values + RefID string + GroupBys []string + AliasBy string +} + +type StackdriverBucketOptions struct { + LinearBuckets *struct { + NumFiniteBuckets int64 `json:"numFiniteBuckets"` + Width int64 `json:"width"` + Offset int64 `json:"offset"` + } `json:"linearBuckets"` + ExponentialBuckets *struct { + NumFiniteBuckets int64 `json:"numFiniteBuckets"` + GrowthFactor float64 `json:"growthFactor"` + Scale float64 `json:"scale"` + } `json:"exponentialBuckets"` + ExplicitBuckets *struct { + Bounds []int64 `json:"bounds"` + } `json:"explicitBuckets"` +} + +// StackdriverResponse is the data returned from the external Google Stackdriver API +type StackdriverResponse struct { + TimeSeries []struct { + Metric struct { + Labels map[string]string `json:"labels"` + Type string `json:"type"` + } `json:"metric"` + Resource struct { + Type string `json:"type"` + Labels map[string]string `json:"labels"` + } `json:"resource"` + MetricKind string `json:"metricKind"` + ValueType string `json:"valueType"` + Points []struct { + Interval struct { + StartTime time.Time `json:"startTime"` + EndTime time.Time `json:"endTime"` + } `json:"interval"` + Value struct { + DoubleValue float64 `json:"doubleValue"` + StringValue string `json:"stringValue"` + BoolValue bool `json:"boolValue"` + IntValue string `json:"int64Value"` + DistributionValue struct { + Count string `json:"count"` + Mean float64 `json:"mean"` + SumOfSquaredDeviation float64 `json:"sumOfSquaredDeviation"` + Range struct { + Min int `json:"min"` + Max int `json:"max"` + } `json:"range"` + BucketOptions StackdriverBucketOptions `json:"bucketOptions"` + BucketCounts []string `json:"bucketCounts"` + Examplars []struct { + Value float64 `json:"value"` + Timestamp string `json:"timestamp"` + // attachments + } `json:"examplars"` + } `json:"distributionValue"` + } `json:"value"` + } `json:"points"` + } `json:"timeSeries"` +} diff --git a/pkg/tsdb/testdata/scenarios.go b/pkg/tsdb/testdata/scenarios.go index e907fa8aae0..421a907b5e9 100644 --- a/pkg/tsdb/testdata/scenarios.go +++ b/pkg/tsdb/testdata/scenarios.go @@ -95,27 +95,20 @@ func init() { Id: "random_walk", Name: "Random Walk", - Handler: func(query *tsdb.Query, tsdbQuery *tsdb.TsdbQuery) *tsdb.QueryResult { - timeWalkerMs := tsdbQuery.TimeRange.GetFromAsMsEpoch() - to := tsdbQuery.TimeRange.GetToAsMsEpoch() + Handler: func(query *tsdb.Query, context *tsdb.TsdbQuery) *tsdb.QueryResult { + return getRandomWalk(query, context) + }, + }) - series := newSeriesForQuery(query) - - points := make(tsdb.TimeSeriesPoints, 0) - walker := rand.Float64() * 100 - - for i := int64(0); i < 10000 && timeWalkerMs < to; i++ { - points = append(points, tsdb.NewTimePoint(null.FloatFrom(walker), float64(timeWalkerMs))) - - walker += rand.Float64() - 0.5 - timeWalkerMs += query.IntervalMs - } - - series.Points = points - - queryRes := tsdb.NewQueryResult() - queryRes.Series = append(queryRes.Series, series) - return queryRes + registerScenario(&Scenario{ + Id: "slow_query", + Name: "Slow Query", + StringInput: "5s", + Handler: func(query *tsdb.Query, context *tsdb.TsdbQuery) *tsdb.QueryResult { + stringInput := query.Model.Get("stringInput").MustString() + parsedInterval, _ := time.ParseDuration(stringInput) + time.Sleep(parsedInterval) + return getRandomWalk(query, context) }, }) @@ -221,6 +214,57 @@ func init() { return queryRes }, }) + + registerScenario(&Scenario{ + Id: "table_static", + Name: "Table Static", + + Handler: func(query *tsdb.Query, context *tsdb.TsdbQuery) *tsdb.QueryResult { + timeWalkerMs := context.TimeRange.GetFromAsMsEpoch() + to := context.TimeRange.GetToAsMsEpoch() + + table := tsdb.Table{ + Columns: []tsdb.TableColumn{ + {Text: "Time"}, + {Text: "Message"}, + {Text: "Description"}, + {Text: "Value"}, + }, + Rows: []tsdb.RowValues{}, + } + for i := int64(0); i < 10 && timeWalkerMs < to; i++ { + table.Rows = append(table.Rows, tsdb.RowValues{float64(timeWalkerMs), "This is a message", "Description", 23.1}) + timeWalkerMs += query.IntervalMs + } + + queryRes := tsdb.NewQueryResult() + queryRes.Tables = append(queryRes.Tables, &table) + return queryRes + }, + }) +} + +func getRandomWalk(query *tsdb.Query, tsdbQuery *tsdb.TsdbQuery) *tsdb.QueryResult { + timeWalkerMs := tsdbQuery.TimeRange.GetFromAsMsEpoch() + to := tsdbQuery.TimeRange.GetToAsMsEpoch() + + series := newSeriesForQuery(query) + + points := make(tsdb.TimeSeriesPoints, 0) + walker := rand.Float64() * 100 + + for i := int64(0); i < 10000 && timeWalkerMs < to; i++ { + points = append(points, tsdb.NewTimePoint(null.FloatFrom(walker), float64(timeWalkerMs))) + + walker += rand.Float64() - 0.5 + timeWalkerMs += query.IntervalMs + } + + series.Points = points + + queryRes := tsdb.NewQueryResult() + queryRes.Series = append(queryRes.Series, series) + return queryRes } func registerScenario(scenario *Scenario) { diff --git a/pkg/tsdb/testdata/testdata.go b/pkg/tsdb/testdata/testdata.go index a1ab250ad37..c2c2ea3f696 100644 --- a/pkg/tsdb/testdata/testdata.go +++ b/pkg/tsdb/testdata/testdata.go @@ -21,7 +21,7 @@ func NewTestDataExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, err } func init() { - tsdb.RegisterTsdbQueryEndpoint("grafana-testdata-datasource", NewTestDataExecutor) + tsdb.RegisterTsdbQueryEndpoint("testdata", NewTestDataExecutor) } func (e *TestDataExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { diff --git a/pkg/tsdb/time_range.go b/pkg/tsdb/time_range.go index fd797bf731a..18e389e5993 100644 --- a/pkg/tsdb/time_range.go +++ b/pkg/tsdb/time_range.go @@ -15,6 +15,14 @@ func NewTimeRange(from, to string) *TimeRange { } } +func NewFakeTimeRange(from, to string, now time.Time) *TimeRange { + return &TimeRange{ + From: from, + To: to, + now: now, + } +} + type TimeRange struct { From string To string @@ -25,24 +33,40 @@ func (tr *TimeRange) GetFromAsMsEpoch() int64 { return tr.MustGetFrom().UnixNano() / int64(time.Millisecond) } +func (tr *TimeRange) GetFromAsSecondsEpoch() int64 { + return tr.GetFromAsMsEpoch() / 1000 +} + +func (tr *TimeRange) GetFromAsTimeUTC() time.Time { + return tr.MustGetFrom().UTC() +} + func (tr *TimeRange) GetToAsMsEpoch() int64 { return tr.MustGetTo().UnixNano() / int64(time.Millisecond) } +func (tr *TimeRange) GetToAsSecondsEpoch() int64 { + return tr.GetToAsMsEpoch() / 1000 +} + +func (tr *TimeRange) GetToAsTimeUTC() time.Time { + return tr.MustGetTo().UTC() +} + func (tr *TimeRange) MustGetFrom() time.Time { - if res, err := tr.ParseFrom(); err != nil { + res, err := tr.ParseFrom() + if err != nil { return time.Unix(0, 0) - } else { - return res } + return res } func (tr *TimeRange) MustGetTo() time.Time { - if res, err := tr.ParseTo(); err != nil { + res, err := tr.ParseTo() + if err != nil { return time.Unix(0, 0) - } else { - return res } + return res } func tryParseUnixMsEpoch(val string) (time.Time, bool) { @@ -88,3 +112,18 @@ func (tr *TimeRange) ParseTo() (time.Time, error) { return time.Time{}, fmt.Errorf("cannot parse to value %s", tr.To) } + +// EpochPrecisionToMs converts epoch precision to millisecond, if needed. +// Only seconds to milliseconds supported right now +func EpochPrecisionToMs(value float64) float64 { + s := strconv.FormatFloat(value, 'e', -1, 64) + if strings.HasSuffix(s, "e+09") { + return value * float64(1e3) + } + + if strings.HasSuffix(s, "e+18") { + return value / float64(time.Millisecond) + } + + return value +} diff --git a/pkg/util/filepath.go b/pkg/util/filepath.go index 3ad8cac3147..d304236fcb1 100644 --- a/pkg/util/filepath.go +++ b/pkg/util/filepath.go @@ -65,9 +65,8 @@ func walk(path string, info os.FileInfo, resolvedPath string, symlinkPathsFollow if _, ok := symlinkPathsFollowed[path2]; ok { errMsg := "Potential SymLink Infinite Loop. Path: %v, Link To: %v" return fmt.Errorf(errMsg, resolvedPath, path2) - } else { - symlinkPathsFollowed[path2] = true } + symlinkPathsFollowed[path2] = true } info2, err := os.Lstat(path2) if err != nil { diff --git a/pkg/util/md5.go b/pkg/util/md5.go new file mode 100644 index 00000000000..2473a1a406c --- /dev/null +++ b/pkg/util/md5.go @@ -0,0 +1,26 @@ +package util + +import ( + "crypto/md5" + "encoding/hex" + "io" + "strings" +) + +// Md5Sum calculates the md5sum of a stream +func Md5Sum(reader io.Reader) (string, error) { + var returnMD5String string + hash := md5.New() + if _, err := io.Copy(hash, reader); err != nil { + return returnMD5String, err + } + hashInBytes := hash.Sum(nil)[:16] + returnMD5String = hex.EncodeToString(hashInBytes) + return returnMD5String, nil +} + +// Md5Sum calculates the md5sum of a string +func Md5SumString(input string) (string, error) { + buffer := strings.NewReader(input) + return Md5Sum(buffer) +} diff --git a/pkg/util/md5_test.go b/pkg/util/md5_test.go new file mode 100644 index 00000000000..43c685b8763 --- /dev/null +++ b/pkg/util/md5_test.go @@ -0,0 +1,17 @@ +package util + +import "testing" + +func TestMd5Sum(t *testing.T) { + input := "don't hash passwords with md5" + + have, err := Md5SumString(input) + if err != nil { + t.Fatal("expected err to be nil") + } + + want := "dd1f7fdb3466c0d09c2e839d1f1530f8" + if have != want { + t.Fatalf("expected: %s got: %s", want, have) + } +} diff --git a/pkg/util/shortid_generator.go b/pkg/util/shortid_generator.go index d87b6f70fe6..f900cb8275e 100644 --- a/pkg/util/shortid_generator.go +++ b/pkg/util/shortid_generator.go @@ -17,11 +17,7 @@ func init() { // IsValidShortUid checks if short unique identifier contains valid characters func IsValidShortUid(uid string) bool { - if !validUidPattern(uid) { - return false - } - - return true + return validUidPattern(uid) } // GenerateShortUid generates a short unique identifier. diff --git a/pkg/util/url.go b/pkg/util/url.go index c82dcef67c5..fad2d79a6d0 100644 --- a/pkg/util/url.go +++ b/pkg/util/url.go @@ -10,7 +10,7 @@ type UrlQueryReader struct { } func NewUrlQueryReader(urlInfo *url.URL) (*UrlQueryReader, error) { - u, err := url.ParseQuery(urlInfo.String()) + u, err := url.ParseQuery(urlInfo.RawQuery) if err != nil { return nil, err } diff --git a/pkg/util/url_test.go b/pkg/util/url_test.go index 4dd221b9e0b..ee29956f60d 100644 --- a/pkg/util/url_test.go +++ b/pkg/util/url_test.go @@ -4,6 +4,7 @@ import ( "testing" . "github.com/smartystreets/goconvey/convey" + "net/url" ) func TestUrl(t *testing.T) { @@ -43,4 +44,30 @@ func TestUrl(t *testing.T) { So(result, ShouldEqual, "http://localhost:8080/api/") }) + + Convey("When joining two urls where lefthand side has a trailing slash and righthand side has preceding slash", t, func() { + result := JoinUrlFragments("http://localhost:8080/", "/api/") + + So(result, ShouldEqual, "http://localhost:8080/api/") + }) +} + +func TestNewUrlQueryReader(t *testing.T) { + u, _ := url.Parse("http://www.abc.com/foo?bar=baz&bar2=baz2") + uqr, _ := NewUrlQueryReader(u) + + Convey("when trying to retrieve the first query value", t, func() { + result := uqr.Get("bar", "foodef") + So(result, ShouldEqual, "baz") + }) + + Convey("when trying to retrieve the second query value", t, func() { + result := uqr.Get("bar2", "foodef") + So(result, ShouldEqual, "baz2") + }) + + Convey("when trying to retrieve from a non-existent key, the default value is returned", t, func() { + result := uqr.Get("bar3", "foodef") + So(result, ShouldEqual, "foodef") + }) } diff --git a/pkg/util/validation_test.go b/pkg/util/validation_test.go new file mode 100644 index 00000000000..124da1b744b --- /dev/null +++ b/pkg/util/validation_test.go @@ -0,0 +1,22 @@ +package util + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestIsEmail(t *testing.T) { + + Convey("When validating a string that is a valid email", t, func() { + result := IsEmail("abc@def.com") + + So(result, ShouldEqual, true) + }) + + Convey("When validating a string that is not a valid email", t, func() { + result := IsEmail("abcdef.com") + + So(result, ShouldEqual, false) + }) +} diff --git a/public/app/app.ts b/public/app/app.ts index 9ac76b8ec91..9647fbe5416 100644 --- a/public/app/app.ts +++ b/public/app/app.ts @@ -21,15 +21,23 @@ import _ from 'lodash'; import moment from 'moment'; // add move to lodash for backward compatabiltiy -_.move = function(array, fromIndex, toIndex) { +_.move = (array, fromIndex, toIndex) => { array.splice(toIndex, 0, array.splice(fromIndex, 1)[0]); return array; }; -import { coreModule, registerAngularDirectives } from './core/core'; -import { setupAngularRoutes } from './routes/routes'; +import { coreModule, angularModules } from 'app/core/core_module'; +import { registerAngularDirectives } from 'app/core/core'; +import { setupAngularRoutes } from 'app/routes/routes'; -declare var System: any; +import 'app/routes/GrafanaCtrl'; +import 'app/features/all'; + +// import symlinked extensions +const extensionsIndex = (require as any).context('.', true, /extensions\/index.ts/); +extensionsIndex.keys().forEach(key => { + extensionsIndex(key); +}); export class GrafanaApp { registerFunctions: any; @@ -53,7 +61,7 @@ export class GrafanaApp { } init() { - var app = angular.module('grafana', []); + const app = angular.module('grafana', []); moment.locale(config.bootData.user.locale); @@ -76,9 +84,9 @@ export class GrafanaApp { $provide.decorator('$http', [ '$delegate', '$templateCache', - function($delegate, $templateCache) { - var get = $delegate.get; - $delegate.get = function(url, config) { + ($delegate, $templateCache) => { + const get = $delegate.get; + $delegate.get = (url, config) => { if (url.match(/\.html$/)) { // some template's already exist in the cache if (!$templateCache.get(url)) { @@ -105,39 +113,26 @@ export class GrafanaApp { 'react', ]; - var module_types = ['controllers', 'directives', 'factories', 'services', 'filters', 'routes']; - - _.each(module_types, type => { - var moduleName = 'grafana.' + type; - this.useModule(angular.module(moduleName, [])); - }); - // makes it possible to add dynamic stuff - this.useModule(coreModule); + _.each(angularModules, m => { + this.useModule(m); + }); // register react angular wrappers coreModule.config(setupAngularRoutes); registerAngularDirectives(); - var preBootRequires = [System.import('app/features/all')]; + // disable tool tip animation + $.fn.tooltip.defaults.animation = false; - Promise.all(preBootRequires) - .then(() => { - // disable tool tip animation - $.fn.tooltip.defaults.animation = false; - - // bootstrap the app - angular.bootstrap(document, this.ngModuleDependencies).invoke(() => { - _.each(this.preBootModules, module => { - _.extend(module, this.registerFunctions); - }); - - this.preBootModules = null; - }); - }) - .catch(function(err) { - console.log('Application boot failed:', err); + // bootstrap the app + angular.bootstrap(document, this.ngModuleDependencies).invoke(() => { + _.each(this.preBootModules, module => { + _.extend(module, this.registerFunctions); }); + + this.preBootModules = null; + }); } } diff --git a/public/app/containers/AlertRuleList/AlertRuleList.jest.tsx b/public/app/containers/AlertRuleList/AlertRuleList.jest.tsx deleted file mode 100644 index eac18a6c69d..00000000000 --- a/public/app/containers/AlertRuleList/AlertRuleList.jest.tsx +++ /dev/null @@ -1,69 +0,0 @@ -import React from 'react'; -import moment from 'moment'; -import { AlertRuleList } from './AlertRuleList'; -import { RootStore } from 'app/stores/RootStore/RootStore'; -import { backendSrv, createNavTree } from 'test/mocks/common'; -import { mount } from 'enzyme'; -import toJson from 'enzyme-to-json'; - -describe('AlertRuleList', () => { - let page, store; - - beforeAll(() => { - backendSrv.get.mockReturnValue( - Promise.resolve([ - { - id: 11, - dashboardId: 58, - panelId: 3, - name: 'Panel Title alert', - state: 'ok', - newStateDate: moment() - .subtract(5, 'minutes') - .format(), - evalData: {}, - executionError: '', - url: 'd/ufkcofof/my-goal', - canEdit: true, - }, - ]) - ); - - store = RootStore.create( - {}, - { - backendSrv: backendSrv, - navTree: createNavTree('alerting', 'alert-list'), - } - ); - - page = mount(); - }); - - it('should call api to get rules', () => { - expect(backendSrv.get.mock.calls[0][0]).toEqual('/api/alerts'); - }); - - it('should render 1 rule', () => { - page.update(); - let ruleNode = page.find('.alert-rule-item'); - expect(toJson(ruleNode)).toMatchSnapshot(); - }); - - it('toggle state should change pause rule if not paused', async () => { - backendSrv.post.mockReturnValue( - Promise.resolve({ - state: 'paused', - }) - ); - - page.find('.fa-pause').simulate('click'); - - // wait for api call to resolve - await Promise.resolve(); - page.update(); - - expect(store.alertList.rules[0].state).toBe('paused'); - expect(page.find('.fa-play')).toHaveLength(1); - }); -}); diff --git a/public/app/containers/AlertRuleList/AlertRuleList.tsx b/public/app/containers/AlertRuleList/AlertRuleList.tsx deleted file mode 100644 index 9ecb9a177d7..00000000000 --- a/public/app/containers/AlertRuleList/AlertRuleList.tsx +++ /dev/null @@ -1,175 +0,0 @@ -import React from 'react'; -import classNames from 'classnames'; -import { inject, observer } from 'mobx-react'; -import PageHeader from 'app/core/components/PageHeader/PageHeader'; -import { IAlertRule } from 'app/stores/AlertListStore/AlertListStore'; -import appEvents from 'app/core/app_events'; -import IContainerProps from 'app/containers/IContainerProps'; -import Highlighter from 'react-highlight-words'; - -@inject('view', 'nav', 'alertList') -@observer -export class AlertRuleList extends React.Component { - stateFilters = [ - { text: 'All', value: 'all' }, - { text: 'OK', value: 'ok' }, - { text: 'Not OK', value: 'not_ok' }, - { text: 'Alerting', value: 'alerting' }, - { text: 'No Data', value: 'no_data' }, - { text: 'Paused', value: 'paused' }, - ]; - - constructor(props) { - super(props); - - this.props.nav.load('alerting', 'alert-list'); - this.fetchRules(); - } - - onStateFilterChanged = evt => { - this.props.view.updateQuery({ state: evt.target.value }); - this.fetchRules(); - }; - - fetchRules() { - this.props.alertList.loadRules({ - state: this.props.view.query.get('state') || 'all', - }); - } - - onOpenHowTo = () => { - appEvents.emit('show-modal', { - src: 'public/app/features/alerting/partials/alert_howto.html', - modalClass: 'confirm-modal', - model: {}, - }); - }; - - onSearchQueryChange = evt => { - this.props.alertList.setSearchQuery(evt.target.value); - }; - - render() { - const { nav, alertList } = this.props; - - return ( -
- -
-
-
- -
-
- - -
- -
-
- - - -
-
    - {alertList.filteredRules.map(rule => ( - - ))} -
-
-
-
- ); - } -} - -function AlertStateFilterOption({ text, value }) { - return ( - - ); -} - -export interface AlertRuleItemProps { - rule: IAlertRule; - search: string; -} - -@observer -export class AlertRuleItem extends React.Component { - toggleState = () => { - this.props.rule.togglePaused(); - }; - - renderText(text: string) { - return ( - - ); - } - - render() { - const { rule } = this.props; - - let stateClass = classNames({ - fa: true, - 'fa-play': rule.isPaused, - 'fa-pause': !rule.isPaused, - }); - - let ruleUrl = `${rule.url}?panelId=${rule.panelId}&fullscreen=true&edit=true&tab=alert`; - - return ( -
  • - - - -
    -
    - -
    - {this.renderText(rule.stateText)} - for {rule.stateAge} -
    -
    - {rule.info &&
    {this.renderText(rule.info)}
    } -
    - -
    - - - - -
    -
    • - ); - } -} diff --git a/public/app/containers/IContainerProps.ts b/public/app/containers/IContainerProps.ts deleted file mode 100644 index 6e790cee06d..00000000000 --- a/public/app/containers/IContainerProps.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { SearchStore } from './../stores/SearchStore/SearchStore'; -import { ServerStatsStore } from './../stores/ServerStatsStore/ServerStatsStore'; -import { NavStore } from './../stores/NavStore/NavStore'; -import { PermissionsStore } from './../stores/PermissionsStore/PermissionsStore'; -import { AlertListStore } from './../stores/AlertListStore/AlertListStore'; -import { ViewStore } from './../stores/ViewStore/ViewStore'; -import { FolderStore } from './../stores/FolderStore/FolderStore'; - -interface IContainerProps { - search: typeof SearchStore.Type; - serverStats: typeof ServerStatsStore.Type; - nav: typeof NavStore.Type; - alertList: typeof AlertListStore.Type; - permissions: typeof PermissionsStore.Type; - view: typeof ViewStore.Type; - folder: typeof FolderStore.Type; - backendSrv: any; -} - -export default IContainerProps; diff --git a/public/app/containers/ManageDashboards/FolderPermissions.tsx b/public/app/containers/ManageDashboards/FolderPermissions.tsx deleted file mode 100644 index 9c82db1c18c..00000000000 --- a/public/app/containers/ManageDashboards/FolderPermissions.tsx +++ /dev/null @@ -1,74 +0,0 @@ -import React, { Component } from 'react'; -import { inject, observer } from 'mobx-react'; -import { toJS } from 'mobx'; -import IContainerProps from 'app/containers/IContainerProps'; -import PageHeader from 'app/core/components/PageHeader/PageHeader'; -import Permissions from 'app/core/components/Permissions/Permissions'; -import Tooltip from 'app/core/components/Tooltip/Tooltip'; -import PermissionsInfo from 'app/core/components/Permissions/PermissionsInfo'; -import AddPermissions from 'app/core/components/Permissions/AddPermissions'; -import SlideDown from 'app/core/components/Animations/SlideDown'; - -@inject('nav', 'folder', 'view', 'permissions') -@observer -export class FolderPermissions extends Component { - constructor(props) { - super(props); - this.handleAddPermission = this.handleAddPermission.bind(this); - this.loadStore(); - } - - componentWillUnmount() { - const { permissions } = this.props; - permissions.hideAddPermissions(); - } - - loadStore() { - const { nav, folder, view } = this.props; - return folder.load(view.routeParams.get('uid') as string).then(res => { - view.updatePathAndQuery(`${res.url}/permissions`, {}, {}); - return nav.initFolderNav(toJS(folder.folder), 'manage-folder-permissions'); - }); - } - - handleAddPermission() { - const { permissions } = this.props; - permissions.toggleAddPermissions(); - } - - render() { - const { nav, folder, permissions, backendSrv } = this.props; - - if (!folder.folder || !nav.main) { - return

    Loading

    ; - } - - const dashboardId = folder.folder.id; - - return ( -
    - -
    -
    -

    Folder Permissions

    - - - -
    - -
    - - - - -
    -
    - ); - } -} diff --git a/public/app/containers/ManageDashboards/FolderSettings.jest.tsx b/public/app/containers/ManageDashboards/FolderSettings.jest.tsx deleted file mode 100644 index bed3d569bcc..00000000000 --- a/public/app/containers/ManageDashboards/FolderSettings.jest.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import React from 'react'; -import { FolderSettings } from './FolderSettings'; -import { RootStore } from 'app/stores/RootStore/RootStore'; -import { backendSrv } from 'test/mocks/common'; -import { shallow } from 'enzyme'; - -describe('FolderSettings', () => { - let wrapper; - let page; - - beforeAll(() => { - backendSrv.getFolderByUid.mockReturnValue( - Promise.resolve({ - id: 1, - uid: 'uid', - title: 'Folder Name', - url: '/dashboards/f/uid/folder-name', - canSave: true, - version: 1, - }) - ); - - const store = RootStore.create( - { - view: { - path: 'asd', - query: {}, - routeParams: { - uid: 'uid-str', - }, - }, - }, - { - backendSrv: backendSrv, - } - ); - - wrapper = shallow(); - page = wrapper.dive(); - return page - .instance() - .loadStore() - .then(() => { - page.update(); - }); - }); - - it('should set the title input field', () => { - const titleInput = page.find('.gf-form-input'); - expect(titleInput).toHaveLength(1); - expect(titleInput.prop('value')).toBe('Folder Name'); - }); - - it('should update title and enable save button when changed', () => { - const titleInput = page.find('.gf-form-input'); - const disabledSubmitButton = page.find('button[type="submit"]'); - expect(disabledSubmitButton.prop('disabled')).toBe(true); - - titleInput.simulate('change', { target: { value: 'New Title' } }); - - const updatedTitleInput = page.find('.gf-form-input'); - expect(updatedTitleInput.prop('value')).toBe('New Title'); - const enabledSubmitButton = page.find('button[type="submit"]'); - expect(enabledSubmitButton.prop('disabled')).toBe(false); - }); - - it('should disable save button if title is changed back to old title', () => { - const titleInput = page.find('.gf-form-input'); - - titleInput.simulate('change', { target: { value: 'Folder Name' } }); - - const enabledSubmitButton = page.find('button[type="submit"]'); - expect(enabledSubmitButton.prop('disabled')).toBe(true); - }); - - it('should disable save button if title is changed to empty string', () => { - const titleInput = page.find('.gf-form-input'); - - titleInput.simulate('change', { target: { value: '' } }); - - const enabledSubmitButton = page.find('button[type="submit"]'); - expect(enabledSubmitButton.prop('disabled')).toBe(true); - }); -}); diff --git a/public/app/containers/ManageDashboards/FolderSettings.tsx b/public/app/containers/ManageDashboards/FolderSettings.tsx deleted file mode 100644 index 586a8f05b4c..00000000000 --- a/public/app/containers/ManageDashboards/FolderSettings.tsx +++ /dev/null @@ -1,158 +0,0 @@ -import React from 'react'; -import { inject, observer } from 'mobx-react'; -import { toJS } from 'mobx'; -import PageHeader from 'app/core/components/PageHeader/PageHeader'; -import IContainerProps from 'app/containers/IContainerProps'; -import { getSnapshot } from 'mobx-state-tree'; -import appEvents from 'app/core/app_events'; - -@inject('nav', 'folder', 'view') -@observer -export class FolderSettings extends React.Component { - formSnapshot: any; - - constructor(props) { - super(props); - this.loadStore(); - } - - loadStore() { - const { nav, folder, view } = this.props; - - return folder.load(view.routeParams.get('uid') as string).then(res => { - this.formSnapshot = getSnapshot(folder); - view.updatePathAndQuery(`${res.url}/settings`, {}, {}); - - return nav.initFolderNav(toJS(folder.folder), 'manage-folder-settings'); - }); - } - - onTitleChange(evt) { - this.props.folder.setTitle(this.getFormSnapshot().folder.title, evt.target.value); - } - - getFormSnapshot() { - if (!this.formSnapshot) { - this.formSnapshot = getSnapshot(this.props.folder); - } - - return this.formSnapshot; - } - - save(evt) { - if (evt) { - evt.stopPropagation(); - evt.preventDefault(); - } - - const { nav, folder, view } = this.props; - - folder - .saveFolder({ overwrite: false }) - .then(newUrl => { - view.updatePathAndQuery(newUrl, {}, {}); - - appEvents.emit('dashboard-saved'); - appEvents.emit('alert-success', ['Folder saved']); - }) - .then(() => { - return nav.initFolderNav(toJS(folder.folder), 'manage-folder-settings'); - }) - .catch(this.handleSaveFolderError.bind(this)); - } - - delete(evt) { - if (evt) { - evt.stopPropagation(); - evt.preventDefault(); - } - - const { folder, view } = this.props; - const title = folder.folder.title; - - appEvents.emit('confirm-modal', { - title: 'Delete', - text: `Do you want to delete this folder and all its dashboards?`, - icon: 'fa-trash', - yesText: 'Delete', - onConfirm: () => { - return folder.deleteFolder().then(() => { - appEvents.emit('alert-success', ['Folder Deleted', `${title} has been deleted`]); - view.updatePathAndQuery('dashboards', '', ''); - }); - }, - }); - } - - handleSaveFolderError(err) { - if (err.data && err.data.status === 'version-mismatch') { - err.isHandled = true; - - const { nav, folder, view } = this.props; - - appEvents.emit('confirm-modal', { - title: 'Conflict', - text: 'Someone else has updated this folder.', - text2: 'Would you still like to save this folder?', - yesText: 'Save & Overwrite', - icon: 'fa-warning', - onConfirm: () => { - folder - .saveFolder({ overwrite: true }) - .then(newUrl => { - view.updatePathAndQuery(newUrl, {}, {}); - - appEvents.emit('dashboard-saved'); - appEvents.emit('alert-success', ['Folder saved']); - }) - .then(() => { - return nav.initFolderNav(toJS(folder.folder), 'manage-folder-settings'); - }); - }, - }); - } - } - - render() { - const { nav, folder } = this.props; - - if (!folder.folder || !nav.main) { - return

    Loading

    ; - } - - return ( -
    - -
    -

    Folder Settings

    - -
    -
    -
    - - -
    -
    - - -
    - -
    -
    -
    - ); - } -} diff --git a/public/app/containers/ServerStats/ServerStats.jest.tsx b/public/app/containers/ServerStats/ServerStats.jest.tsx deleted file mode 100644 index a329a47527d..00000000000 --- a/public/app/containers/ServerStats/ServerStats.jest.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import React from 'react'; -import renderer from 'react-test-renderer'; -import { ServerStats } from './ServerStats'; -import { RootStore } from 'app/stores/RootStore/RootStore'; -import { backendSrv, createNavTree } from 'test/mocks/common'; - -describe('ServerStats', () => { - it('Should render table with stats', done => { - backendSrv.get.mockReturnValue( - Promise.resolve({ - dashboards: 10, - }) - ); - - const store = RootStore.create( - {}, - { - backendSrv: backendSrv, - navTree: createNavTree('cfg', 'admin', 'server-stats'), - } - ); - - const page = renderer.create(); - - setTimeout(() => { - expect(page.toJSON()).toMatchSnapshot(); - done(); - }); - }); -}); diff --git a/public/app/containers/ServerStats/ServerStats.tsx b/public/app/containers/ServerStats/ServerStats.tsx deleted file mode 100644 index e40b441d967..00000000000 --- a/public/app/containers/ServerStats/ServerStats.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import React from 'react'; -import { inject, observer } from 'mobx-react'; -import PageHeader from 'app/core/components/PageHeader/PageHeader'; -import IContainerProps from 'app/containers/IContainerProps'; - -@inject('nav', 'serverStats') -@observer -export class ServerStats extends React.Component { - constructor(props) { - super(props); - const { nav, serverStats } = this.props; - - nav.load('cfg', 'admin', 'server-stats'); - serverStats.load(); - } - - render() { - const { nav, serverStats } = this.props; - return ( -
    - -
    - - - - - - - - {serverStats.stats.map(StatItem)} -
    NameValue
    -
    -
    - ); - } -} - -function StatItem(stat) { - return ( - - {stat.name} - {stat.value} - - ); -} diff --git a/public/app/core/actions/appNotification.ts b/public/app/core/actions/appNotification.ts new file mode 100644 index 00000000000..b79b642eef1 --- /dev/null +++ b/public/app/core/actions/appNotification.ts @@ -0,0 +1,28 @@ +import { AppNotification } from 'app/types/'; + +export enum ActionTypes { + AddAppNotification = 'ADD_APP_NOTIFICATION', + ClearAppNotification = 'CLEAR_APP_NOTIFICATION', +} + +interface AddAppNotificationAction { + type: ActionTypes.AddAppNotification; + payload: AppNotification; +} + +interface ClearAppNotificationAction { + type: ActionTypes.ClearAppNotification; + payload: number; +} + +export type Action = AddAppNotificationAction | ClearAppNotificationAction; + +export const clearAppNotification = (appNotificationId: number) => ({ + type: ActionTypes.ClearAppNotification, + payload: appNotificationId, +}); + +export const notifyApp = (appNotification: AppNotification) => ({ + type: ActionTypes.AddAppNotification, + payload: appNotification, +}); diff --git a/public/app/core/actions/index.ts b/public/app/core/actions/index.ts new file mode 100644 index 00000000000..f7ce2dda945 --- /dev/null +++ b/public/app/core/actions/index.ts @@ -0,0 +1,5 @@ +import { updateLocation } from './location'; +import { updateNavIndex, UpdateNavIndexAction } from './navModel'; +import { notifyApp, clearAppNotification } from './appNotification'; + +export { updateLocation, updateNavIndex, UpdateNavIndexAction, notifyApp, clearAppNotification }; diff --git a/public/app/core/actions/location.ts b/public/app/core/actions/location.ts new file mode 100644 index 00000000000..6f7ac67363e --- /dev/null +++ b/public/app/core/actions/location.ts @@ -0,0 +1,13 @@ +import { LocationUpdate } from 'app/types'; + +export type Action = UpdateLocationAction; + +export interface UpdateLocationAction { + type: 'UPDATE_LOCATION'; + payload: LocationUpdate; +} + +export const updateLocation = (location: LocationUpdate): UpdateLocationAction => ({ + type: 'UPDATE_LOCATION', + payload: location, +}); diff --git a/public/app/core/actions/navModel.ts b/public/app/core/actions/navModel.ts new file mode 100644 index 00000000000..a40a0e880ee --- /dev/null +++ b/public/app/core/actions/navModel.ts @@ -0,0 +1,17 @@ +import { NavModelItem } from '../../types'; + +export enum ActionTypes { + UpdateNavIndex = 'UPDATE_NAV_INDEX', +} + +export type Action = UpdateNavIndexAction; + +export interface UpdateNavIndexAction { + type: ActionTypes.UpdateNavIndex; + payload: NavModelItem; +} + +export const updateNavIndex = (item: NavModelItem): UpdateNavIndexAction => ({ + type: ActionTypes.UpdateNavIndex, + payload: item, +}); diff --git a/public/app/core/angular_wrappers.ts b/public/app/core/angular_wrappers.ts index ace0eb00b07..7be28272f11 100644 --- a/public/app/core/angular_wrappers.ts +++ b/public/app/core/angular_wrappers.ts @@ -2,23 +2,21 @@ import { react2AngularDirective } from 'app/core/utils/react2angular'; import { PasswordStrength } from './components/PasswordStrength'; import PageHeader from './components/PageHeader/PageHeader'; import EmptyListCTA from './components/EmptyListCTA/EmptyListCTA'; -import LoginBackground from './components/Login/LoginBackground'; import { SearchResult } from './components/search/SearchResult'; import { TagFilter } from './components/TagFilter/TagFilter'; -import UserPicker from './components/Picker/UserPicker'; -import DashboardPermissions from './components/Permissions/DashboardPermissions'; +import { SideMenu } from './components/sidemenu/SideMenu'; +import AppNotificationList from './components/AppNotifications/AppNotificationList'; export function registerAngularDirectives() { react2AngularDirective('passwordStrength', PasswordStrength, ['password']); + react2AngularDirective('sidemenu', SideMenu, []); + react2AngularDirective('appNotificationsList', AppNotificationList, []); react2AngularDirective('pageHeader', PageHeader, ['model', 'noTabs']); react2AngularDirective('emptyListCta', EmptyListCTA, ['model']); - react2AngularDirective('loginBackground', LoginBackground, []); react2AngularDirective('searchResult', SearchResult, []); react2AngularDirective('tagFilter', TagFilter, [ 'tags', ['onSelect', { watchDepth: 'reference' }], ['tagOptions', { watchDepth: 'reference' }], ]); - react2AngularDirective('selectUserPicker', UserPicker, ['backendSrv', 'handlePicked']); - react2AngularDirective('dashboardPermissions', DashboardPermissions, ['backendSrv', 'dashboardId', 'folder']); } diff --git a/public/app/core/app_events.ts b/public/app/core/app_events.ts index 26dd74bcb00..6af7913167b 100644 --- a/public/app/core/app_events.ts +++ b/public/app/core/app_events.ts @@ -1,4 +1,4 @@ import { Emitter } from './utils/emitter'; -var appEvents = new Emitter(); +const appEvents = new Emitter(); export default appEvents; diff --git a/public/app/core/components/Animations/SlideDown.tsx b/public/app/core/components/Animations/SlideDown.tsx index 4d515f98f16..70dacd73849 100644 --- a/public/app/core/components/Animations/SlideDown.tsx +++ b/public/app/core/components/Animations/SlideDown.tsx @@ -1,15 +1,22 @@ -import React from 'react'; +import React from 'react'; import Transition from 'react-transition-group/Transition'; -const defaultMaxHeight = '200px'; // When animating using max-height we need to use a static value. +interface Style { + transition?: string; + overflow?: string; +} + +// When animating using max-height we need to use a static value. // If this is not enough, pass in + +
    +
    +
    {appNotification.title}
    +
    {appNotification.text}
    +
    + +
    + ); + } +} diff --git a/public/app/core/components/AppNotifications/AppNotificationList.tsx b/public/app/core/components/AppNotifications/AppNotificationList.tsx new file mode 100644 index 00000000000..c91f8372384 --- /dev/null +++ b/public/app/core/components/AppNotifications/AppNotificationList.tsx @@ -0,0 +1,60 @@ +import React, { PureComponent } from 'react'; +import appEvents from 'app/core/app_events'; +import AppNotificationItem from './AppNotificationItem'; +import { notifyApp, clearAppNotification } from 'app/core/actions'; +import { connectWithStore } from 'app/core/utils/connectWithReduxStore'; +import { AppNotification, StoreState } from 'app/types'; +import { + createErrorNotification, + createSuccessNotification, + createWarningNotification, +} from '../../copy/appNotification'; + +export interface Props { + appNotifications: AppNotification[]; + notifyApp: typeof notifyApp; + clearAppNotification: typeof clearAppNotification; +} + +export class AppNotificationList extends PureComponent { + componentDidMount() { + const { notifyApp } = this.props; + + appEvents.on('alert-warning', options => notifyApp(createWarningNotification(options[0], options[1]))); + appEvents.on('alert-success', options => notifyApp(createSuccessNotification(options[0], options[1]))); + appEvents.on('alert-error', options => notifyApp(createErrorNotification(options[0], options[1]))); + } + + onClearAppNotification = id => { + this.props.clearAppNotification(id); + }; + + render() { + const { appNotifications } = this.props; + + return ( +
    + {appNotifications.map((appNotification, index) => { + return ( + this.onClearAppNotification(id)} + /> + ); + })} +
    + ); + } +} + +const mapStateToProps = (state: StoreState) => ({ + appNotifications: state.appNotifications.appNotifications, +}); + +const mapDispatchToProps = { + notifyApp, + clearAppNotification, +}; + +export default connectWithStore(AppNotificationList, mapStateToProps, mapDispatchToProps); diff --git a/public/app/core/components/CustomScrollbar/CustomScrollbar.test.tsx b/public/app/core/components/CustomScrollbar/CustomScrollbar.test.tsx new file mode 100644 index 00000000000..4edcf7313db --- /dev/null +++ b/public/app/core/components/CustomScrollbar/CustomScrollbar.test.tsx @@ -0,0 +1,16 @@ +import React from 'react'; +import renderer from 'react-test-renderer'; +import CustomScrollbar from './CustomScrollbar'; + +describe('CustomScrollbar', () => { + it('renders correctly', () => { + const tree = renderer + .create( + +

    Scrollable content

    +     + ) + .toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/public/app/core/components/CustomScrollbar/CustomScrollbar.tsx b/public/app/core/components/CustomScrollbar/CustomScrollbar.tsx new file mode 100644 index 00000000000..9b9a9c4d02a --- /dev/null +++ b/public/app/core/components/CustomScrollbar/CustomScrollbar.tsx @@ -0,0 +1,46 @@ +import React, { PureComponent } from 'react'; +import Scrollbars from 'react-custom-scrollbars'; + +interface Props { + customClassName?: string; + autoHide?: boolean; + autoHideTimeout?: number; + autoHideDuration?: number; + hideTracksWhenNotNeeded?: boolean; +} + +/** + * Wraps component into component from `react-custom-scrollbars` + */ +class CustomScrollbar extends PureComponent { + static defaultProps: Partial = { + customClassName: 'custom-scrollbars', + autoHide: true, + autoHideTimeout: 200, + autoHideDuration: 200, + hideTracksWhenNotNeeded: false, + }; + + render() { + const { customClassName, children, ...scrollProps } = this.props; + + return ( +
    } + renderTrackVertical={props =>
    } + renderThumbHorizontal={props =>
    } + renderThumbVertical={props =>
    } + renderView={props =>
    } + {...scrollProps} + > + {children} + + ); + } +} + +export default CustomScrollbar; diff --git a/public/app/core/components/CustomScrollbar/__snapshots__/CustomScrollbar.test.tsx.snap b/public/app/core/components/CustomScrollbar/__snapshots__/CustomScrollbar.test.tsx.snap new file mode 100644 index 00000000000..37d8cea45be --- /dev/null +++ b/public/app/core/components/CustomScrollbar/__snapshots__/CustomScrollbar.test.tsx.snap @@ -0,0 +1,86 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CustomScrollbar renders correctly 1`] = ` +
    +
    +

    + Scrollable content +

    +
    +
    +
    +
    +
    +
    +
    +
    +`; diff --git a/public/app/core/components/DeleteButton/DeleteButton.test.tsx b/public/app/core/components/DeleteButton/DeleteButton.test.tsx new file mode 100644 index 00000000000..12acadee18a --- /dev/null +++ b/public/app/core/components/DeleteButton/DeleteButton.test.tsx @@ -0,0 +1,44 @@ +import React from 'react'; +import DeleteButton from './DeleteButton'; +import { shallow } from 'enzyme'; + +describe('DeleteButton', () => { + let wrapper; + let deleted; + + beforeAll(() => { + deleted = false; + + function deleteItem() { + deleted = true; + } + wrapper = shallow( deleteItem()} />); + }); + + it('should show confirm delete when clicked', () => { + expect(wrapper.state().showConfirm).toBe(false); + wrapper.find('.delete-button').simulate('click'); + expect(wrapper.state().showConfirm).toBe(true); + }); + + it('should hide confirm delete when clicked', () => { + wrapper.find('.delete-button').simulate('click'); + expect(wrapper.state().showConfirm).toBe(true); + wrapper + .find('.confirm-delete') + .find('.btn') + .at(0) + .simulate('click'); + expect(wrapper.state().showConfirm).toBe(false); + }); + + it('should show confirm delete when clicked', () => { + expect(deleted).toBe(false); + wrapper + .find('.confirm-delete') + .find('.btn') + .at(1) + .simulate('click'); + expect(deleted).toBe(true); + }); +}); diff --git a/public/app/core/components/DeleteButton/DeleteButton.tsx b/public/app/core/components/DeleteButton/DeleteButton.tsx new file mode 100644 index 00000000000..a83ce6097ad --- /dev/null +++ b/public/app/core/components/DeleteButton/DeleteButton.tsx @@ -0,0 +1,66 @@ +import React, { PureComponent } from 'react'; + +export interface DeleteButtonProps { + onConfirmDelete(); +} + +export interface DeleteButtonStates { + showConfirm: boolean; +} + +export default class DeleteButton extends PureComponent { + state: DeleteButtonStates = { + showConfirm: false, + }; + + onClickDelete = event => { + if (event) { + event.preventDefault(); + } + + this.setState({ + showConfirm: true, + }); + }; + + onClickCancel = event => { + if (event) { + event.preventDefault(); + } + this.setState({ + showConfirm: false, + }); + }; + + render() { + const onClickConfirm = this.props.onConfirmDelete; + let showConfirm; + let showDeleteButton; + + if (this.state.showConfirm) { + showConfirm = 'show'; + showDeleteButton = 'hide'; + } else { + showConfirm = 'hide'; + showDeleteButton = 'show'; + } + + return ( + + + + + + + + Cancel + + + Confirm Delete + + + + + ); + } +} diff --git a/public/app/core/components/EmptyListCTA/EmptyListCTA.jest.tsx b/public/app/core/components/EmptyListCTA/EmptyListCTA.test.tsx similarity index 96% rename from public/app/core/components/EmptyListCTA/EmptyListCTA.jest.tsx rename to public/app/core/components/EmptyListCTA/EmptyListCTA.test.tsx index 4af60f3c839..21700bb4d03 100644 --- a/public/app/core/components/EmptyListCTA/EmptyListCTA.jest.tsx +++ b/public/app/core/components/EmptyListCTA/EmptyListCTA.test.tsx @@ -7,6 +7,7 @@ const model = { buttonIcon: 'ga css class', buttonLink: 'http://url/to/destination', buttonTitle: 'Click me', + onClick: jest.fn(), proTip: 'This is a tip', proTipLink: 'http://url/to/tip/destination', proTipLinkTitle: 'Learn more', diff --git a/public/app/core/components/EmptyListCTA/EmptyListCTA.tsx b/public/app/core/components/EmptyListCTA/EmptyListCTA.tsx index 1583303dfa1..ae0e39cc26d 100644 --- a/public/app/core/components/EmptyListCTA/EmptyListCTA.tsx +++ b/public/app/core/components/EmptyListCTA/EmptyListCTA.tsx @@ -1,34 +1,38 @@ import React, { Component } from 'react'; -export interface IProps { - model: any; +export interface Props { + model: any; } -class EmptyListCTA extends Component { - render() { - const { - title, - buttonIcon, - buttonLink, - buttonTitle, - proTip, - proTipLink, - proTipLinkTitle, - proTipTarget - } = this.props.model; - return ( -
    -
    {title}
    - {buttonTitle} -
    - ProTip: {proTip} - {proTipLinkTitle} -
    -
    - ); - } +class EmptyListCTA extends Component { + render() { + const { + title, + buttonIcon, + buttonLink, + buttonTitle, + onClick, + proTip, + proTipLink, + proTipLinkTitle, + proTipTarget, + } = this.props.model; + return ( +
    +
    {title}
    + + + {buttonTitle} + +
    + ProTip: {proTip} + + {proTipLinkTitle} + +
    +
    + ); + } } export default EmptyListCTA; diff --git a/public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.jest.tsx.snap b/public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.test.tsx.snap similarity index 95% rename from public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.jest.tsx.snap rename to public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.test.tsx.snap index 6d47c984d5e..b85660bcc6f 100644 --- a/public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.jest.tsx.snap +++ b/public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.test.tsx.snap @@ -12,6 +12,7 @@ exports[`EmptyListCTA renders correctly 1`] = ` = props => { + return ( + + {props.children} + {props.tooltip && ( + + + + )} + + ); +}; diff --git a/public/app/core/components/LayoutSelector/LayoutSelector.tsx b/public/app/core/components/LayoutSelector/LayoutSelector.tsx new file mode 100644 index 00000000000..d9e00102438 --- /dev/null +++ b/public/app/core/components/LayoutSelector/LayoutSelector.tsx @@ -0,0 +1,39 @@ +import React, { SFC } from 'react'; + +export type LayoutMode = LayoutModes.Grid | LayoutModes.List; + +export enum LayoutModes { + Grid = 'grid', + List = 'list', +} + +interface Props { + mode: LayoutMode; + onLayoutModeChanged: (mode: LayoutMode) => {}; +} + +const LayoutSelector: SFC = props => { + const { mode, onLayoutModeChanged } = props; + return ( +
    + + +
    + ); +}; + +export default LayoutSelector; diff --git a/public/app/core/components/Login/LoginBackground.tsx b/public/app/core/components/Login/LoginBackground.tsx deleted file mode 100644 index 83e228ab6e0..00000000000 --- a/public/app/core/components/Login/LoginBackground.tsx +++ /dev/null @@ -1,1240 +0,0 @@ -import React, { Component } from 'react'; - -const xCount = 50; -const yCount = 50; - -function Cell({ x, y, flipIndex }) { - const index = (y * xCount) + x; - const bgColor1 = getColor(x, y); - return ( -
    - ); -} - -function getRandomInt(min, max) { - min = Math.ceil(min); - max = Math.floor(max); - return Math.floor(Math.random() * (max - min)) + min; //The maximum is exclusive and the minimum is inclusive -} - -export default class LoginBackground extends Component { - cancelInterval: any; - - constructor(props) { - super(props); - - this.state = { - flipIndex: null, - }; - - this.flipElements = this.flipElements.bind(this); - } - - flipElements() { - const elementIndexToFlip = getRandomInt(0, (xCount * yCount) - 1); - this.setState(prevState => { - return { - ...prevState, - flipIndex: elementIndexToFlip, - }; - }); - } - - componentWillMount() { - this.cancelInterval = setInterval(this.flipElements, 3000); - } - - componentWillUnmount() { - clearInterval(this.cancelInterval); - } - - render() { - console.log('re-render!', this.state.flipIndex); - - return ( -
    - {Array.from(Array(yCount)).map((el, y) => { - return ( -
    - {Array.from(Array(xCount)).map((el2, x) => { - return ( - - ); - })} -
    - ); - })} -
    - ); - } -} - -function getColor(x, y) { - const colors = [ - '#14161A', - '#111920', - '#121E27', - '#13212B', - '#122029', - '#101C24', - '#0F1B23', - '#0F1B22', - '#111C24', - '#101A22', - '#101A21', - '#111D25', - '#101E27', - '#101D26', - '#101B23', - '#11191E', - '#131519', - '#131518', - '#101B21', - '#121F29', - '#10232D', - '#11212B', - '#0E1C25', - '#0E1C24', - '#111F29', - '#11222B', - '#101E28', - '#102028', - '#111F2A', - '#11202A', - '#11191F', - '#121417', - '#12191D', - '#101D25', - '#11212C', - '#10242F', - '#0F212B', - '#0F1E27', - '#0F1D26', - '#0F1F29', - '#0F2029', - '#11232E', - '#10212B', - '#10222C', - '#0F202A', - '#112530', - '#10252F', - '#0F242E', - '#10222D', - '#10202A', - '#0F1C24', - '#0F1E28', - '#0F212A', - '#0F222B', - '#14171A', - '#0F1A20', - '#0F1C25', - '#10232E', - '#0E202A', - '#0E1E27', - '#0E1D26', - '#0F202B', - '#11232F', - '#102632', - '#102530', - '#122430', - '#0F1B21', - '#0F212C', - '#0E1F29', - '#112531', - '#0F2734', - '#0F2835', - '#0D1B23', - '#0F1A21', - '#0F1A23', - '#0F1D27', - '#0F222D', - '#102430', - '#102531', - '#10222E', - '#0F232D', - '#0E2633', - '#0E2734', - '#0F2834', - '#0E2835', - '#0F2633', - '#0F2532', - '#0E1A22', - '#0D1C24', - '#0F2735', - '#0F2937', - '#102A38', - '#112938', - '#102A39', - '#0F2A38', - '#102836', - '#0E1B23', - '#0F2938', - '#102A3A', - '#102D3D', - '#0F3040', - '#102D3E', - '#0F2E3E', - '#112C3B', - '#102B3B', - '#102B3A', - '#102D3C', - '#0F2A39', - '#0F2634', - '#0E2029', - '#0E1A21', - '#0F2B39', - '#0F2D3D', - '#0F2F40', - '#0E3142', - '#113445', - '#122431', - '#102E3E', - '#0F3345', - '#0E2F40', - '#0F3143', - '#102C3C', - '#0F2B3A', - '#0F1F28', - '#0F3344', - '#113548', - '#113C51', - '#144258', - '#103A4E', - '#103A4F', - '#103547', - '#10364A', - '#103649', - '#0F3448', - '#102C3A', - '#0F2836', - '#103447', - '#0F384C', - '#123F55', - '#15445A', - '#133F55', - '#103B50', - '#113E54', - '#103446', - '#0F3A4F', - '#0F3548', - '#0D3142', - '#102C3B', - '#0E2937', - '#103D52', - '#0E3544', - '#184C65', - '#154760', - '#14435B', - '#15465F', - '#124159', - '#0F3D53', - '#103C51', - '#0F3447', - '#0E3243', - '#113143', - '#113D53', - '#184B64', - '#184D67', - '#184C66', - '#174A63', - '#15455C', - '#13425A', - '#14445A', - '#10384C', - '#0E3446', - '#10181E', - '#103243', - '#0F384D', - '#14455C', - '#164761', - '#164C66', - '#1D627D', - '#12425A', - '#164A63', - '#14465D', - '#13435A', - '#0A2B38', - '#0F3446', - '#0D2F40', - '#0D2F3F', - '#0F2531', - '#102937', - '#10384B', - '#0F3649', - '#184E68', - '#1A5472', - '#184D68', - '#154A63', - '#19506B', - '#19536F', - '#1A4F69', - '#144760', - '#114058', - '#0E3A4F', - '#0E3547', - '#0C3042', - '#0E1B24', - '#11222C', - '#154C65', - '#1A5776', - '#1B5675', - '#113847', - '#1A5371', - '#194E68', - '#0E2D3D', - '#112D3B', - '#113D52', - '#18516D', - '#1A5979', - '#1B5878', - '#19526E', - '#1A526E', - '#13435B', - '#0F3E55', - '#0B374C', - '#0E3448', - '#0D2E3F', - '#0F2B3B', - '#112E3E', - '#113B50', - '#15465D', - '#1A526F', - '#1E5E81', - '#1D5B7B', - '#1A5777', - '#154456', - '#113949', - '#0D394E', - '#0F3549', - '#0F2C3B', - '#0E2733', - '#112E3D', - '#123D52', - '#10394C', - '#1B5674', - '#1A5370', - '#144861', - '#104058', - '#104159', - '#0E384C', - '#0D2D3D', - '#0E2533', - '#112C3A', - '#1B5979', - '#1B5C7D', - '#1A5675', - '#104057', - '#0F3C51', - '#11425A', - '#0E394D', - '#0C3243', - '#0E2735', - '#112F3E', - '#134158', - '#1D5E7F', - '#1D6083', - '#1C5877', - '#1A5573', - '#184D66', - '#164962', - '#0F3D54', - '#0E3D53', - '#0E3447', - '#0F2A3A', - '#0F2936', - '#101F28', - '#103040', - '#124056', - '#164E69', - '#144B64', - '#164D66', - '#0F3E54', - '#0E3B51', - '#0D3346', - '#0E1F27', - '#124158', - '#164961', - '#0E3C52', - '#19506C', - '#0F2C3C', - '#0E3244', - '#0E2A39', - '#0E2938', - '#113040', - '#134057', - '#1A5471', - '#154B63', - '#1C597A', - '#164760', - '#10374B', - '#0E374C', - '#0E384D', - '#11242F', - '#10394D', - '#18526E', - '#154B65', - '#103F55', - '#0D3345', - '#102532', - '#102029', - '#113142', - '#1B5973', - '#1A516B', - '#1C5979', - '#1C5A7A', - '#184A65', - '#164C65', - '#0D3041', - '#123142', - '#123E54', - '#1B5877', - '#1A5574', - '#1C5878', - '#13435C', - '#0F374B', - '#0C3143', - '#112F40', - '#123C51', - '#174E68', - '#1D5C7D', - '#14465F', - '#0F3F56', - '#0B3041', - '#123243', - '#15435B', - '#19516D', - '#1D5D7E', - '#1C5C7D', - '#184F69', - '#11374B', - '#103E54', - '#0E3143', - '#0F2D3C', - '#11242E', - '#133445', - '#1A5674', - '#1D6184', - '#1F658B', - '#0D3A50', - '#0C374B', - '#154862', - '#164B64', - '#154961', - '#0D384D', - '#102631', - '#113242', - '#134259', - '#185270', - '#1D6386', - '#1E678C', - '#1C5978', - '#0D3549', - '#0F2632', - '#184961', - '#1D5E80', - '#1E6488', - '#1F678D', - '#1E5B7C', - '#164862', - '#19526D', - '#113C52', - '#15455E', - '#0F2F3F', - '#144259', - '#194D67', - '#1D6991', - '#195777', - '#19516C', - '#103F56', - '#144660', - '#0D2E3E', - '#10212A', - '#113141', - '#16455C', - '#1D5B7C', - '#1F6589', - '#1E668C', - '#1E5F81', - '#0F3B50', - '#0D3244', - '#164A64', - '#184E69', - '#0E364A', - '#0E2E3E', - '#10222B', - '#19475E', - '#1B5A7B', - '#1E5D7F', - '#1E678D', - '#1E6184', - '#19506A', - '#1B5370', - '#1B5573', - '#0E3041', - '#122E3E', - '#16455B', - '#195370', - '#1D6489', - '#1D6B93', - '#164A65', - '#154A64', - '#1A5572', - '#1D6082', - '#1F6286', - '#1D6C94', - '#1E709A', - '#174A65', - '#1B526F', - '#1E6589', - '#1D6384', - '#0D3143', - '#0E2F3F', - '#174760', - '#1F6487', - '#1D668C', - '#0D2F41', - '#103B4F', - '#1C5C7E', - '#1F688F', - '#1C5B7C', - '#164D68', - '#1D6285', - '#0D364A', - '#1D5A7A', - '#1E6990', - '#1D6488', - '#18516B', - '#1A506B', - '#0E3B50', - '#0E3548', - '#124259', - '#13455C', - '#14485F', - '#1E5C7D', - '#122D3C', - '#1E6E98', - '#1E6A91', - '#1E6286', - '#1E6C95', - '#1D6990', - '#101F29', - '#174A62', - '#10394E', - '#1D6D96', - '#1E688E', - '#1D6E97', - '#1E6C94', - '#0E394E', - '#112B39', - '#195270', - '#1E668B', - '#1E6386', - '#1D6385', - '#0C3142', - '#1E6083', - '#1E729C', - '#1F709A', - '#1E6F98', - '#1D5F81', - '#1F688D', - '#1C6488', - '#1D6588', - '#1C6A93', - '#1E658B', - '#1F6C95', - '#0D3C52', - '#1C6385', - '#1E5F82', - '#0E3D54', - '#0F3244', - '#18485F', - '#1E6991', - '#1C5B7B', - '#1F6082', - '#0F3346', - '#18536F', - '#114056', - '#1D6B92', - '#1B5776', - '#0F3C52', - '#1E6890', - '#1F688E', - '#0C394E', - '#0F1D25', - '#1F6386', - '#1E688D', - '#1F6488', - '#20668C', - '#1D5978', - '#0F3D52', - '#0F1E26', - '#13465F', - '#0D374C', - '#1B5C7C', - '#0E1A23', - '#0F374A', - '#1B5574', - '#0F394C', - '#0E2A38', - '#102A37', - '#18506B', - '#1E5A7A', - '#0F3245', - '#0E2E3F', - '#1E678E', - '#1C5D7E', - '#1A5A7A', - '#0E2837', - '#102733', - '#0F3B51', - '#15475E', - '#1E6B93', - '#1E648A', - '#194961', - '#0F3A4E', - '#0E1D25', - '#194F69', - '#103345', - '#0F394D', - '#102B39', - '#103E55', - '#1B5572', - '#164861', - '#174861', - '#113B4F', - '#102936', - '#0F3041', - '#174961', - '#113E53', - '#134056', - '#124057', - '#194B63', - '#0E364B', - '#15445B', - '#16475E', - '#102F3F', - '#16485F', - '#0F2E3D', - '#101920', - '#12222C', - '#122C3B', - '#144157', - '#123B50', - '#16465D', - '#184960', - '#112B3A', - '#12232F', - '#132430', - '#113344', - '#11394C', - '#113649', - '#11364A', - '#133F56', - '#121D25', - '#112733', - '#112A38', - '#0F1F2A', - '#113447', - '#113A4E', - '#0F222C', - '#13222B', - '#112836', - '#102F3E', - '#113243', - '#123445', - '#12374B', - '#121E26', - '#122531', - '#11303F', - '#0D1D25', - '#102835', - '#112834', - '#101C23', - '#111C23', - '#12212B', - '#11222D', - '#0E1B22', - '#0E1D27', - '#121C22', - '#12202A', - '#101A20', - '#13191E', - '#111E28', - '#11212D', - '#0F1B24', - '#0F1C23', - '#13181D', - '#15171A', - '#121D23', - '#121F27', - '#111E27', - '#101B22', - '#121F28', - '#111E26', - '#101D24', - '#111C22', - '#12161E', - '#101925', - '#121E2D', - '#112033', - '#111E2F', - '#0F1B29', - '#0F1A28', - '#101B2A', - '#0E1A27', - '#101C2B', - '#111D2D', - '#111D2B', - '#0F1B28', - '#101923', - '#13161D', - '#13161C', - '#0F1A26', - '#101E2F', - '#112235', - '#102031', - '#0F1B2A', - '#112031', - '#102032', - '#101D2E', - '#121F2F', - '#112133', - '#101E30', - '#101F30', - '#102336', - '#101B2C', - '#0F1C2B', - '#111E2E', - '#0F2134', - '#102236', - '#0F2133', - '#101F31', - '#0F2438', - '#102337', - '#102235', - '#102133', - '#11171E', - '#101F2F', - '#102030', - '#102234', - '#102132', - '#12181F', - '#0F1A25', - '#0F2135', - '#0F1F30', - '#0F1C2D', - '#101D2C', - '#0F2033', - '#0E2338', - '#0F2237', - '#0F2236', - '#0B243B', - '#0D2338', - '#0E1A26', - '#0F1D2E', - '#0F2032', - '#0D2339', - '#0B253F', - '#0A253F', - '#0A253E', - '#0C2439', - '#0E1925', - '#0E2135', - '#0F2235', - '#0A243A', - '#08253E', - '#09253E', - '#0A263F', - '#0A243C', - '#0B233B', - '#0E1A28', - '#0D1A26', - '#09253F', - '#0A2743', - '#0B2844', - '#0B2641', - '#0A2744', - '#0A2844', - '#0B2743', - '#092745', - '#0F2337', - '#101D2D', - '#092743', - '#092846', - '#0E2B4C', - '#102E4F', - '#0E2C4D', - '#0B2A49', - '#082947', - '#0D2B4B', - '#0C2A4A', - '#092946', - '#082845', - '#0C2B4B', - '#0F2D4E', - '#103051', - '#133257', - '#0E2D4E', - '#143156', - '#112F51', - '#0B243A', - '#082744', - '#092844', - '#123054', - '#143359', - '#173A64', - '#183F6E', - '#173F6D', - '#153961', - '#163962', - '#133358', - '#15345B', - '#14345A', - '#102F50', - '#0A2948', - '#082844', - '#092641', - '#16375F', - '#193C69', - '#174170', - '#173E6B', - '#163A63', - '#173D69', - '#183D6A', - '#15365E', - '#112E50', - '#0A2A49', - '#082743', - '#0E1927', - '#173C68', - '#13487E', - '#164476', - '#174375', - '#193F6F', - '#173B66', - '#163B65', - '#082A48', - '#0A2641', - '#09243C', - '#174171', - '#14477C', - '#124980', - '#14487F', - '#174374', - '#15467B', - '#184172', - '#17406F', - '#184070', - '#163C67', - '#16355D', - '#123256', - '#0E1B29', - '#0F1923', - '#113052', - '#184274', - '#164579', - '#13477C', - '#193E6D', - '#0A243E', - '#0B233A', - '#0D1A29', - '#0B2742', - '#17365E', - '#163860', - '#124A84', - '#095191', - '#114A83', - '#0D4D8A', - '#0C4D8C', - '#104B85', - '#15477E', - '#174477', - '#183862', - '#0A233A', - '#092947', - '#09243D', - '#173963', - '#194173', - '#085396', - '#085394', - '#114B87', - '#144983', - '#094F8E', - '#075090', - '#0F4C89', - '#215287', - '#0E1A29', - '#184376', - '#0C4D8B', - '#07549A', - '#0A4E8D', - '#0F4C88', - '#0A4E8C', - '#174273', - '#193C6A', - '#0B2948', - '#0B2C4B', - '#0C4E8D', - '#1259A4', - '#0C579E', - '#0D4D8B', - '#095397', - '#085397', - '#085295', - '#144880', - '#173861', - '#15335A', - '#0F2C4D', - '#0C2949', - '#0B4E8D', - '#08559C', - '#07508F', - '#154578', - '#17365F', - '#122F53', - '#111D2C', - '#092A48', - '#08559D', - '#08559E', - '#0C56A1', - '#164271', - '#163E6A', - '#194071', - '#082642', - '#0F1E30', - '#0D2D4D', - '#114C87', - '#0E59A3', - '#135BA6', - '#085498', - '#085497', - '#095192', - '#0E4D8B', - '#0C4E8A', - '#134982', - '#17457B', - '#121F2E', - '#183E6C', - '#153E69', - '#07508E', - '#173F6C', - '#193D6B', - '#112D4F', - '#0A243B', - '#072946', - '#111E2D', - '#0B2740', - '#10497F', - '#17406E', - '#084F8D', - '#104A80', - '#0E2E4F', - '#143358', - '#16365D', - '#0A2742', - '#13477B', - '#154474', - '#104C86', - '#095291', - '#0B4F8E', - '#114A80', - '#095090', - '#075296', - '#163760', - '#2D6DB5', - '#0C2843', - '#0C233A', - '#153A62', - '#14467A', - '#075498', - '#085293', - '#09263F', - '#122030', - '#09559D', - '#0F4B83', - '#08549A', - '#14375D', - '#085499', - '#075499', - '#0A243D', - '#143E68', - '#10497E', - '#074F8E', - '#085496', - '#0C58A3', - '#065499', - '#085190', - '#0A2B4A', - '#104C88', - '#0D4F8E', - '#0F58A2', - '#0B569B', - '#0D58A1', - '#134A81', - '#09559C', - '#0A5293', - '#114B86', - '#0D2C4C', - '#103255', - '#16457A', - '#074F8C', - '#07559C', - '#185DA9', - '#1D61AD', - '#175CA8', - '#16406D', - '#153C65', - '#0E243A', - '#144679', - '#085192', - '#1A5EAC', - '#1D61AE', - '#11497F', - '#12487E', - '#0C243C', - '#123155', - '#0F59A3', - '#1B5FAB', - '#1E61AD', - '#145CA4', - '#0E599F', - '#11497E', - '#094F8D', - '#15345A', - '#134A85', - '#165CA8', - '#2263AF', - '#124466', - '#0A518F', - '#08569D', - '#16416F', - '#0B2B4A', - '#124A83', - '#0C57A2', - '#1E60AD', - '#1E62AE', - '#165DA8', - '#1059A4', - '#15406C', - '#0A4F8E', - '#12365A', - '#0A5191', - '#16355C', - '#1C5EAB', - '#155CA7', - '#085292', - '#174478', - '#153258', - '#111F2F', - '#174272', - '#1159A5', - '#1C5EAC', - '#2F74BB', - '#0C58A2', - '#0D59A3', - '#14477D', - '#132F53', - '#155BA6', - '#195FAA', - '#2366B1', - '#2967B2', - '#14477E', - '#1B5EAB', - '#175DA8', - '#0F4C86', - '#065090', - '#1C5FAC', - '#185CA8', - '#0D58A3', - '#0C4E8C', - '#134981', - '#14416D', - '#0F5AA5', - '#1F63AF', - '#114B88', - '#09508E', - '#0A569D', - '#195DAA', - '#0F1D2F', - '#1059A2', - '#0E599E', - '#2063AF', - '#1F63AE', - '#1A5EAA', - '#0C57A0', - '#195EAA', - '#1A5EA9', - '#0E4E8A', - '#12487D', - '#185DAA', - '#175EAA', - '#0A508E', - '#1559A6', - '#0E58A3', - '#095399', - '#0B4E8B', - '#0B569F', - '#0C57A1', - '#2967B1', - '#2365B0', - '#2163AE', - '#1A5DAA', - '#195EAB', - '#1E5FAC', - '#2564AF', - '#2767B1', - '#2766B1', - '#0D5A9F', - '#2062AE', - '#1F61AD', - '#195FAB', - '#0D4E8D', - '#173760', - '#111D2E', - '#09518F', - '#1A5FAC', - '#135BA7', - '#085291', - '#183761', - '#0B2845', - '#113457', - '#075393', - '#185EA9', - '#2B69B3', - '#2A67B2', - '#2867B1', - '#155DA8', - '#135CA6', - '#135AA5', - '#114980', - '#2566B1', - '#2064AF', - '#2364AF', - '#13365B', - '#154475', - '#08549B', - '#164373', - '#085392', - '#144576', - '#12497E', - '#0E5392', - '#135BA3', - '#0C5395', - '#0C5291', - '#0E579C', - '#0E5290', - '#134C83', - '#2163AC', - '#195CA6', - '#0D4E8C', - '#082945', - '#133256', - '#0E2F50', - '#105AA6', - '#134677', - '#144475', - '#145BA7', - '#154270', - '#1D60AD', - '#09569B', - '#09243E', - '#134A86', - '#0E59A4', - '#0A4E8B', - '#0E4B83', - '#1D5EAC', - '#101C2A', - '#134A84', - '#0E518F', - '#145CA7', - '#0E5699', - '#145BA5', - '#095292', - '#15416E', - '#153D67', - '#153F6B', - '#125AA5', - '#16406E', - '#0E1B27', - '#0D4F8C', - '#0F58A3', - '#114A82', - '#09569C', - '#0C2339', - '#0E1B28', - '#0D59A4', - '#07559D', - '#08569E', - '#095190', - '#0B253E', - '#0C2B49', - '#2264AF', - '#09549A', - '#09569F', - '#163D68', - '#0C263F', - '#143960', - '#183A65', - '#075496', - '#0C579F', - '#085191', - '#102438', - '#075295', - '#082946', - '#102437', - '#0C2642', - '#101C29', - '#0C253E', - '#15355C', - '#0B2E4D', - '#0F3253', - '#154577', - '#16335B', - '#0F1925', - '#0C2742', - '#0B2946', - '#0E2C4B', - '#0E2B48', - '#0E2237', - '#102237', - '#0B253D', - '#0A2946', - '#0C2841', - '#0D2A47', - '#0C2C4A', - '#08253F', - '#08243D', - '#111C2B', - '#0C2844', - '#0C2945', - '#0D243A', - '#122134', - '#0B2642', - '#113154', - '#113255', - '#0A2642', - '#0A2945', - '#0B263F', - '#0D2E4E', - '#0F1E2E', - '#0A2845', - '#0D2439', - '#0F1A29', - '#101C2E', - '#111923', - '#13181F', - '#111D2F', - '#111F30', - '#121E30', - '#121E2E', - '#101B27', - '#101A27', - '#13171F', - ]; - - // let randX = getRandomInt(0, x); - // let randY = getRandomInt(0, y); - // let randIndex = randY * xCount + randX; - - return colors[(y*xCount + x) % colors.length]; -} diff --git a/public/app/core/components/OrgActionBar/OrgActionBar.test.tsx b/public/app/core/components/OrgActionBar/OrgActionBar.test.tsx new file mode 100644 index 00000000000..9faf07f18d1 --- /dev/null +++ b/public/app/core/components/OrgActionBar/OrgActionBar.test.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import OrgActionBar, { Props } from './OrgActionBar'; + +const setup = (propOverrides?: object) => { + const props: Props = { + searchQuery: '', + setSearchQuery: jest.fn(), + target: '_blank', + linkButton: { href: 'some/url', title: 'test' }, + }; + + Object.assign(props, propOverrides); + + return shallow(); +}; + +describe('Render', () => { + it('should render component', () => { + const wrapper = setup(); + + expect(wrapper).toMatchSnapshot(); + }); +}); diff --git a/public/app/core/components/OrgActionBar/OrgActionBar.tsx b/public/app/core/components/OrgActionBar/OrgActionBar.tsx new file mode 100644 index 00000000000..8fc34a018e1 --- /dev/null +++ b/public/app/core/components/OrgActionBar/OrgActionBar.tsx @@ -0,0 +1,44 @@ +import React, { PureComponent } from 'react'; +import LayoutSelector, { LayoutMode } from '../LayoutSelector/LayoutSelector'; + +export interface Props { + searchQuery: string; + layoutMode?: LayoutMode; + onSetLayoutMode?: (mode: LayoutMode) => {}; + setSearchQuery: (value: string) => {}; + linkButton: { href: string; title: string }; + target?: string; +} + +export default class OrgActionBar extends PureComponent { + render() { + const { searchQuery, layoutMode, onSetLayoutMode, linkButton, setSearchQuery, target } = this.props; + const linkProps = { href: linkButton.href, target: undefined }; + + if (target) { + linkProps.target = target; + } + + return ( +
    +
    + + onSetLayoutMode(mode)} /> +
    +     + ); + } +} diff --git a/public/app/core/components/OrgActionBar/__snapshots__/OrgActionBar.test.tsx.snap b/public/app/core/components/OrgActionBar/__snapshots__/OrgActionBar.test.tsx.snap new file mode 100644 index 00000000000..dc53e7863ea --- /dev/null +++ b/public/app/core/components/OrgActionBar/__snapshots__/OrgActionBar.test.tsx.snap @@ -0,0 +1,39 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Render should render component 1`] = ` +
    +
    + + +
    + +`; diff --git a/public/app/core/components/PageHeader/PageHeader.jest.tsx b/public/app/core/components/PageHeader/PageHeader.test.tsx similarity index 100% rename from public/app/core/components/PageHeader/PageHeader.jest.tsx rename to public/app/core/components/PageHeader/PageHeader.test.tsx diff --git a/public/app/core/components/PageHeader/PageHeader.tsx b/public/app/core/components/PageHeader/PageHeader.tsx index f998cb9981f..c176095afa4 100644 --- a/public/app/core/components/PageHeader/PageHeader.tsx +++ b/public/app/core/components/PageHeader/PageHeader.tsx @@ -1,11 +1,9 @@ import React from 'react'; -import { observer } from 'mobx-react'; -import { NavModel, NavModelItem } from '../../nav_model_srv'; +import { NavModel, NavModelItem } from 'app/types'; import classNames from 'classnames'; import appEvents from 'app/core/app_events'; -import { toJS } from 'mobx'; -export interface IProps { +export interface Props { model: NavModel; } @@ -15,8 +13,8 @@ const SelectNav = ({ main, customCss }: { main: NavModelItem; customCss: string }); const gotoUrl = evt => { - var element = evt.target; - var url = element.options[element.selectedIndex].value; + const element = evt.target; + const url = element.options[element.selectedIndex].value; appEvents.emit('location-change', { href: url }); }; @@ -81,8 +79,7 @@ const Navigation = ({ main }: { main: NavModelItem }) => { ); }; -@observer -export default class PageHeader extends React.Component { +export default class PageHeader extends React.Component { constructor(props) { super(props); } @@ -148,7 +145,7 @@ export default class PageHeader extends React.Component { return null; } - const main = toJS(model.main); // Convert to JS if its a mobx observable + const main = model.main; return (
    diff --git a/public/app/core/components/PageLoader/PageLoader.tsx b/public/app/core/components/PageLoader/PageLoader.tsx new file mode 100644 index 00000000000..dcb67dde220 --- /dev/null +++ b/public/app/core/components/PageLoader/PageLoader.tsx @@ -0,0 +1,17 @@ +import React, { SFC } from 'react'; + +interface Props { + pageName: string; +} + +const PageLoader: SFC = ({ pageName }) => { + const loadingText = `Loading ${pageName}...`; + return ( +
    + +
    {loadingText}
    +
    + ); +}; + +export default PageLoader; diff --git a/public/app/core/components/PasswordStrength.tsx b/public/app/core/components/PasswordStrength.tsx index 8f92b18445c..1d676a00a37 100644 --- a/public/app/core/components/PasswordStrength.tsx +++ b/public/app/core/components/PasswordStrength.tsx @@ -1,32 +1,31 @@ import React from 'react'; -export interface IProps { +export interface Props { password: string; } -export class PasswordStrength extends React.Component { - +export class PasswordStrength extends React.Component { constructor(props) { super(props); } render() { const { password } = this.props; - let strengthText = "strength: strong like a bull."; - let strengthClass = "password-strength-good"; + let strengthText = 'strength: strong like a bull.'; + let strengthClass = 'password-strength-good'; if (!password) { return null; } if (password.length <= 8) { - strengthText = "strength: you can do better."; - strengthClass = "password-strength-ok"; + strengthText = 'strength: you can do better.'; + strengthClass = 'password-strength-ok'; } if (password.length < 4) { - strengthText = "strength: weak sauce."; - strengthClass = "password-strength-bad"; + strengthText = 'strength: weak sauce.'; + strengthClass = 'password-strength-bad'; } return ( @@ -36,5 +35,3 @@ export class PasswordStrength extends React.Component { ); } } - - diff --git a/public/app/core/components/PermissionList/AddPermission.tsx b/public/app/core/components/PermissionList/AddPermission.tsx new file mode 100644 index 00000000000..71cc937ddfa --- /dev/null +++ b/public/app/core/components/PermissionList/AddPermission.tsx @@ -0,0 +1,144 @@ +import React, { Component } from 'react'; +import { UserPicker } from 'app/core/components/Picker/UserPicker'; +import { TeamPicker, Team } from 'app/core/components/Picker/TeamPicker'; +import DescriptionPicker, { OptionWithDescription } from 'app/core/components/Picker/DescriptionPicker'; +import { User } from 'app/types'; +import { + dashboardPermissionLevels, + dashboardAclTargets, + AclTarget, + PermissionLevel, + NewDashboardAclItem, + OrgRole, +} from 'app/types/acl'; + +export interface Props { + onAddPermission: (item: NewDashboardAclItem) => void; + onCancel: () => void; +} + +class AddPermissions extends Component { + static defaultProps = { + showPermissionLevels: true, + }; + + constructor(props) { + super(props); + this.state = this.getCleanState(); + } + + getCleanState() { + return { + userId: 0, + teamId: 0, + type: AclTarget.Team, + permission: PermissionLevel.View, + }; + } + + onTypeChanged = evt => { + const type = evt.target.value as AclTarget; + + switch (type) { + case AclTarget.User: + case AclTarget.Team: + this.setState({ type: type, userId: 0, teamId: 0, role: undefined }); + break; + case AclTarget.Editor: + this.setState({ type: type, userId: 0, teamId: 0, role: OrgRole.Editor }); + break; + case AclTarget.Viewer: + this.setState({ type: type, userId: 0, teamId: 0, role: OrgRole.Viewer }); + break; + } + }; + + onUserSelected = (user: User) => { + this.setState({ userId: user && !Array.isArray(user) ? user.id : 0 }); + }; + + onTeamSelected = (team: Team) => { + this.setState({ teamId: team && !Array.isArray(team) ? team.id : 0 }); + }; + + onPermissionChanged = (permission: OptionWithDescription) => { + this.setState({ permission: permission.value }); + }; + + onSubmit = async evt => { + evt.preventDefault(); + await this.props.onAddPermission(this.state); + this.setState(this.getCleanState()); + }; + + isValid() { + switch (this.state.type) { + case AclTarget.Team: + return this.state.teamId > 0; + case AclTarget.User: + return this.state.userId > 0; + } + return true; + } + + render() { + const { onCancel } = this.props; + const newItem = this.state; + const pickerClassName = 'width-20'; + const isValid = this.isValid(); + return ( +
    + +
    +
    Add Permission For
    +
    +
    +
    + +
    +
    + + {newItem.type === AclTarget.User ? ( +
    + +
    + ) : null} + + {newItem.type === AclTarget.Team ? ( +
    + +
    + ) : null} + +
    + +
    + +
    + +
    +
    +
    +
    + ); + } +} + +export default AddPermissions; diff --git a/public/app/core/components/Permissions/DisabledPermissionsListItem.tsx b/public/app/core/components/PermissionList/DisabledPermissionListItem.tsx similarity index 54% rename from public/app/core/components/Permissions/DisabledPermissionsListItem.tsx rename to public/app/core/components/PermissionList/DisabledPermissionListItem.tsx index db45714136e..ff679f67ae2 100644 --- a/public/app/core/components/Permissions/DisabledPermissionsListItem.tsx +++ b/public/app/core/components/PermissionList/DisabledPermissionListItem.tsx @@ -1,31 +1,34 @@ -import React, { Component } from 'react'; +import React, { Component } from 'react'; import DescriptionPicker from 'app/core/components/Picker/DescriptionPicker'; -import { permissionOptions } from 'app/stores/PermissionsStore/PermissionsStore'; +import { dashboardPermissionLevels } from 'app/types/acl'; -export interface IProps { +export interface Props { item: any; } -export default class DisabledPermissionListItem extends Component { +export default class DisabledPermissionListItem extends Component { render() { const { item } = this.props; return ( - - - + + + + + {item.name} + (Role) Can
    {}} - value={item.permission} + optionsWithDesc={dashboardPermissionLevels} + onSelected={() => {}} disabled={true} - className={'gf-form-input--form-dropdown-right'} + className={'gf-form-select-box__control--menu-right'} + value={item.permission} />
    diff --git a/public/app/core/components/Permissions/PermissionsList.tsx b/public/app/core/components/PermissionList/PermissionList.tsx similarity index 50% rename from public/app/core/components/Permissions/PermissionsList.tsx rename to public/app/core/components/PermissionList/PermissionList.tsx index b215dad2391..772baa0c274 100644 --- a/public/app/core/components/Permissions/PermissionsList.tsx +++ b/public/app/core/components/PermissionList/PermissionList.tsx @@ -1,21 +1,20 @@ -import React, { Component } from 'react'; -import PermissionsListItem from './PermissionsListItem'; -import DisabledPermissionsListItem from './DisabledPermissionsListItem'; -import { observer } from 'mobx-react'; -import { FolderInfo } from './FolderInfo'; +import React, { PureComponent } from 'react'; +import PermissionsListItem from './PermissionListItem'; +import DisabledPermissionsListItem from './DisabledPermissionListItem'; +import { FolderInfo } from 'app/types'; +import { DashboardAcl } from 'app/types/acl'; -export interface IProps { - permissions: any[]; - removeItem: any; - permissionChanged: any; - fetching: boolean; +export interface Props { + items: DashboardAcl[]; + onRemoveItem: (item: DashboardAcl) => void; + onPermissionChanged: any; + isFetching: boolean; folderInfo?: FolderInfo; } -@observer -class PermissionsList extends Component { +class PermissionList extends PureComponent { render() { - const { permissions, removeItem, permissionChanged, fetching, folderInfo } = this.props; + const { items, onRemoveItem, onPermissionChanged, isFetching, folderInfo } = this.props; return ( @@ -23,24 +22,23 @@ class PermissionsList extends Component { Admin Role', + name: 'Admin', permission: 4, icon: 'fa fa-fw fa-street-view', }} /> - {permissions.map((item, idx) => { + {items.map((item, idx) => { return ( ); })} - {fetching === true && permissions.length < 1 ? ( + {isFetching === true && items.length < 1 ? ( ) : null} - {fetching === false && permissions.length < 1 ? ( + {isFetching === false && items.length < 1 ? ( + + + + + + + + ); + } +} diff --git a/public/app/core/components/Permissions/PermissionsInfo.tsx b/public/app/core/components/PermissionList/PermissionsInfo.tsx similarity index 100% rename from public/app/core/components/Permissions/PermissionsInfo.tsx rename to public/app/core/components/PermissionList/PermissionsInfo.tsx diff --git a/public/app/core/components/Permissions/AddPermissions.jest.tsx b/public/app/core/components/Permissions/AddPermissions.jest.tsx deleted file mode 100644 index fe97c4c7e62..00000000000 --- a/public/app/core/components/Permissions/AddPermissions.jest.tsx +++ /dev/null @@ -1,90 +0,0 @@ -import React from 'react'; -import AddPermissions from './AddPermissions'; -import { RootStore } from 'app/stores/RootStore/RootStore'; -import { backendSrv } from 'test/mocks/common'; -import { shallow } from 'enzyme'; - -describe('AddPermissions', () => { - let wrapper; - let store; - let instance; - - beforeAll(() => { - backendSrv.get.mockReturnValue( - Promise.resolve([ - { id: 2, dashboardId: 1, role: 'Viewer', permission: 1, permissionName: 'View' }, - { id: 3, dashboardId: 1, role: 'Editor', permission: 1, permissionName: 'Edit' }, - ]) - ); - - backendSrv.post = jest.fn(() => Promise.resolve({})); - - store = RootStore.create( - {}, - { - backendSrv: backendSrv, - } - ); - - wrapper = shallow(); - instance = wrapper.instance(); - return store.permissions.load(1, true, false); - }); - - describe('when permission for a user is added', () => { - it('should save permission to db', () => { - const evt = { - target: { - value: 'User', - }, - }; - const userItem = { - id: 2, - login: 'user2', - }; - - instance.typeChanged(evt); - instance.userPicked(userItem); - - wrapper.update(); - - expect(wrapper.find('[data-save-permission]').prop('disabled')).toBe(false); - - wrapper.find('form').simulate('submit', { preventDefault() {} }); - - expect(backendSrv.post.mock.calls.length).toBe(1); - expect(backendSrv.post.mock.calls[0][0]).toBe('/api/dashboards/id/1/permissions'); - }); - }); - - describe('when permission for team is added', () => { - it('should save permission to db', () => { - const evt = { - target: { - value: 'Group', - }, - }; - - const teamItem = { - id: 2, - name: 'ug1', - }; - - instance.typeChanged(evt); - instance.teamPicked(teamItem); - - wrapper.update(); - - expect(wrapper.find('[data-save-permission]').prop('disabled')).toBe(false); - - wrapper.find('form').simulate('submit', { preventDefault() {} }); - - expect(backendSrv.post.mock.calls.length).toBe(1); - expect(backendSrv.post.mock.calls[0][0]).toBe('/api/dashboards/id/1/permissions'); - }); - }); - - afterEach(() => { - backendSrv.post.mockClear(); - }); -}); diff --git a/public/app/core/components/Permissions/AddPermissions.tsx b/public/app/core/components/Permissions/AddPermissions.tsx deleted file mode 100644 index 07ccfdbbef5..00000000000 --- a/public/app/core/components/Permissions/AddPermissions.tsx +++ /dev/null @@ -1,143 +0,0 @@ -import React, { Component } from 'react'; -import { observer } from 'mobx-react'; -import { aclTypes } from 'app/stores/PermissionsStore/PermissionsStore'; -import UserPicker, { User } from 'app/core/components/Picker/UserPicker'; -import TeamPicker, { Team } from 'app/core/components/Picker/TeamPicker'; -import DescriptionPicker, { OptionWithDescription } from 'app/core/components/Picker/DescriptionPicker'; -import { permissionOptions } from 'app/stores/PermissionsStore/PermissionsStore'; - -export interface IProps { - permissions: any; - backendSrv: any; -} -@observer -class AddPermissions extends Component { - constructor(props) { - super(props); - this.userPicked = this.userPicked.bind(this); - this.teamPicked = this.teamPicked.bind(this); - this.permissionPicked = this.permissionPicked.bind(this); - this.typeChanged = this.typeChanged.bind(this); - this.handleSubmit = this.handleSubmit.bind(this); - } - - componentWillMount() { - const { permissions } = this.props; - permissions.resetNewType(); - } - - typeChanged(evt) { - const { value } = evt.target; - const { permissions } = this.props; - - permissions.setNewType(value); - } - - userPicked(user: User) { - const { permissions } = this.props; - if (!user) { - permissions.newItem.setUser(null, null); - return; - } - return permissions.newItem.setUser(user.id, user.login); - } - - teamPicked(team: Team) { - const { permissions } = this.props; - if (!team) { - permissions.newItem.setTeam(null, null); - return; - } - return permissions.newItem.setTeam(team.id, team.name); - } - - permissionPicked(permission: OptionWithDescription) { - const { permissions } = this.props; - return permissions.newItem.setPermission(permission.value); - } - - resetNewType() { - const { permissions } = this.props; - return permissions.resetNewType(); - } - - handleSubmit(evt) { - evt.preventDefault(); - const { permissions } = this.props; - permissions.addStoreItem(); - } - - render() { - const { permissions, backendSrv } = this.props; - const newItem = permissions.newItem; - const pickerClassName = 'width-20'; - - const isValid = newItem.isValid(); - - return ( -
    - -
    -
    Add Permission For
    -
    -
    -
    - -
    -
    - - {newItem.type === 'User' ? ( -
    - -
    - ) : null} - - {newItem.type === 'Group' ? ( -
    - -
    - ) : null} - -
    - -
    - -
    - -
    -
    - -
    - ); - } -} - -export default AddPermissions; diff --git a/public/app/core/components/Permissions/DashboardPermissions.tsx b/public/app/core/components/Permissions/DashboardPermissions.tsx deleted file mode 100644 index 12339cc7c34..00000000000 --- a/public/app/core/components/Permissions/DashboardPermissions.tsx +++ /dev/null @@ -1,70 +0,0 @@ -import React, { Component } from 'react'; -import { observer } from 'mobx-react'; -import { store } from 'app/stores/store'; -import Permissions from 'app/core/components/Permissions/Permissions'; -import Tooltip from 'app/core/components/Tooltip/Tooltip'; -import PermissionsInfo from 'app/core/components/Permissions/PermissionsInfo'; -import AddPermissions from 'app/core/components/Permissions/AddPermissions'; -import SlideDown from 'app/core/components/Animations/SlideDown'; -import { FolderInfo } from './FolderInfo'; - -export interface IProps { - dashboardId: number; - folder?: FolderInfo; - backendSrv: any; -} -@observer -class DashboardPermissions extends Component { - permissions: any; - - constructor(props) { - super(props); - this.handleAddPermission = this.handleAddPermission.bind(this); - this.permissions = store.permissions; - } - - handleAddPermission() { - this.permissions.toggleAddPermissions(); - } - - componentWillUnmount() { - this.permissions.hideAddPermissions(); - } - - render() { - const { dashboardId, folder, backendSrv } = this.props; - - return ( -
    -
    -
    -

    Permissions

    - - - -
    - -
    -
    - - - - -
    - ); - } -} - -export default DashboardPermissions; diff --git a/public/app/core/components/Permissions/FolderInfo.ts b/public/app/core/components/Permissions/FolderInfo.ts deleted file mode 100644 index d4a6020bb71..00000000000 --- a/public/app/core/components/Permissions/FolderInfo.ts +++ /dev/null @@ -1,5 +0,0 @@ -export interface FolderInfo { - id: number; - title: string; - url: string; -} diff --git a/public/app/core/components/Permissions/Permissions.tsx b/public/app/core/components/Permissions/Permissions.tsx deleted file mode 100644 index 0a0572ed86e..00000000000 --- a/public/app/core/components/Permissions/Permissions.tsx +++ /dev/null @@ -1,92 +0,0 @@ -import React, { Component } from 'react'; -import PermissionsList from './PermissionsList'; -import { observer } from 'mobx-react'; -import { FolderInfo } from './FolderInfo'; - -export interface DashboardAcl { - id?: number; - dashboardId?: number; - userId?: number; - userLogin?: string; - userEmail?: string; - teamId?: number; - team?: string; - permission?: number; - permissionName?: string; - role?: string; - icon?: string; - nameHtml?: string; - inherited?: boolean; - sortName?: string; - sortRank?: number; -} - -export interface IProps { - dashboardId: number; - folderInfo?: FolderInfo; - permissions?: any; - isFolder: boolean; - backendSrv: any; -} - -@observer -class Permissions extends Component { - constructor(props) { - super(props); - const { dashboardId, isFolder, folderInfo } = this.props; - this.permissionChanged = this.permissionChanged.bind(this); - this.typeChanged = this.typeChanged.bind(this); - this.removeItem = this.removeItem.bind(this); - this.loadStore(dashboardId, isFolder, folderInfo && folderInfo.id === 0); - } - - loadStore(dashboardId, isFolder, isInRoot = false) { - return this.props.permissions.load(dashboardId, isFolder, isInRoot); - } - - permissionChanged(index: number, permission: number, permissionName: string) { - const { permissions } = this.props; - permissions.updatePermissionOnIndex(index, permission, permissionName); - } - - removeItem(index: number) { - const { permissions } = this.props; - permissions.removeStoreItem(index); - } - - resetNewType() { - const { permissions } = this.props; - permissions.resetNewType(); - } - - typeChanged(evt) { - const { value } = evt.target; - const { permissions, dashboardId } = this.props; - - if (value === 'Viewer' || value === 'Editor') { - permissions.addStoreItem({ permission: 1, role: value, dashboardId: dashboardId }, dashboardId); - this.resetNewType(); - return; - } - - permissions.setNewType(value); - } - - render() { - const { permissions, folderInfo } = this.props; - - return ( -
    - -
    - ); - } -} - -export default Permissions; diff --git a/public/app/core/components/Permissions/PermissionsListItem.tsx b/public/app/core/components/Permissions/PermissionsListItem.tsx deleted file mode 100644 index 3140b8fcc0c..00000000000 --- a/public/app/core/components/Permissions/PermissionsListItem.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import React from 'react'; -import { observer } from 'mobx-react'; -import DescriptionPicker from 'app/core/components/Picker/DescriptionPicker'; -import { permissionOptions } from 'app/stores/PermissionsStore/PermissionsStore'; - -const setClassNameHelper = inherited => { - return inherited ? 'gf-form-disabled' : ''; -}; - -export default observer(({ item, removeItem, permissionChanged, itemIndex, folderInfo }) => { - const handleRemoveItem = evt => { - evt.preventDefault(); - removeItem(itemIndex); - }; - - const handleChangePermission = permissionOption => { - permissionChanged(itemIndex, permissionOption.value, permissionOption.label); - }; - - const inheritedFromRoot = item.dashboardId === -1 && folderInfo && folderInfo.id === 0; - - return ( -
    - - - - - - - ); -}); diff --git a/public/app/core/components/Picker/DescriptionOption.tsx b/public/app/core/components/Picker/DescriptionOption.tsx index 12a1fdd9163..9ddf13f7532 100644 --- a/public/app/core/components/Picker/DescriptionOption.tsx +++ b/public/app/core/components/Picker/DescriptionOption.tsx @@ -1,56 +1,25 @@ -import React, { Component } from 'react'; +import React from 'react'; +import { components } from 'react-select'; +import { OptionProps } from 'react-select/lib/components/Option'; -export interface IProps { - onSelect: any; - onFocus: any; - option: any; - isFocused: any; - className: any; +// https://github.com/JedWatson/react-select/issues/3038 +interface ExtendedOptionProps extends OptionProps { + data: any; } -class DescriptionOption extends Component { - constructor(props) { - super(props); - this.handleMouseDown = this.handleMouseDown.bind(this); - this.handleMouseEnter = this.handleMouseEnter.bind(this); - this.handleMouseMove = this.handleMouseMove.bind(this); - } - - handleMouseDown(event) { - event.preventDefault(); - event.stopPropagation(); - this.props.onSelect(this.props.option, event); - } - - handleMouseEnter(event) { - this.props.onFocus(this.props.option, event); - } - - handleMouseMove(event) { - if (this.props.isFocused) { - return; - } - this.props.onFocus(this.props.option, event); - } - - render() { - const { option, children, className } = this.props; - return ( -
    Loading permissions... @@ -48,7 +46,7 @@ class PermissionsList extends Component {
    No permissions are set. Will only be accessible by admins. @@ -61,4 +59,4 @@ class PermissionsList extends Component { } } -export default PermissionsList; +export default PermissionList; diff --git a/public/app/core/components/PermissionList/PermissionListItem.tsx b/public/app/core/components/PermissionList/PermissionListItem.tsx new file mode 100644 index 00000000000..56b6114d236 --- /dev/null +++ b/public/app/core/components/PermissionList/PermissionListItem.tsx @@ -0,0 +1,100 @@ +import React, { PureComponent } from 'react'; +import DescriptionPicker from 'app/core/components/Picker/DescriptionPicker'; +import { dashboardPermissionLevels, DashboardAcl, PermissionLevel } from 'app/types/acl'; +import { FolderInfo } from 'app/types'; + +const setClassNameHelper = inherited => { + return inherited ? 'gf-form-disabled' : ''; +}; + +function ItemAvatar({ item }) { + if (item.userAvatarUrl) { + return ; + } + if (item.teamAvatarUrl) { + return ; + } + if (item.role === 'Editor') { + return ; + } + + return ; +} + +function ItemDescription({ item }) { + if (item.userId) { + return (User); + } + if (item.teamId) { + return (Team); + } + return (Role); +} + +interface Props { + item: DashboardAcl; + onRemoveItem: (item: DashboardAcl) => void; + onPermissionChanged: (item: DashboardAcl, level: PermissionLevel) => void; + folderInfo?: FolderInfo; +} + +export default class PermissionsListItem extends PureComponent { + onPermissionChanged = option => { + this.props.onPermissionChanged(this.props.item, option.value as PermissionLevel); + }; + + onRemoveItem = () => { + this.props.onRemoveItem(this.props.item); + }; + + render() { + const { item, folderInfo } = this.props; + const inheritedFromRoot = item.dashboardId === -1 && !item.inherited; + + return ( +
    + + + {item.name} + + {item.inherited && + folderInfo && ( + + Inherited from folder{' '} + + {folderInfo.title} + {' '} + + )} + {inheritedFromRoot && Default Permission} + Can +
    + +
    +     		+ {!item.inherited ? ( + + + + ) : ( + + )} +
    - - - - {item.inherited && - folderInfo && ( - - Inherited from folder{' '} - - {folderInfo.title} - {' '} - - )} - {inheritedFromRoot && Default Permission} - Can -
    - -
    -     		- {!item.inherited ? ( - - - - ) : ( - - )} -
    + {LEGEND_STATS.map( + statName => + seriesValuesProps[statName] && ( + + ) + )} +