diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 57fca7f44cb..082482fcb74 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,8 +4,6 @@ Read before posting: - Checkout FAQ: https://community.grafana.com/c/howto/faq - Checkout How to troubleshoot metric query issues: https://community.grafana.com/t/how-to-troubleshoot-metric-query-issues/50 -Please prefix your title with [Bug] or [Feature request]. - Please include this information: - What Grafana version are you using? - What datasource are you using? diff --git a/.gitignore b/.gitignore index ddabf2b680c..f743dcdc015 100644 --- a/.gitignore +++ b/.gitignore @@ -38,12 +38,16 @@ public/css/*.min.css conf/custom.ini fig.yml docker-compose.yml +docker-compose.yaml +/conf/dashboards/custom.yaml +/conf/datasources/custom.yaml profile.cov /grafana .notouch /pkg/cmd/grafana-cli/grafana-cli /pkg/cmd/grafana-server/grafana-server /pkg/cmd/grafana-server/debug +debug.test /examples/*/dist /packaging/**/*.rpm /packaging/**/*.deb diff --git a/CHANGELOG.md b/CHANGELOG.md index c0833f60e37..1b4adfcdda2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,20 @@ ## New Features * **Data Source Proxy**: Add support for whitelisting specified cookies that will be passed through to the data source when proxying data source requests [#5457](https://github.com/grafana/grafana/issues/5457), thanks [@robingustafsson](https://github.com/robingustafsson) +* **Postgres/MySQL**: add __timeGroup macro for mysql [#9596](https://github.com/grafana/grafana/pull/9596), thanks [@svenklemm](https://github.com/svenklemm) +* **Text**: Text panel are now edited in the ace editor. [#9698](https://github.com/grafana/grafana/pull/9698), thx [@mtanda](https://github.com/mtanda) +* **Teams**: Add Microsoft Teams notifier as [#8523](https://github.com/grafana/grafana/issues/8523), thx [@anthu](https://github.com/anthu) +* **Datasources**: Its now possible to configure datasources with config files [#1789](https://github.com/grafana/grafana/issues/1789) +* **Graphite**: Query editor updated to support new query by tag features [#9230](https://github.com/grafana/grafana/issues/9230) +* **Dashboard history**: New config file option versions_to_keep sets how many versions per dashboard to store, [#9671](https://github.com/grafana/grafana/issues/9671) - +## Minor +* **Alert panel**: Adds placeholder text when no alerts are within the time range [#9624](https://github.com/grafana/grafana/issues/9624), thx [@straend](https://github.com/straend) +* **Mysql**: MySQL enable MaxOpenCon and MaxIdleCon regards how constring is configured. [#9784](https://github.com/grafana/grafana/issues/9784), thx [@dfredell](https://github.com/dfredell) +* **Cloudwatch**: Fixes broken query inspector for cloudwatch [#9661](https://github.com/grafana/grafana/issues/9661), thx [@mtanda](https://github.com/mtanda) +* **Dashboard**: Make it possible to start dashboards from search and dashboard list panel [#1871](https://github.com/grafana/grafana/issues/1871) +* **Annotations**: Posting annotations now return the id of the annotation [#9798](https://github.com/grafana/grafana/issues/9798) +* **Systemd**: Use systemd notification ready flag [#10024](https://github.com/grafana/grafana/issues/10024), thx [@jgrassler](https://github.com/jgrassler) ## Tech * **RabbitMq**: Remove support for publishing events to RabbitMQ [#9645](https://github.com/grafana/grafana/issues/9645) @@ -21,6 +33,32 @@ * **Singlestat**: suppress error when result contains no datapoints [#9636](https://github.com/grafana/grafana/issues/9636), thx [@utkarshcmu](https://github.com/utkarshcmu) * **Postgres/MySQL**: Control quoting in SQL-queries when using template variables [#9030](https://github.com/grafana/grafana/issues/9030), thanks [@svenklemm](https://github.com/svenklemm) +# 4.6.3 (unreleased) + +## Fixes +* **Gzip**: Fixes bug gravatar images when gzip was enabled [#5952](https://github.com/grafana/grafana/issues/5952) +* **Alert list**: Now shows alert state changes even after adding manual annotations on dashboard [#9951](https://github.com/grafana/grafana/issues/9951) + +# 4.6.2 (2017-11-16) + +## Important +* **Prometheus**: Fixes bug with new prometheus alerts in Grafana. Make sure to download this version if your using Prometheus for alerting. More details in the issue. [#9777](https://github.com/grafana/grafana/issues/9777) + +## Fixes +* **Color picker**: Bug after using textbox input field to change/paste color string [#9769](https://github.com/grafana/grafana/issues/9769) +* **Cloudwatch**: Fix for cloudwatch templating query `ec2_instance_attribute` [#9667](https://github.com/grafana/grafana/issues/9667), thanks [@mtanda](https://github.com/mtanda) +* **Heatmap**: Fixed tooltip for "time series buckets" mode [#9332](https://github.com/grafana/grafana/issues/9332) +* **InfluxDB**: Fixed query editor issue when using `>` or `<` operators in WHERE clause [#9871](https://github.com/grafana/grafana/issues/9871) + + +# 4.6.1 (2017-11-01) + +* **Singlestat**: Lost thresholds when using save dashboard as [#9681](https://github.com/grafana/grafana/issues/9681) +* **Graph**: Fix for series override color picker [#9715](https://github.com/grafana/grafana/issues/9715) +* **Go**: build using golang 1.9.2 [#9713](https://github.com/grafana/grafana/issues/9713) +* **Plugins**: Fixed problem with loading plugin js files behind auth proxy [#9509](https://github.com/grafana/grafana/issues/9509) +* **Graphite**: Annotation tooltip should render empty string when undefined [#9707](https://github.com/grafana/grafana/issues/9707) + # 4.6.0 (2017-10-26) ## Fixes diff --git a/LICENSE.md b/LICENSE.md index 4c6a79691f0..d6456956733 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,14 +1,202 @@ -Copyright 2014-2017 Torkel Ödegaard, Raintank Inc. -Licensed under the Apache License, Version 2.0 (the "License"); you -may not use this file except in compliance with the License. You may -obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -implied. See the License for the specific language governing -permissions and limitations under the License. + 1. Definitions. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE.md b/NOTICE.md index 171332f00d4..ca148971b62 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,16 +1,6 @@ -This software is based on Kibana: -======================================== +Copyright 2014-2017 Grafana Labs + +This software is based on Kibana: Copyright 2012-2013 Elasticsearch BV -Licensed under the Apache License, Version 2.0 (the "License"); you -may not use this file except in compliance with the License. You may -obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -implied. See the License for the specific language governing -permissions and limitations under the License. diff --git a/ROADMAP.md b/ROADMAP.md index 3ce0c33f088..4273d8df6a9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,29 +1,36 @@ -# Roadmap (2017-08-29) +# Roadmap (2017-10-31) This roadmap is a tentative plan for the core development team. Things change constantly as PRs come in and priorities change. But it will give you an idea of our current vision and plan. ### Short term (1-4 months) - - Release Grafana v4.5 with fixes and minor enhancements - Release Grafana v5 - User groups - Dashboard folders - - Dashboard permissions (on folders as well), permissions on groups or users + - Dashboard & folder permissions (assigned to users or groups) - New Dashboard layout engine - New sidemenu & nav UX - Elasticsearch alerting + - React migration foundation (core components) + - Graphite 1.1 Tags Support -### Long term +### Long term (4 - 8 months) - Backend plugins to support more Auth options, Alerting data sources & notifications -- Universal time series transformations for any data source (meta queries) -- Reporting -- Web socket & live data streams -- Migrate to Angular2 or react +- Alerting improvements (silence, per series tracking, etc) +- Dashboard as configuration and other automation / provisioning improvements +- Progress on React migration +- Change visualization (panel type) on the fly. +- Multi stat panel (vertical version of singlestat with bars/graph mode with big number etc) +- Repeat panel by query results +### In a distant future far far away + +- Meta queries +- Integrated light weight TSDB +- Web socket & live data sources ### Outside contributions We know this is being worked on right now by contributors (and we hope to merge it when it's ready). -- Clustering for alert engine (load distribution) diff --git a/appveyor.yml b/appveyor.yml index 19de1d3a793..5d67edca9d9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,7 +7,7 @@ clone_folder: c:\gopath\src\github.com\grafana\grafana environment: nodejs_version: "6" GOPATH: c:\gopath - GOVERSION: 1.9.1 + GOVERSION: 1.9.2 install: - rmdir c:\go /s /q diff --git a/build.go b/build.go index a1d1d3012ab..1c61c72d5dc 100644 --- a/build.go +++ b/build.go @@ -95,9 +95,9 @@ func main() { case "package": grunt(gruntBuildArg("release")...) - if runtime.GOOS != "windows" { - createLinuxPackages() - } + if runtime.GOOS != "windows" { + createLinuxPackages() + } case "pkg-rpm": grunt(gruntBuildArg("release")...) diff --git a/circle.yml b/circle.yml index 8380bc2a2ff..4eb600bfde3 100644 --- a/circle.yml +++ b/circle.yml @@ -9,7 +9,7 @@ machine: GOPATH: "/home/ubuntu/.go_workspace" ORG_PATH: "github.com/grafana" REPO_PATH: "${ORG_PATH}/grafana" - GODIST: "go1.9.1.linux-amd64.tar.gz" + GODIST: "go1.9.2.linux-amd64.tar.gz" post: - mkdir -p ~/download - mkdir -p ~/docker diff --git a/codecov.yml b/codecov.yml index 3d764c1a5b1..82a86e0232b 100644 --- a/codecov.yml +++ b/codecov.yml @@ -7,5 +7,7 @@ coverage: project: yes patch: yes changes: no - -comment: false + +comment: + layout: "diff" + behavior: "once" diff --git a/conf/datasources/datasources.yaml b/conf/datasources/datasources.yaml new file mode 100644 index 00000000000..d8ddc9c6bed --- /dev/null +++ b/conf/datasources/datasources.yaml @@ -0,0 +1,48 @@ +# list of datasources that should be deleted from the database +delete_datasources: + # - name: Graphite + # org_id: 1 + +# list of datasources to insert/update depending +# whats available in the datbase +datasources: +# # name of the datasource. Required +# - name: Graphite +# # datasource type. Required +# type: graphite +# # access mode. direct or proxy. Required +# access: proxy +# # org id. will default to org_id 1 if not specified +# org_id: 1 +# # url +# url: http://localhost:8080 +# # database password, if used +# password: +# # database user, if used +# user: +# # database name, if used +# database: +# # enable/disable basic auth +# basic_auth: +# # basic auth username +# basic_auth_user: +# # basic auth password +# basic_auth_password: +# # enable/disable with credentials headers +# with_credentials: +# # mark as default datasource. Max one per org +# is_default: +# # fields that will be converted to json and stored in json_data +# json_data: +# graphiteVersion: "1.1" +# tlsAuth: true +# tlsAuthWithCACert: true +# # json object of data that will be encrypted. +# secure_json_data: +# tlsCACert: "..." +# tlsClientCert: "..." +# tlsClientKey: "..." +# version: 1 +# # allow users to edit datasources from the UI. +# editable: false + diff --git a/conf/defaults.ini b/conf/defaults.ini index 404a7950bf3..a145d57482b 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -12,17 +12,17 @@ instance_name = ${HOSTNAME} #################################### Paths ############################### [paths] # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) -# data = data -# + # Directory where grafana can store logs -# logs = data/log -# + # Directory where grafana will automatically scan and look for plugins -# plugins = data/plugins +# Config files containing datasources that will be configured at startup +datasources = conf/datasources + #################################### Server ############################## [server] # Protocol (http, https, socket) @@ -82,6 +82,9 @@ max_idle_conn = 2 # Max conn setting default is 0 (mean not set) max_open_conn = +# Set to true to log the sql calls and execution times. +log_queries = + # For "postgres", use either "disable", "require" or "verify-full" # For "mysql", use either "true", "false", or "skip-verify". ssl_mode = disable @@ -171,6 +174,7 @@ disable_gravatar = false # data source proxy whitelist (ip_or_domain:port separated by spaces) data_source_proxy_whitelist = +#################################### Snapshots ########################### [snapshots] # snapshot sharing options external_enabled = true @@ -183,7 +187,13 @@ snapshot_remove_expired = true # remove snapshots after 90 days snapshot_TTL_days = 90 -#################################### Users #################################### +#################################### Dashboards ################## + +[dashboards] +# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1 +versions_to_keep = 20 + +#################################### Users ############################### [users] # disable user signup / registration allow_sign_up = false @@ -429,7 +439,7 @@ enabled = true execute_alerts = true #################################### Internal Grafana Metrics ############ -# Metrics available at HTTP API Url /api/metrics +# Metrics available at HTTP API Url /metrics [metrics] enabled = true interval_seconds = 10 @@ -485,3 +495,4 @@ public_url = [external_image_storage.gcs] key_file = bucket = +path = \ No newline at end of file diff --git a/conf/sample.ini b/conf/sample.ini index c7d2d1d2695..233a97deef8 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -12,18 +12,17 @@ #################################### Paths #################################### [paths] # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) -# ;data = /var/lib/grafana -# + # Directory where grafana can store logs -# ;logs = /var/log/grafana -# + # Directory where grafana will automatically scan and look for plugins -# ;plugins = /var/lib/grafana/plugins -# +# Config files containing datasources that will be configured at startup +;datasources = conf/datasources + #################################### Server #################################### [server] # Protocol (http, https, socket) @@ -91,6 +90,8 @@ # Max conn setting default is 0 (mean not set) ;max_open_conn = +# Set to true to log the sql calls and execution times. +log_queries = #################################### Session #################################### [session] @@ -161,6 +162,7 @@ # data source proxy whitelist (ip_or_domain:port separated by spaces) ;data_source_proxy_whitelist = +#################################### Snapshots ########################### [snapshots] # snapshot sharing options ;external_enabled = true @@ -173,7 +175,12 @@ # remove snapshots after 90 days ;snapshot_TTL_days = 90 -#################################### Users #################################### +#################################### Dashboards History ################## +[dashboards] +# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1 +;versions_to_keep = 20 + +#################################### Users ############################### [users] # disable user signup / registration ;allow_sign_up = true @@ -373,7 +380,7 @@ ;execute_alerts = true #################################### Internal Grafana Metrics ########################## -# Metrics available at HTTP API Url /api/metrics +# Metrics available at HTTP API Url /metrics [metrics] # Disable / Enable internal metrics ;enabled = true @@ -431,3 +438,4 @@ [external_image_storage.gcs] ;key_file = ;bucket = +;path = \ No newline at end of file diff --git a/docker/blocks/collectd/docker-compose.yaml b/docker/blocks/collectd/docker-compose.yaml new file mode 100644 index 00000000000..c95827f7928 --- /dev/null +++ b/docker/blocks/collectd/docker-compose.yaml @@ -0,0 +1,11 @@ + collectd: + build: blocks/collectd + environment: + HOST_NAME: myserver + GRAPHITE_HOST: graphite + GRAPHITE_PORT: 2003 + GRAPHITE_PREFIX: collectd. + REPORT_BY_CPU: 'false' + COLLECT_INTERVAL: 10 + links: + - graphite diff --git a/docker/blocks/collectd/fig b/docker/blocks/collectd/fig deleted file mode 100644 index 99f45a66d12..00000000000 --- a/docker/blocks/collectd/fig +++ /dev/null @@ -1,11 +0,0 @@ -collectd: - build: blocks/collectd - environment: - HOST_NAME: myserver - GRAPHITE_HOST: graphite - GRAPHITE_PORT: 2003 - GRAPHITE_PREFIX: collectd. - REPORT_BY_CPU: 'false' - COLLECT_INTERVAL: 10 - links: - - graphite diff --git a/docker/blocks/elastic/docker-compose.yaml b/docker/blocks/elastic/docker-compose.yaml new file mode 100644 index 00000000000..193b8f252f6 --- /dev/null +++ b/docker/blocks/elastic/docker-compose.yaml @@ -0,0 +1,8 @@ + elasticsearch: + image: elasticsearch:2.4.1 + command: elasticsearch -Des.network.host=0.0.0.0 + ports: + - "9200:9200" + - "9300:9300" + volumes: + - ./blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml diff --git a/docker/blocks/elastic/fig b/docker/blocks/elastic/fig deleted file mode 100644 index fa79a9af59c..00000000000 --- a/docker/blocks/elastic/fig +++ /dev/null @@ -1,8 +0,0 @@ -elasticsearch: - image: elasticsearch:2.4.1 - command: elasticsearch -Des.network.host=0.0.0.0 - ports: - - "9200:9200" - - "9300:9300" - volumes: - - ./blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml diff --git a/docker/blocks/elastic1/docker-compose.yaml b/docker/blocks/elastic1/docker-compose.yaml new file mode 100644 index 00000000000..518ae76e6ee --- /dev/null +++ b/docker/blocks/elastic1/docker-compose.yaml @@ -0,0 +1,8 @@ + elasticsearch1: + image: elasticsearch:1.7.6 + command: elasticsearch -Des.network.host=0.0.0.0 + ports: + - "11200:9200" + - "11300:9300" + volumes: + - ./blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml diff --git a/docker/blocks/elastic1/fig b/docker/blocks/elastic1/fig deleted file mode 100644 index c33e51f16a2..00000000000 --- a/docker/blocks/elastic1/fig +++ /dev/null @@ -1,8 +0,0 @@ -elasticsearch1: - image: elasticsearch:1.7.6 - command: elasticsearch -Des.network.host=0.0.0.0 - ports: - - "11200:9200" - - "11300:9300" - volumes: - - ./blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml diff --git a/docker/blocks/elastic5/docker-compose.yaml b/docker/blocks/elastic5/docker-compose.yaml new file mode 100644 index 00000000000..5b12be9ada4 --- /dev/null +++ b/docker/blocks/elastic5/docker-compose.yaml @@ -0,0 +1,8 @@ +# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine + + elasticsearch5: + image: elasticsearch:5 + command: elasticsearch + ports: + - "10200:9200" + - "10300:9300" diff --git a/docker/blocks/elastic5/fig b/docker/blocks/elastic5/fig deleted file mode 100644 index 6e5cd89ab3d..00000000000 --- a/docker/blocks/elastic5/fig +++ /dev/null @@ -1,8 +0,0 @@ -# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine - -elasticsearch5: - image: elasticsearch:5 - command: elasticsearch - ports: - - "10200:9200" - - "10300:9300" diff --git a/docker/blocks/graphite/docker-compose.yaml b/docker/blocks/graphite/docker-compose.yaml new file mode 100644 index 00000000000..2bd0dc322cc --- /dev/null +++ b/docker/blocks/graphite/docker-compose.yaml @@ -0,0 +1,16 @@ + graphite: + build: blocks/graphite + ports: + - "8080:80" + - "2003:2003" + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + + fake-graphite-data: + image: grafana/fake-data-gen + network_mode: bridge + environment: + FD_DATASOURCE: graphite + FD_PORT: 2003 + diff --git a/docker/blocks/graphite/fig b/docker/blocks/graphite/fig deleted file mode 100644 index b7e030e388e..00000000000 --- a/docker/blocks/graphite/fig +++ /dev/null @@ -1,16 +0,0 @@ -graphite: - build: blocks/graphite - ports: - - "8080:80" - - "2003:2003" - volumes: - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro - -fake-graphite-data: - image: grafana/fake-data-gen - net: bridge - environment: - FD_DATASOURCE: graphite - FD_PORT: 2003 - diff --git a/docker/blocks/graphite1/Dockerfile b/docker/blocks/graphite1/Dockerfile index a3ab7c4f4af..7bf8b15fa84 100644 --- a/docker/blocks/graphite1/Dockerfile +++ b/docker/blocks/graphite1/Dockerfile @@ -1,9 +1,10 @@ FROM phusion/baseimage:0.9.22 MAINTAINER Denys Zhdanov + RUN apt-get -y update \ && apt-get -y upgrade \ - && apt-get -y --force-yes install vim \ + && apt-get -y install vim \ nginx \ python-dev \ python-flup \ @@ -22,38 +23,67 @@ RUN apt-get -y update \ nodejs \ && rm -rf /var/lib/apt/lists/* +# choose a timezone at build-time +# use `--build-arg CONTAINER_TIMEZONE=Europe/Brussels` in `docker build` +ARG CONTAINER_TIMEZONE +ENV DEBIAN_FRONTEND noninteractive + +RUN if [ ! -z "${CONTAINER_TIMEZONE}" ]; \ + then ln -sf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata; \ + fi + # fix python dependencies (LTS Django and newer memcached/txAMQP) -RUN pip install django==1.8.18 \ +RUN pip install --upgrade pip && \ + pip install django==1.8.18 \ python-memcached==1.53 \ - txAMQP==0.6.2 \ - && pip install --upgrade pip + txAMQP==0.6.2 + +ARG version=1.0.2 +ARG whisper_version=${version} +ARG carbon_version=${version} +ARG graphite_version=${version} + +RUN echo "Building Version: $version" + +ARG whisper_repo=https://github.com/graphite-project/whisper.git +ARG carbon_repo=https://github.com/graphite-project/carbon.git +ARG graphite_repo=https://github.com/graphite-project/graphite-web.git + +ARG statsd_version=v0.8.0 + +ARG statsd_repo=https://github.com/etsy/statsd.git # install whisper -RUN git clone -b 1.0.2 --depth 1 https://github.com/graphite-project/whisper.git /usr/local/src/whisper +RUN git clone -b ${whisper_version} --depth 1 ${whisper_repo} /usr/local/src/whisper WORKDIR /usr/local/src/whisper RUN python ./setup.py install # install carbon -RUN git clone -b 1.0.2 --depth 1 https://github.com/graphite-project/carbon.git /usr/local/src/carbon +RUN git clone -b ${carbon_version} --depth 1 ${carbon_repo} /usr/local/src/carbon WORKDIR /usr/local/src/carbon RUN pip install -r requirements.txt \ && python ./setup.py install # install graphite -RUN git clone -b 1.0.2 --depth 1 https://github.com/graphite-project/graphite-web.git /usr/local/src/graphite-web +RUN git clone -b ${graphite_version} --depth 1 ${graphite_repo} /usr/local/src/graphite-web WORKDIR /usr/local/src/graphite-web RUN pip install -r requirements.txt \ && python ./setup.py install + +# install statsd +RUN git clone -b ${statsd_version} ${statsd_repo} /opt/statsd + +# config graphite ADD conf/opt/graphite/conf/*.conf /opt/graphite/conf/ ADD conf/opt/graphite/webapp/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py -ADD conf/opt/graphite/webapp/graphite/app_settings.py /opt/graphite/webapp/graphite/app_settings.py +# ADD conf/opt/graphite/webapp/graphite/app_settings.py /opt/graphite/webapp/graphite/app_settings.py WORKDIR /opt/graphite/webapp RUN mkdir -p /var/log/graphite/ \ && PYTHONPATH=/opt/graphite/webapp django-admin.py collectstatic --noinput --settings=graphite.settings -# install statsd -RUN git clone -b v0.7.2 https://github.com/etsy/statsd.git /opt/statsd -ADD conf/opt/statsd/config.js /opt/statsd/config.js +# config statsd +ADD conf/opt/statsd/config.js /opt/statsd/ # config nginx RUN rm /etc/nginx/sites-enabled/default @@ -63,8 +93,7 @@ ADD conf/etc/nginx/sites-enabled/graphite-statsd.conf /etc/nginx/sites-enabled/g # init django admin ADD conf/usr/local/bin/django_admin_init.exp /usr/local/bin/django_admin_init.exp ADD conf/usr/local/bin/manage.sh /usr/local/bin/manage.sh -RUN chmod +x /usr/local/bin/manage.sh \ - && /usr/local/bin/django_admin_init.exp +RUN chmod +x /usr/local/bin/manage.sh && /usr/local/bin/django_admin_init.exp # logging support RUN mkdir -p /var/log/carbon /var/log/graphite /var/log/nginx @@ -86,8 +115,10 @@ RUN apt-get clean\ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # defaults -EXPOSE 80 2003-2004 2023-2024 8125/udp 8126 +EXPOSE 80 2003-2004 2023-2024 8125 8125/udp 8126 VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/etc/nginx", "/opt/statsd", "/etc/logrotate.d", "/var/log"] WORKDIR / ENV HOME /root +ENV STATSD_INTERFACE udp + CMD ["/sbin/my_init"] diff --git a/docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh b/docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh index edd642a9c1b..0937f78cd6a 100755 --- a/docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh +++ b/docker/blocks/graphite1/conf/etc/my_init.d/01_conf_init.sh @@ -12,7 +12,7 @@ graphite_conf_dir_contents=$(find /opt/graphite/conf -mindepth 1 -print -quit) graphite_webapp_dir_contents=$(find /opt/graphite/webapp/graphite -mindepth 1 -print -quit) graphite_storage_dir_contents=$(find /opt/graphite/storage -mindepth 1 -print -quit) if [[ -z $graphite_dir_contents ]]; then - git clone -b 1.0.2 --depth 1 https://github.com/graphite-project/graphite-web.git /usr/local/src/graphite-web + # git clone -b 1.0.2 --depth 1 https://github.com/graphite-project/graphite-web.git /usr/local/src/graphite-web cd /usr/local/src/graphite-web && python ./setup.py install fi if [[ -z $graphite_storage_dir_contents ]]; then diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf b/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf index eaf6a938f27..3e10dcec9cf 100644 --- a/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf +++ b/docker/blocks/graphite1/conf/opt/graphite/conf/carbon.conf @@ -8,18 +8,18 @@ # Defaults to ../ # GRAPHITE_CONF_DIR - Configuration directory (where this file lives). # Defaults to $GRAPHITE_ROOT/conf/ -# GRAPHITE_STORAGE_DIR - Storage directory for whipser/rrd/log/pid files. +# GRAPHITE_STORAGE_DIR - Storage directory for whisper/rrd/log/pid files. # Defaults to $GRAPHITE_ROOT/storage/ # # To change other directory paths, add settings to this file. The following # configuration variables are available with these default values: # # STORAGE_DIR = $GRAPHITE_STORAGE_DIR -# LOCAL_DATA_DIR = STORAGE_DIR/whisper/ -# WHITELISTS_DIR = STORAGE_DIR/lists/ -# CONF_DIR = STORAGE_DIR/conf/ -# LOG_DIR = STORAGE_DIR/log/ -# PID_DIR = STORAGE_DIR/ +# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ +# WHITELISTS_DIR = %(STORAGE_DIR)s/lists/ +# CONF_DIR = %(STORAGE_DIR)s/conf/ +# LOG_DIR = %(STORAGE_DIR)s/log/ +# PID_DIR = %(STORAGE_DIR)s/ # # For FHS style directory structures, use: # @@ -30,20 +30,30 @@ # #LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ -# Enable daily log rotation. If disabled, a kill -HUP can be used after a manual rotate +# Specify the database library used to store metric data on disk. Each database +# may have configurable options to change the behaviour of how it writes to +# persistent storage. +# +# whisper - Fixed-size database, similar in design and purpose to RRD. This is +# the default storage backend for carbon and the most rigorously tested. +# +# ceres - Experimental alternative database that supports storing data in sparse +# files of arbitrary fixed-size resolutions. +DATABASE = whisper + +# Enable daily log rotation. If disabled, a new file will be opened whenever the log file path no +# longer exists (i.e. it is removed or renamed) ENABLE_LOGROTATION = True # Specify the user to drop privileges to -# If this is blank carbon runs as the user that invokes it +# If this is blank carbon-cache runs as the user that invokes it # This user must have write access to the local data directory USER = -# -# NOTE: The above settings must be set under [relay] and [aggregator] -# to take effect for those daemons as well # Limit the size of the cache to avoid swapping or becoming CPU bound. # Sorts and serving cache queries gets more expensive as the cache grows. # Use the value "inf" (infinity) for an unlimited cache size. +# value should be an integer number of metric datapoints. MAX_CACHE_SIZE = inf # Limits the number of whisper update_many() calls per second, which effectively @@ -60,14 +70,30 @@ MAX_UPDATES_PER_SECOND = 500 # MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000 # Softly limits the number of whisper files that get created each minute. -# Setting this value low (like at 50) is a good way to ensure your graphite +# Setting this value low (e.g. 50) is a good way to ensure that your carbon # system will not be adversely impacted when a bunch of new metrics are -# sent to it. The trade off is that it will take much longer for those metrics' -# database files to all get created and thus longer until the data becomes usable. -# Setting this value high (like "inf" for infinity) will cause graphite to create -# the files quickly but at the risk of slowing I/O down considerably for a while. +# sent to it. The trade off is that any metrics received in excess of this +# value will be silently dropped, and the whisper file will not be created +# until such point as a subsequent metric is received and fits within the +# defined rate limit. Setting this value high (like "inf" for infinity) will +# cause carbon to create the files quickly but at the risk of increased I/O. MAX_CREATES_PER_MINUTE = 50 +# Set the minimum timestamp resolution supported by this instance. This allows +# internal optimisations by overwriting points with equal truncated timestamps +# in order to limit the number of updates to the database. It defaults to one +# second. +MIN_TIMESTAMP_RESOLUTION = 1 + +# Set the minimum lag in seconds for a point to be written to the database +# in order to optimize batching. This means that each point will wait at least +# the duration of this lag before being written. Setting this to 0 disable the feature. +# This currently only works when using the timesorted write strategy. +# MIN_TIMESTAMP_LAG = 0 + +# Set the interface and port for the line (plain text) listener. Setting the +# interface to 0.0.0.0 listens on all interfaces. Port can be set to 0 to +# disable this listener if it is not required. LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2003 @@ -78,11 +104,23 @@ ENABLE_UDP_LISTENER = False UDP_RECEIVER_INTERFACE = 0.0.0.0 UDP_RECEIVER_PORT = 2003 +# Set the interface and port for the pickle listener. Setting the interface to +# 0.0.0.0 listens on all interfaces. Port can be set to 0 to disable this +# listener if it is not required. PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2004 -# Set to false to disable logging of successful connections -LOG_LISTENER_CONNECTIONS = True +# Set the interface and port for the protobuf listener. Setting the interface to +# 0.0.0.0 listens on all interfaces. Port can be set to 0 to disable this +# listener if it is not required. +# PROTOBUF_RECEIVER_INTERFACE = 0.0.0.0 +# PROTOBUF_RECEIVER_PORT = 2005 + +# Limit the number of open connections the receiver can handle as any time. +# Default is no limit. Setting up a limit for sites handling high volume +# traffic may be recommended to avoid running out of TCP memory or having +# thousands of TCP connections reduce the throughput of the service. +#MAX_RECEIVER_CONNECTIONS = inf # Per security concerns outlined in Bug #817247 the pickle receiver # will use a more secure and slightly less efficient unpickler. @@ -98,13 +136,19 @@ CACHE_QUERY_PORT = 7002 # data until the cache size falls below 95% MAX_CACHE_SIZE. USE_FLOW_CONTROL = True -# By default, carbon-cache will log every whisper update and cache hit. This can be excessive and -# degrade performance if logging on the same volume as the whisper data is stored. -LOG_UPDATES = False -LOG_CACHE_HITS = False -LOG_CACHE_QUEUE_SORTS = True +# If enabled this setting is used to timeout metric client connection if no +# metrics have been sent in specified time in seconds +#METRIC_CLIENT_IDLE_TIMEOUT = None -# The thread that writes metrics to disk can use on of the following strategies +# By default, carbon-cache will log every whisper update and cache hit. +# This can be excessive and degrade performance if logging on the same +# volume as the whisper data is stored. +LOG_UPDATES = False +LOG_CREATES = False +LOG_CACHE_HITS = False +LOG_CACHE_QUEUE_SORTS = False + +# The thread that writes metrics to disk can use one of the following strategies # determining the order in which metrics are removed from cache and flushed to # disk. The default option preserves the same behavior as has been historically # available in version 0.9.10. @@ -114,6 +158,12 @@ LOG_CACHE_QUEUE_SORTS = True # moment of the list's creation. Metrics will then be flushed from the cache to # disk in that order. # +# timesorted - All metrics in the list will be looked at and sorted according +# to the timestamp of there datapoints. The metric that were the least recently +# written will be written first. This is an hybrid strategy between max and +# sorted which is particularly adapted to sets of metrics with non-uniform +# resolutions. +# # max - The writer thread will always pop and flush the metric from cache # that has the most datapoints. This will give a strong flush preference to # frequently updated metrics and will also reduce random file-io. Infrequently @@ -152,12 +202,61 @@ WHISPER_FALLOCATE_CREATE = True # Enabling this option will cause Whisper to lock each Whisper file it writes # to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when -# multiple carbon-cache daemons are writing to the same files +# multiple carbon-cache daemons are writing to the same files. # WHISPER_LOCK_WRITES = False +# On systems which has a large number of metrics, an amount of Whisper write(2)'s +# pageback sometimes cause disk thrashing due to memory shortage, so that abnormal +# disk reads occur. Enabling this option makes it possible to decrease useless +# page cache memory by posix_fadvise(2) with POSIX_FADVISE_RANDOM option. +# WHISPER_FADVISE_RANDOM = False + +# By default all nodes stored in Ceres are cached in memory to improve the +# throughput of reads and writes to underlying slices. Turning this off will +# greatly reduce memory consumption for databases with millions of metrics, at +# the cost of a steep increase in disk i/o, approximately an extra two os.stat +# calls for every read and write. Reasons to do this are if the underlying +# storage can handle stat() with practically zero cost (SSD, NVMe, zRAM). +# Valid values are: +# all - all nodes are cached +# none - node caching is disabled +# CERES_NODE_CACHING_BEHAVIOR = all + +# Ceres nodes can have many slices and caching the right ones can improve +# performance dramatically. Note that there are many trade-offs to tinkering +# with this, and unless you are a ceres developer you *really* should not +# mess with this. Valid values are: +# latest - only the most recent slice is cached +# all - all slices are cached +# none - slice caching is disabled +# CERES_SLICE_CACHING_BEHAVIOR = latest + +# If a Ceres node accumulates too many slices, performance can suffer. +# This can be caused by intermittently reported data. To mitigate +# slice fragmentation there is a tolerance for how much space can be +# wasted within a slice file to avoid creating a new one. That tolerance +# level is determined by MAX_SLICE_GAP, which is the number of consecutive +# null datapoints allowed in a slice file. +# If you set this very low, you will waste less of the *tiny* bit disk space +# that this feature wastes, and you will be prone to performance problems +# caused by slice fragmentation, which can be pretty severe. +# If you set this really high, you will waste a bit more disk space (each +# null datapoint wastes 8 bytes, but keep in mind your filesystem's block +# size). If you suffer slice fragmentation issues, you should increase this or +# run the ceres-maintenance defrag plugin more often. However you should not +# set it to be huge because then if a large but allowed gap occurs it has to +# get filled in, which means instead of a simple 8-byte write to a new file we +# could end up doing an (8 * MAX_SLICE_GAP)-byte write to the latest slice. +# CERES_MAX_SLICE_GAP = 80 + +# Enabling this option will cause Ceres to lock each Ceres file it writes to +# to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when +# multiple carbon-cache daemons are writing to the same files. +# CERES_LOCK_WRITES = False + # Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or -# empty, all metrics will pass through +# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is +# missing or empty, all metrics will pass through # USE_WHITELIST = False # By default, carbon itself will log statistics (such as a count, @@ -203,16 +302,25 @@ WHISPER_FALLOCATE_CREATE = True # Example: store everything # BIND_PATTERNS = # +# URL of graphite-web instance, this is used to add incoming series to the tag database +GRAPHITE_URL = http://127.0.0.1:80 + +# Tag update interval, this specifies how frequently updates to existing series will trigger +# an update to the tag index, the default setting is once every 100 updates +# TAG_UPDATE_INTERVAL = 100 + # To configure special settings for the carbon-cache instance 'b', uncomment this: #[cache:b] #LINE_RECEIVER_PORT = 2103 #PICKLE_RECEIVER_PORT = 2104 #CACHE_QUERY_PORT = 7102 # and any other settings you want to customize, defaults are inherited -# from [carbon] section. +# from the [cache] section. # You can then specify the --instance=b option to manage this instance - - +# +# In order to turn off logging of successful connections for the line +# receiver, set this to False +# LOG_LISTENER_CONN_SUCCESS = True [relay] LINE_RECEIVER_INTERFACE = 0.0.0.0 @@ -220,9 +328,6 @@ LINE_RECEIVER_PORT = 2013 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2014 -# Set to false to disable logging of successful connections -LOG_LISTENER_CONNECTIONS = True - # Carbon-relay has several options for metric routing controlled by RELAY_METHOD # # Use relay-rules.conf to route metrics to destinations based on pattern rules @@ -237,12 +342,24 @@ LOG_LISTENER_CONNECTIONS = True # instance. # Enable this for carbon-relays that send to a group of carbon-aggregators #RELAY_METHOD = aggregated-consistent-hashing +# +# You can also use fast-hashing and fast-aggregated-hashing which are in O(1) +# and will always redirect the metrics to the same destination but do not try +# to minimize rebalancing when the list of destinations is changing. RELAY_METHOD = rules # If you use consistent-hashing you can add redundancy by replicating every # datapoint to more than one machine. REPLICATION_FACTOR = 1 +# For REPLICATION_FACTOR >=2, set DIVERSE_REPLICAS to True to guarantee replicas +# across distributed hosts. With this setting disabled, it's possible that replicas +# may be sent to different caches on the same host. This has been the default +# behavior since introduction of 'consistent-hashing' relay method. +# Note that enabling this on an existing pre-0.9.14 cluster will require rebalancing +# your metrics across the cluster nodes using a tool like Carbonate. +#DIVERSE_REPLICAS = True + # This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single # carbon-cache instance on the default port. However if you @@ -261,20 +378,71 @@ REPLICATION_FACTOR = 1 # must be defined in this list DESTINATIONS = 127.0.0.1:2004 -# This defines the maximum "message size" between carbon daemons. -# You shouldn't need to tune this unless you really know what you're doing. -MAX_DATAPOINTS_PER_MESSAGE = 500 +# This define the protocol to use to contact the destination. It can be +# set to one of "line", "pickle", "udp" and "protobuf". This list can be +# extended with CarbonClientFactory plugins and defaults to "pickle". +# DESTINATION_PROTOCOL = pickle + +# When using consistent hashing it sometime makes sense to make +# the ring dynamic when you don't want to loose points when a +# single destination is down. Replication is an answer to that +# but it can be quite expensive. +# DYNAMIC_ROUTER = False + +# Controls the number of connection attempts before marking a +# destination as down. We usually do one connection attempt per +# second. +# DYNAMIC_ROUTER_MAX_RETRIES = 5 + +# This is the maximum number of datapoints that can be queued up +# for a single destination. Once this limit is hit, we will +# stop accepting new data if USE_FLOW_CONTROL is True, otherwise +# we will drop any subsequently received datapoints. MAX_QUEUE_SIZE = 10000 +# This defines the maximum "message size" between carbon daemons. If +# your queue is large, setting this to a lower number will cause the +# relay to forward smaller discrete chunks of stats, which may prevent +# overloading on the receiving side after a disconnect. +MAX_DATAPOINTS_PER_MESSAGE = 500 + +# Limit the number of open connections the receiver can handle as any time. +# Default is no limit. Setting up a limit for sites handling high volume +# traffic may be recommended to avoid running out of TCP memory or having +# thousands of TCP connections reduce the throughput of the service. +#MAX_RECEIVER_CONNECTIONS = inf + +# Specify the user to drop privileges to +# If this is blank carbon-relay runs as the user that invokes it +# USER = + +# This is the percentage that the queue must be empty before it will accept +# more messages. For a larger site, if the queue is very large it makes sense +# to tune this to allow for incoming stats. So if you have an average +# flow of 100k stats/minute, and a MAX_QUEUE_SIZE of 3,000,000, it makes sense +# to allow stats to start flowing when you've cleared the queue to 95% since +# you should have space to accommodate the next minute's worth of stats +# even before the relay incrementally clears more of the queue +QUEUE_LOW_WATERMARK_PCT = 0.8 + +# To allow for batch efficiency from the pickle protocol and to benefit from +# other batching advantages, all writes are deferred by putting them into a queue, +# and then the queue is flushed and sent a small fraction of a second later. +TIME_TO_DEFER_SENDING = 0.0001 + # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting -# data until the send queues fall below 80% MAX_QUEUE_SIZE. +# data until the send queues fall below QUEUE_LOW_WATERMARK_PCT * MAX_QUEUE_SIZE. USE_FLOW_CONTROL = True +# If enabled this setting is used to timeout metric client connection if no +# metrics have been sent in specified time in seconds +#METRIC_CLIENT_IDLE_TIMEOUT = None + # Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or -# empty, all metrics will pass through +# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is +# missing or empty, all metrics will pass through # USE_WHITELIST = False # By default, carbon itself will log statistics (such as a count, @@ -282,7 +450,40 @@ USE_FLOW_CONTROL = True # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation # CARBON_METRIC_PREFIX = carbon # CARBON_METRIC_INTERVAL = 60 +# +# In order to turn off logging of successful connections for the line +# receiver, set this to False +# LOG_LISTENER_CONN_SUCCESS = True +# If you're connecting from the relay to a destination that's over the +# internet or similarly iffy connection, a backlog can develop because +# of internet weather conditions, e.g. acks getting lost or similar issues. +# To deal with that, you can enable USE_RATIO_RESET which will let you +# re-set the connection to an individual destination. Defaults to being off. +USE_RATIO_RESET=False + +# When there is a small number of stats flowing, it's not desirable to +# perform any actions based on percentages - it's just too "twitchy". +MIN_RESET_STAT_FLOW=1000 + +# When the ratio of stats being sent in a reporting interval is far +# enough from 1.0, we will disconnect the socket and reconnecto to +# clear out queued stats. The default ratio of 0.9 indicates that 10% +# of stats aren't being delivered within one CARBON_METRIC_INTERVAL +# (default of 60 seconds), which can lead to a queue backup. Under +# some circumstances re-setting the connection can fix this, so +# set this according to your tolerance, and look in the logs for +# "resetConnectionForQualityReasons" to observe whether this is kicking +# in when your sent queue is building up. +MIN_RESET_RATIO=0.9 + +# The minimum time between resets. When a connection is re-set, we +# need to wait before another reset is performed. +# (2*CARBON_METRIC_INTERVAL) + 1 second is the minimum time needed +# before stats for the new connection will be available. Setting this +# below (2*CARBON_METRIC_INTERVAL) + 1 second will result in a lot of +# reset connections for no good reason. +MIN_RESET_INTERVAL=121 [aggregator] LINE_RECEIVER_INTERFACE = 0.0.0.0 @@ -291,14 +492,17 @@ LINE_RECEIVER_PORT = 2023 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2024 -# Set to false to disable logging of successful connections -LOG_LISTENER_CONNECTIONS = True - # If set true, metric received will be forwarded to DESTINATIONS in addition to # the output of the aggregation rules. If set false the carbon-aggregator will # only ever send the output of aggregation. FORWARD_ALL = True +# Filenames of the configuration files to use for this instance of aggregator. +# Filenames are relative to CONF_DIR. +# +# AGGREGATION_RULES = aggregation-rules.conf +# REWRITE_RULES = rewrite-rules.conf + # This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single # carbon-cache instance on the default port. However if you @@ -330,6 +534,10 @@ MAX_QUEUE_SIZE = 10000 # data until the send queues fall below 80% MAX_QUEUE_SIZE. USE_FLOW_CONTROL = True +# If enabled this setting is used to timeout metric client connection if no +# metrics have been sent in specified time in seconds +#METRIC_CLIENT_IDLE_TIMEOUT = None + # This defines the maximum "message size" between carbon daemons. # You shouldn't need to tune this unless you really know what you're doing. MAX_DATAPOINTS_PER_MESSAGE = 500 @@ -339,6 +547,12 @@ MAX_DATAPOINTS_PER_MESSAGE = 500 # the past MAX_AGGREGATION_INTERVALS * intervalSize seconds. MAX_AGGREGATION_INTERVALS = 5 +# Limit the number of open connections the receiver can handle as any time. +# Default is no limit. Setting up a limit for sites handling high volume +# traffic may be recommended to avoid running out of TCP memory or having +# thousands of TCP connections reduce the throughput of the service. +#MAX_RECEIVER_CONNECTIONS = inf + # By default (WRITE_BACK_FREQUENCY = 0), carbon-aggregator will write back # aggregated data points once every rule.frequency seconds, on a per-rule basis. # Set this (WRITE_BACK_FREQUENCY = N) to write back all aggregated data points @@ -348,8 +562,8 @@ MAX_AGGREGATION_INTERVALS = 5 # WRITE_BACK_FREQUENCY = 0 # Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or -# empty, all metrics will pass through +# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is +# missing or empty, all metrics will pass through # USE_WHITELIST = False # By default, carbon itself will log statistics (such as a count, @@ -357,3 +571,24 @@ MAX_AGGREGATION_INTERVALS = 5 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation # CARBON_METRIC_PREFIX = carbon # CARBON_METRIC_INTERVAL = 60 + +# In order to turn off logging of successful connections for the line +# receiver, set this to False +# LOG_LISTENER_CONN_SUCCESS = True + +# In order to turn off logging of metrics with no corresponding +# aggregation rules receiver, set this to False +# LOG_AGGREGATOR_MISSES = False + +# Specify the user to drop privileges to +# If this is blank carbon-aggregator runs as the user that invokes it +# USER = + +# Part of the code, and particularly aggregator rules, need +# to cache metric names. To avoid leaking too much memory you +# can tweak the size of this cache. The default allow for 1M +# different metrics per rule (~200MiB). +# CACHE_METRIC_NAMES_MAX=1000000 + +# You can optionally set a ttl to this cache. +# CACHE_METRIC_NAMES_TTL=600 diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf b/docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf index ddadd3667bc..80a384285af 100644 --- a/docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf +++ b/docker/blocks/graphite1/conf/opt/graphite/conf/storage-aggregation.conf @@ -40,4 +40,3 @@ aggregationMethod = sum pattern = .* xFilesFactor = 0.3 aggregationMethod = average - diff --git a/docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf b/docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf index 4623058d3d8..d5300a3b22d 100644 --- a/docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf +++ b/docker/blocks/graphite1/conf/opt/graphite/conf/storage-schemas.conf @@ -1,4 +1,23 @@ # Schema definitions for Whisper files. Entries are scanned in order, +# and first match wins. This file is scanned for changes every 60 seconds. +# +# Definition Syntax: +# +# [name] +# pattern = regex +# retentions = timePerPoint:timeToStore, timePerPoint:timeToStore, ... +# +# Remember: To support accurate aggregation from higher to lower resolution +# archives, the precision of a longer retention archive must be +# cleanly divisible by precision of next lower retention archive. +# +# Valid: 60s:7d,300s:30d (300/60 = 5) +# Invalid: 180s:7d,300s:30d (300/180 = 3.333) +# + +# Carbon's internal metrics. This entry should match what is specified in +# CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings + [carbon] pattern = ^carbon\..* retentions = 1m:31d,10m:1y,1h:5y diff --git a/docker/blocks/graphite1/conf/usr/local/bin/manage.sh b/docker/blocks/graphite1/conf/usr/local/bin/manage.sh old mode 100644 new mode 100755 index a94fbcd3615..8915907a3ac --- a/docker/blocks/graphite1/conf/usr/local/bin/manage.sh +++ b/docker/blocks/graphite1/conf/usr/local/bin/manage.sh @@ -1,3 +1,3 @@ #!/bin/bash PYTHONPATH=/opt/graphite/webapp django-admin.py syncdb --settings=graphite.settings -PYTHONPATH=/opt/graphite/webapp django-admin.py update_users --settings=graphite.settings \ No newline at end of file +# PYTHONPATH=/opt/graphite/webapp django-admin.py update_users --settings=graphite.settings \ No newline at end of file diff --git a/docker/blocks/graphite1/docker-compose.yaml b/docker/blocks/graphite1/docker-compose.yaml new file mode 100644 index 00000000000..cd10593f423 --- /dev/null +++ b/docker/blocks/graphite1/docker-compose.yaml @@ -0,0 +1,21 @@ + graphite: + build: + context: blocks/graphite1 + args: + version: master + ports: + - "8080:80" + - "2003:2003" + - "8125:8125/udp" + - "8126:8126" + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + + fake-graphite-data: + image: grafana/fake-data-gen + network_mode: bridge + environment: + FD_DATASOURCE: graphite + FD_PORT: 2003 + diff --git a/docker/blocks/graphite1/fig b/docker/blocks/graphite1/fig deleted file mode 100644 index 5337376ff5c..00000000000 --- a/docker/blocks/graphite1/fig +++ /dev/null @@ -1,16 +0,0 @@ -graphite: - build: blocks/graphite1 - ports: - - "8080:80" - - "2003:2003" - volumes: - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro - -fake-graphite-data: - image: grafana/fake-data-gen - net: bridge - environment: - FD_DATASOURCE: graphite - FD_PORT: 2003 - diff --git a/docker/blocks/graphite1/files/carbon.conf b/docker/blocks/graphite1/files/carbon.conf deleted file mode 100644 index 50762b3fff5..00000000000 --- a/docker/blocks/graphite1/files/carbon.conf +++ /dev/null @@ -1,76 +0,0 @@ -[cache] -LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ - -# Specify the user to drop privileges to -# If this is blank carbon runs as the user that invokes it -# This user must have write access to the local data directory -USER = - -# Limit the size of the cache to avoid swapping or becoming CPU bound. -# Sorts and serving cache queries gets more expensive as the cache grows. -# Use the value "inf" (infinity) for an unlimited cache size. -MAX_CACHE_SIZE = inf - -# Limits the number of whisper update_many() calls per second, which effectively -# means the number of write requests sent to the disk. This is intended to -# prevent over-utilizing the disk and thus starving the rest of the system. -# When the rate of required updates exceeds this, then carbon's caching will -# take effect and increase the overall throughput accordingly. -MAX_UPDATES_PER_SECOND = 1000 - -# Softly limits the number of whisper files that get created each minute. -# Setting this value low (like at 50) is a good way to ensure your graphite -# system will not be adversely impacted when a bunch of new metrics are -# sent to it. The trade off is that it will take much longer for those metrics' -# database files to all get created and thus longer until the data becomes usable. -# Setting this value high (like "inf" for infinity) will cause graphite to create -# the files quickly but at the risk of slowing I/O down considerably for a while. -MAX_CREATES_PER_MINUTE = inf - -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2003 - -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2004 - -CACHE_QUERY_INTERFACE = 0.0.0.0 -CACHE_QUERY_PORT = 7002 - -LOG_UPDATES = False - -# Enable AMQP if you want to receve metrics using an amqp broker -# ENABLE_AMQP = False - -# Verbose means a line will be logged for every metric received -# useful for testing -# AMQP_VERBOSE = False - -# AMQP_HOST = localhost -# AMQP_PORT = 5672 -# AMQP_VHOST = / -# AMQP_USER = guest -# AMQP_PASSWORD = guest -# AMQP_EXCHANGE = graphite - -# Patterns for all of the metrics this machine will store. Read more at -# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings -# -# Example: store all sales, linux servers, and utilization metrics -# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization -# -# Example: store everything -# BIND_PATTERNS = # - -# NOTE: you cannot run both a cache and a relay on the same server -# with the default configuration, you have to specify a distinict -# interfaces and ports for the listeners. - -[relay] -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2003 - -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2004 - -CACHE_SERVERS = server1, server2, server3 -MAX_QUEUE_SIZE = 10000 diff --git a/docker/blocks/graphite1/files/events_views.py b/docker/blocks/graphite1/files/events_views.py deleted file mode 100644 index 6c12987af3c..00000000000 --- a/docker/blocks/graphite1/files/events_views.py +++ /dev/null @@ -1,102 +0,0 @@ -import datetime -import time - -from django.utils.timezone import get_current_timezone -from django.core.urlresolvers import get_script_prefix -from django.http import HttpResponse -from django.shortcuts import render_to_response, get_object_or_404 -from pytz import timezone - -from graphite.util import json -from graphite.events import models -from graphite.render.attime import parseATTime - - -def to_timestamp(dt): - return time.mktime(dt.timetuple()) - - -class EventEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, datetime.datetime): - return to_timestamp(obj) - return json.JSONEncoder.default(self, obj) - - -def view_events(request): - if request.method == "GET": - context = { 'events' : fetch(request), - 'slash' : get_script_prefix() - } - return render_to_response("events.html", context) - else: - return post_event(request) - -def detail(request, event_id): - e = get_object_or_404(models.Event, pk=event_id) - context = { 'event' : e, - 'slash' : get_script_prefix() - } - return render_to_response("event.html", context) - - -def post_event(request): - if request.method == 'POST': - event = json.loads(request.body) - assert isinstance(event, dict) - - values = {} - values["what"] = event["what"] - values["tags"] = event.get("tags", None) - values["when"] = datetime.datetime.fromtimestamp( - event.get("when", time.time())) - if "data" in event: - values["data"] = event["data"] - - e = models.Event(**values) - e.save() - - return HttpResponse(status=200) - else: - return HttpResponse(status=405) - -def get_data(request): - if 'jsonp' in request.REQUEST: - response = HttpResponse( - "%s(%s)" % (request.REQUEST.get('jsonp'), - json.dumps(fetch(request), cls=EventEncoder)), - mimetype='text/javascript') - else: - response = HttpResponse( - json.dumps(fetch(request), cls=EventEncoder), - mimetype="application/json") - return response - -def fetch(request): - #XXX we need to move to USE_TZ=True to get rid of naive-time conversions - def make_naive(dt): - if 'tz' in request.GET: - tz = timezone(request.GET['tz']) - else: - tz = get_current_timezone() - local_dt = dt.astimezone(tz) - if hasattr(local_dt, 'normalize'): - local_dt = local_dt.normalize() - return local_dt.replace(tzinfo=None) - - if request.GET.get("from", None) is not None: - time_from = make_naive(parseATTime(request.GET["from"])) - else: - time_from = datetime.datetime.fromtimestamp(0) - - if request.GET.get("until", None) is not None: - time_until = make_naive(parseATTime(request.GET["until"])) - else: - time_until = datetime.datetime.now() - - tags = request.GET.get("tags", None) - if tags is not None: - tags = request.GET.get("tags").split(" ") - - return [x.as_dict() for x in - models.Event.find_events(time_from, time_until, tags=tags)] diff --git a/docker/blocks/graphite1/files/initial_data.json b/docker/blocks/graphite1/files/initial_data.json deleted file mode 100644 index b3ac9b1ebb0..00000000000 --- a/docker/blocks/graphite1/files/initial_data.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "pk": 1, - "model": "auth.user", - "fields": { - "username": "admin", - "first_name": "", - "last_name": "", - "is_active": true, - "is_superuser": true, - "is_staff": true, - "last_login": "2011-09-20 17:02:14", - "groups": [], - "user_permissions": [], - "password": "sha1$1b11b$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236", - "email": "root@example.com", - "date_joined": "2011-09-20 17:02:14" - } - } -] diff --git a/docker/blocks/graphite1/files/local_settings.py b/docker/blocks/graphite1/files/local_settings.py deleted file mode 100644 index 177d674e9dd..00000000000 --- a/docker/blocks/graphite1/files/local_settings.py +++ /dev/null @@ -1,42 +0,0 @@ -# Edit this file to override the default graphite settings, do not edit settings.py - -# Turn on debugging and restart apache if you ever see an "Internal Server Error" page -#DEBUG = True - -# Set your local timezone (django will try to figure this out automatically) -TIME_ZONE = 'UTC' - -# Setting MEMCACHE_HOSTS to be empty will turn off use of memcached entirely -#MEMCACHE_HOSTS = ['127.0.0.1:11211'] - -# Sometimes you need to do a lot of rendering work but cannot share your storage mount -#REMOTE_RENDERING = True -#RENDERING_HOSTS = ['fastserver01','fastserver02'] -#LOG_RENDERING_PERFORMANCE = True -#LOG_CACHE_PERFORMANCE = True - -# If you've got more than one backend server they should all be listed here -#CLUSTER_SERVERS = [] - -# Override this if you need to provide documentation specific to your graphite deployment -#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite" - -# Enable email-related features -#SMTP_SERVER = "mail.mycompany.com" - -# LDAP / ActiveDirectory authentication setup -#USE_LDAP_AUTH = True -#LDAP_SERVER = "ldap.mycompany.com" -#LDAP_PORT = 389 -#LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com" -#LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com" -#LDAP_BASE_PASS = "readonly_account_password" -#LDAP_USER_QUERY = "(username=%s)" #For Active Directory use "(sAMAccountName=%s)" - -# If sqlite won't cut it, configure your real database here (don't forget to run manage.py syncdb!) -#DATABASE_ENGINE = 'mysql' # or 'postgres' -#DATABASE_NAME = 'graphite' -#DATABASE_USER = 'graphite' -#DATABASE_PASSWORD = 'graphite-is-awesome' -#DATABASE_HOST = 'mysql.mycompany.com' -#DATABASE_PORT = '3306' diff --git a/docker/blocks/graphite1/files/my_htpasswd b/docker/blocks/graphite1/files/my_htpasswd deleted file mode 100644 index 52a72d01b4c..00000000000 --- a/docker/blocks/graphite1/files/my_htpasswd +++ /dev/null @@ -1 +0,0 @@ -grafana:$apr1$4R/20xhC$8t37jPP5dbcLr48btdkU// diff --git a/docker/blocks/graphite1/files/nginx.conf b/docker/blocks/graphite1/files/nginx.conf deleted file mode 100644 index 5b5c562ce9d..00000000000 --- a/docker/blocks/graphite1/files/nginx.conf +++ /dev/null @@ -1,70 +0,0 @@ -daemon off; -user www-data; -worker_processes 1; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - server_tokens off; - - server_names_hash_bucket_size 32; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - gzip on; - gzip_disable "msie6"; - - server { - listen 80 default_server; - server_name _; - - open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m; - - location / { - proxy_pass http://127.0.0.1:8000; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header Host $host; - - client_max_body_size 10m; - client_body_buffer_size 128k; - - proxy_connect_timeout 90; - proxy_send_timeout 90; - proxy_read_timeout 90; - - proxy_buffer_size 4k; - proxy_buffers 4 32k; - proxy_busy_buffers_size 64k; - proxy_temp_file_write_size 64k; - } - - add_header Access-Control-Allow-Origin "*"; - add_header Access-Control-Allow-Methods "GET, OPTIONS"; - add_header Access-Control-Allow-Headers "origin, authorization, accept"; - - location /content { - alias /opt/graphite/webapp/content; - - } - - location /media { - alias /usr/share/pyshared/django/contrib/admin/media; - } - } -} diff --git a/docker/blocks/graphite1/files/statsd_config.js b/docker/blocks/graphite1/files/statsd_config.js deleted file mode 100644 index a03003dcd79..00000000000 --- a/docker/blocks/graphite1/files/statsd_config.js +++ /dev/null @@ -1,8 +0,0 @@ -{ - graphitePort: 2003, - graphiteHost: "127.0.0.1", - port: 8125, - mgmt_port: 8126, - backends: ['./backends/graphite'], - debug: true -} diff --git a/docker/blocks/graphite1/files/storage-aggregation.conf b/docker/blocks/graphite1/files/storage-aggregation.conf deleted file mode 100644 index 5c4bc19fde1..00000000000 --- a/docker/blocks/graphite1/files/storage-aggregation.conf +++ /dev/null @@ -1,19 +0,0 @@ -[min] -pattern = \.min$ -xFilesFactor = 0.1 -aggregationMethod = min - -[max] -pattern = \.max$ -xFilesFactor = 0.1 -aggregationMethod = max - -[sum] -pattern = \.count$ -xFilesFactor = 0 -aggregationMethod = sum - -[default_average] -pattern = .* -xFilesFactor = 0.5 -aggregationMethod = average diff --git a/docker/blocks/graphite1/files/storage-schemas.conf b/docker/blocks/graphite1/files/storage-schemas.conf deleted file mode 100644 index 7e28d3a6649..00000000000 --- a/docker/blocks/graphite1/files/storage-schemas.conf +++ /dev/null @@ -1,16 +0,0 @@ -[carbon] -pattern = ^carbon\..* -retentions = 1m:31d,10m:1y,1h:5y - -[highres] -pattern = ^highres.* -retentions = 1s:1d,1m:7d - -[statsd] -pattern = ^statsd.* -retentions = 1m:7d,10m:1y - -[default] -pattern = .* -retentions = 10s:1d,1m:7d,10m:1y - diff --git a/docker/blocks/graphite1/files/supervisord.conf b/docker/blocks/graphite1/files/supervisord.conf deleted file mode 100644 index c9812bb16dc..00000000000 --- a/docker/blocks/graphite1/files/supervisord.conf +++ /dev/null @@ -1,26 +0,0 @@ -[supervisord] -nodaemon = true -environment = GRAPHITE_STORAGE_DIR='/opt/graphite/storage',GRAPHITE_CONF_DIR='/opt/graphite/conf' - -[program:nginx] -command = /usr/sbin/nginx -stdout_logfile = /var/log/supervisor/%(program_name)s.log -stderr_logfile = /var/log/supervisor/%(program_name)s.log -autorestart = true - -[program:carbon-cache] -;user = www-data -command = /opt/graphite/bin/carbon-cache.py --debug start -stdout_logfile = /var/log/supervisor/%(program_name)s.log -stderr_logfile = /var/log/supervisor/%(program_name)s.log -autorestart = true - -[program:graphite-webapp] -;user = www-data -directory = /opt/graphite/webapp -environment = PYTHONPATH='/opt/graphite/webapp' -command = /usr/bin/gunicorn_django -b127.0.0.1:8000 -w2 graphite/settings.py -stdout_logfile = /var/log/supervisor/%(program_name)s.log -stderr_logfile = /var/log/supervisor/%(program_name)s.log -autorestart = true - diff --git a/docker/blocks/influxdb/docker-compose.yaml b/docker/blocks/influxdb/docker-compose.yaml new file mode 100644 index 00000000000..3434f5d09b9 --- /dev/null +++ b/docker/blocks/influxdb/docker-compose.yaml @@ -0,0 +1,17 @@ + influxdb: + image: influxdb:latest + container_name: influxdb + ports: + - "2004:2004" + - "8083:8083" + - "8086:8086" + volumes: + - ./blocks/influxdb/influxdb.conf:/etc/influxdb/influxdb.conf + + fake-influxdb-data: + image: grafana/fake-data-gen + network_mode: bridge + environment: + FD_DATASOURCE: influxdb + FD_PORT: 8086 + diff --git a/docker/blocks/influxdb/fig b/docker/blocks/influxdb/fig deleted file mode 100644 index 8821c010a98..00000000000 --- a/docker/blocks/influxdb/fig +++ /dev/null @@ -1,17 +0,0 @@ -influxdb: - image: influxdb:latest - container_name: influxdb - ports: - - "2004:2004" - - "8083:8083" - - "8086:8086" - volumes: - - ./blocks/influxdb/influxdb.conf:/etc/influxdb/influxdb.conf - -fake-influxdb-data: - image: grafana/fake-data-gen - net: bridge - environment: - FD_DATASOURCE: influxdb - FD_PORT: 8086 - diff --git a/docker/blocks/jaeger/docker-compose.yaml b/docker/blocks/jaeger/docker-compose.yaml new file mode 100644 index 00000000000..2b57c863425 --- /dev/null +++ b/docker/blocks/jaeger/docker-compose.yaml @@ -0,0 +1,6 @@ + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - "127.0.0.1:6831:6831/udp" + - "16686:16686" + diff --git a/docker/blocks/jaeger/fig b/docker/blocks/jaeger/fig deleted file mode 100644 index ab9e2ec599b..00000000000 --- a/docker/blocks/jaeger/fig +++ /dev/null @@ -1,6 +0,0 @@ -jaeger: - image: jaegertracing/all-in-one:latest - ports: - - "localhost:6831:6831/udp" - - "16686:16686" - diff --git a/docker/blocks/memcached/docker-compose.yaml b/docker/blocks/memcached/docker-compose.yaml new file mode 100644 index 00000000000..b3201da0f95 --- /dev/null +++ b/docker/blocks/memcached/docker-compose.yaml @@ -0,0 +1,5 @@ + memcached: + image: memcached:latest + ports: + - "11211:11211" + diff --git a/docker/blocks/memcached/fig b/docker/blocks/memcached/fig deleted file mode 100644 index a0da9df2bc2..00000000000 --- a/docker/blocks/memcached/fig +++ /dev/null @@ -1,5 +0,0 @@ -memcached: - image: memcached:latest - ports: - - "11211:11211" - diff --git a/docker/blocks/mysql/docker-compose.yaml b/docker/blocks/mysql/docker-compose.yaml new file mode 100644 index 00000000000..6eee158ac43 --- /dev/null +++ b/docker/blocks/mysql/docker-compose.yaml @@ -0,0 +1,14 @@ + mysql: + image: mysql:latest + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: grafana + MYSQL_USER: grafana + MYSQL_PASSWORD: password + ports: + - "3306:3306" + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --innodb_monitor_enable=all] + diff --git a/docker/blocks/mysql/fig b/docker/blocks/mysql/fig deleted file mode 100644 index 24cb47b61a7..00000000000 --- a/docker/blocks/mysql/fig +++ /dev/null @@ -1,14 +0,0 @@ -mysql: - image: mysql:latest - environment: - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_DATABASE: grafana - MYSQL_USER: grafana - MYSQL_PASSWORD: password - ports: - - "3306:3306" - volumes: - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro - command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --innodb_monitor_enable=all] - diff --git a/docker/blocks/mysql_opendata/docker-compose.yaml b/docker/blocks/mysql_opendata/docker-compose.yaml new file mode 100644 index 00000000000..594eeed284a --- /dev/null +++ b/docker/blocks/mysql_opendata/docker-compose.yaml @@ -0,0 +1,9 @@ + mysql_opendata: + build: blocks/mysql_opendata + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: testdata + MYSQL_USER: grafana + MYSQL_PASSWORD: password + ports: + - "3307:3306" diff --git a/docker/blocks/mysql_opendata/fig b/docker/blocks/mysql_opendata/fig deleted file mode 100644 index a374fbd0931..00000000000 --- a/docker/blocks/mysql_opendata/fig +++ /dev/null @@ -1,9 +0,0 @@ -mysql_opendata: - build: blocks/mysql_opendata - environment: - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_DATABASE: testdata - MYSQL_USER: grafana - MYSQL_PASSWORD: password - ports: - - "3307:3306" diff --git a/docker/blocks/mysql_tests/docker-compose.yaml b/docker/blocks/mysql_tests/docker-compose.yaml new file mode 100644 index 00000000000..c6c3097d463 --- /dev/null +++ b/docker/blocks/mysql_tests/docker-compose.yaml @@ -0,0 +1,10 @@ + mysqltests: + image: mysql:latest + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: grafana_tests + MYSQL_USER: grafana + MYSQL_PASSWORD: password + ports: + - "3306:3306" + tmpfs: /var/lib/mysql:rw diff --git a/docker/blocks/mysql_tests/fig b/docker/blocks/mysql_tests/fig deleted file mode 100644 index 880c955d218..00000000000 --- a/docker/blocks/mysql_tests/fig +++ /dev/null @@ -1,9 +0,0 @@ -mysqltests: - image: mysql:latest - environment: - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_DATABASE: grafana_tests - MYSQL_USER: grafana - MYSQL_PASSWORD: password - ports: - - "3306:3306" diff --git a/docker/blocks/openldap/Dockerfile b/docker/blocks/openldap/Dockerfile index d16987cb3ab..d073e274356 100644 --- a/docker/blocks/openldap/Dockerfile +++ b/docker/blocks/openldap/Dockerfile @@ -1,6 +1,6 @@ FROM debian:jessie -MAINTAINER Christian Luginbühl +LABEL maintainer="Christian Luginbühl " ENV OPENLDAP_VERSION 2.4.40 diff --git a/docker/blocks/openldap/docker-compose.yaml b/docker/blocks/openldap/docker-compose.yaml new file mode 100644 index 00000000000..be06524a57d --- /dev/null +++ b/docker/blocks/openldap/docker-compose.yaml @@ -0,0 +1,10 @@ + openldap: + build: blocks/openldap + environment: + SLAPD_PASSWORD: grafana + SLAPD_DOMAIN: grafana.org + SLAPD_ADDITIONAL_MODULES: memberof + ports: + - "389:389" + + diff --git a/docker/blocks/openldap/fig b/docker/blocks/openldap/fig deleted file mode 100644 index b9528f2d4d7..00000000000 --- a/docker/blocks/openldap/fig +++ /dev/null @@ -1,10 +0,0 @@ -openldap: - build: blocks/openldap - environment: - SLAPD_PASSWORD: grafana - SLAPD_DOMAIN: grafana.org - SLAPD_ADDITIONAL_MODULES: memberof - ports: - - "389:389" - - diff --git a/docker/blocks/opentsdb/docker-compose.yaml b/docker/blocks/opentsdb/docker-compose.yaml new file mode 100644 index 00000000000..ee064bb107d --- /dev/null +++ b/docker/blocks/opentsdb/docker-compose.yaml @@ -0,0 +1,11 @@ + opentsdb: + image: opower/opentsdb:latest + ports: + - "4242:4242" + + fake-opentsdb-data: + image: grafana/fake-data-gen + network_mode: bridge + environment: + FD_DATASOURCE: opentsdb + diff --git a/docker/blocks/opentsdb/fig b/docker/blocks/opentsdb/fig deleted file mode 100644 index c346475e9a3..00000000000 --- a/docker/blocks/opentsdb/fig +++ /dev/null @@ -1,11 +0,0 @@ -opentsdb: - image: opower/opentsdb:latest - ports: - - "4242:4242" - -fake-opentsdb-data: - image: grafana/fake-data-gen - net: bridge - environment: - FD_DATASOURCE: opentsdb - diff --git a/docker/blocks/postgres/docker-compose.yaml b/docker/blocks/postgres/docker-compose.yaml new file mode 100644 index 00000000000..eced00aafeb --- /dev/null +++ b/docker/blocks/postgres/docker-compose.yaml @@ -0,0 +1,9 @@ + postgrestest: + image: postgres:latest + environment: + POSTGRES_USER: grafana + POSTGRES_PASSWORD: password + POSTGRES_DATABASE: grafana + ports: + - "5432:5432" + command: postgres -c log_connections=on -c logging_collector=on -c log_destination=stderr -c log_directory=/var/log/postgresql diff --git a/docker/blocks/postgres/fig b/docker/blocks/postgres/fig deleted file mode 100644 index 9f39a0ffb1d..00000000000 --- a/docker/blocks/postgres/fig +++ /dev/null @@ -1,9 +0,0 @@ -postgrestest: - image: postgres:9.4.14 - environment: - POSTGRES_USER: grafana - POSTGRES_PASSWORD: password - POSTGRES_DATABASE: grafana - ports: - - "5432:5432" - command: postgres -c log_connections=on -c logging_collector=on -c log_destination=stderr -c log_directory=/var/log/postgresql diff --git a/docker/blocks/postgres_tests/docker-compose.yaml b/docker/blocks/postgres_tests/docker-compose.yaml new file mode 100644 index 00000000000..44b66e8e558 --- /dev/null +++ b/docker/blocks/postgres_tests/docker-compose.yaml @@ -0,0 +1,8 @@ + postgrestest: + image: postgres:latest + environment: + POSTGRES_USER: grafanatest + POSTGRES_PASSWORD: grafanatest + ports: + - "5432:5432" + tmpfs: /var/lib/postgresql/data:rw \ No newline at end of file diff --git a/docker/blocks/postgres_tests/fig b/docker/blocks/postgres_tests/fig deleted file mode 100644 index 049afe185c8..00000000000 --- a/docker/blocks/postgres_tests/fig +++ /dev/null @@ -1,7 +0,0 @@ -postgrestest: - image: postgres:latest - environment: - POSTGRES_USER: grafanatest - POSTGRES_PASSWORD: grafanatest - ports: - - "5432:5432" diff --git a/docker/blocks/prometheus/docker-compose.yaml b/docker/blocks/prometheus/docker-compose.yaml new file mode 100644 index 00000000000..ccb1238a179 --- /dev/null +++ b/docker/blocks/prometheus/docker-compose.yaml @@ -0,0 +1,25 @@ + prometheus: + build: blocks/prometheus + network_mode: host + ports: + - "9090:9090" + + node_exporter: + image: prom/node-exporter + network_mode: host + ports: + - "9100:9100" + + fake-prometheus-data: + image: grafana/fake-data-gen + network_mode: host + ports: + - "9091:9091" + environment: + FD_DATASOURCE: prom + + alertmanager: + image: quay.io/prometheus/alertmanager + network_mode: host + ports: + - "9093:9093" diff --git a/docker/blocks/prometheus/fig b/docker/blocks/prometheus/fig deleted file mode 100644 index 7d9bea68046..00000000000 --- a/docker/blocks/prometheus/fig +++ /dev/null @@ -1,25 +0,0 @@ -prometheus: - build: blocks/prometheus - net: host - ports: - - "9090:9090" - -node_exporter: - image: prom/node-exporter - net: host - ports: - - "9100:9100" - -fake-prometheus-data: - image: grafana/fake-data-gen - net: host - ports: - - "9091:9091" - environment: - FD_DATASOURCE: prom - -alertmanager: - image: quay.io/prometheus/alertmanager - net: host - ports: - - "9093:9093" diff --git a/docker/blocks/prometheus2/Dockerfile b/docker/blocks/prometheus2/Dockerfile new file mode 100644 index 00000000000..d4a9eb2d75d --- /dev/null +++ b/docker/blocks/prometheus2/Dockerfile @@ -0,0 +1,3 @@ +FROM prom/prometheus:v2.0.0 +ADD prometheus.yml /etc/prometheus/ +ADD alert.rules /etc/prometheus/ diff --git a/docker/blocks/prometheus2/alert.rules b/docker/blocks/prometheus2/alert.rules new file mode 100644 index 00000000000..563d1e89994 --- /dev/null +++ b/docker/blocks/prometheus2/alert.rules @@ -0,0 +1,10 @@ +# Alert Rules + +ALERT AppCrash + IF process_open_fds > 0 + FOR 15s + LABELS { severity="critical" } + ANNOTATIONS { + summary = "Number of open fds > 0", + description = "Just testing" + } diff --git a/docker/blocks/prometheus2/prometheus.yml b/docker/blocks/prometheus2/prometheus.yml new file mode 100644 index 00000000000..83dda78bb3c --- /dev/null +++ b/docker/blocks/prometheus2/prometheus.yml @@ -0,0 +1,35 @@ +# my global config +global: + scrape_interval: 10s # By default, scrape targets every 15 seconds. + evaluation_interval: 10s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + +# Load and evaluate rules in this file every 'evaluation_interval' seconds. +#rule_files: +# - "alert.rules" +# - "first.rules" +# - "second.rules" + +# alerting: +# alertmanagers: +# - scheme: http +# static_configs: +# - targets: +# - "127.0.0.1:9093" + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'node_exporter' + static_configs: + - targets: ['127.0.0.1:9100'] + + - job_name: 'fake-data-gen' + static_configs: + - targets: ['127.0.0.1:9091'] + + - job_name: 'grafana' + static_configs: + - targets: ['127.0.0.1:3000'] diff --git a/docker/blocks/smtp/Dockerfile b/docker/blocks/smtp/Dockerfile index c1a3adba7c8..9326e077ed9 100644 --- a/docker/blocks/smtp/Dockerfile +++ b/docker/blocks/smtp/Dockerfile @@ -1,5 +1,5 @@ FROM centos:centos7 -MAINTAINER Przemyslaw Ozgo +LABEL maintainer="Przemyslaw Ozgo " RUN \ yum update -y && \ diff --git a/docker/blocks/smtp/docker-compose.yaml b/docker/blocks/smtp/docker-compose.yaml new file mode 100644 index 00000000000..85d598b6167 --- /dev/null +++ b/docker/blocks/smtp/docker-compose.yaml @@ -0,0 +1,4 @@ + snmpd: + image: namshi/smtp + ports: + - "25:25" diff --git a/docker/blocks/smtp/fig b/docker/blocks/smtp/fig deleted file mode 100644 index 3aa25e01311..00000000000 --- a/docker/blocks/smtp/fig +++ /dev/null @@ -1,4 +0,0 @@ -snmpd: - image: namshi/smtp - ports: - - "25:25" diff --git a/docker/compose_header.yml b/docker/compose_header.yml new file mode 100644 index 00000000000..e7bf4f38b02 --- /dev/null +++ b/docker/compose_header.yml @@ -0,0 +1,2 @@ +version: "2" +services: diff --git a/docker/create_docker_compose.sh b/docker/create_docker_compose.sh index 8588c1c474a..9d28ede8e7e 100755 --- a/docker/create_docker_compose.sh +++ b/docker/create_docker_compose.sh @@ -7,8 +7,9 @@ template_dir=templates grafana_config_file=conf.tmp grafana_config=config -fig_file=docker-compose.yml -fig_config=fig +compose_header_file=compose_header.yml +fig_file=docker-compose.yaml +fig_config=docker-compose.yaml if [ "$#" == 0 ]; then blocks=`ls $blocks_dir` @@ -23,13 +24,16 @@ if [ "$#" == 0 ]; then exit 0 fi -for file in $gogs_config_file $fig_file; do +for file in $grafana_config_file $fig_file; do if [ -e $file ]; then echo "Deleting $file" rm $file fi done +echo "Adding Compose header to $fig_file" +cat $compose_header_file >> $fig_file + for dir in $@; do current_dir=$blocks_dir/$dir if [ ! -d "$current_dir" ]; then @@ -45,7 +49,7 @@ for dir in $@; do if [ -e $current_dir/$fig_config ]; then echo "Adding $current_dir/$fig_config to $fig_file" - cat $current_dir/fig >> $fig_file + cat $current_dir/$fig_config >> $fig_file echo "" >> $fig_file fi done diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md new file mode 100644 index 00000000000..0745794496f --- /dev/null +++ b/docs/sources/administration/provisioning.md @@ -0,0 +1,167 @@ ++++ +title = "Provisioning" +description = "" +keywords = ["grafana", "provisioning"] +type = "docs" +[menu.docs] +parent = "admin" +weight = 8 ++++ + +# Provisioning Grafana + +## Config file + +Checkout the [configuration](/installation/configuration) page for more information about what you can configure in `grafana.ini` + +### Config file locations + +- Default configuration from `$WORKING_DIR/conf/defaults.ini` +- Custom configuration from `$WORKING_DIR/conf/custom.ini` +- The custom configuration file path can be overridden using the `--config` parameter + +> **Note.** If you have installed Grafana using the `deb` or `rpm` +> packages, then your configuration file is located at +> `/etc/grafana/grafana.ini`. This path is specified in the Grafana +> init.d script using `--config` file parameter. + +### Using environment variables + +All options in the configuration file (listed below) can be overridden +using environment variables using the syntax: + +```bash +GF__ +``` + +Where the section name is the text within the brackets. Everything +should be upper case, `.` should be replaced by `_`. For example, given these configuration settings: + +```bash +# default section +instance_name = ${HOSTNAME} + +[security] +admin_user = admin + +[auth.google] +client_secret = 0ldS3cretKey +``` + +Then you can override them using: + +```bash +export GF_DEFAULT_INSTANCE_NAME=my-instance +export GF_SECURITY_ADMIN_USER=true +export GF_AUTH_GOOGLE_CLIENT_SECRET=newS3cretKey +``` + +
+ +## Configuration management tools + +Currently we do not provide any scripts/manifests for configuring Grafana. Rather then spending time learning and creating scripts/manifests for each tool, we think our time is better spent making Grafana easier to provision. Therefor, we heavily relay on the expertise of he community. + +Tool | Project +-----|------------ +Puppet | [https://forge.puppet.com/puppet/grafana](https://forge.puppet.com/puppet/grafana) +Ansible | [https://github.com/cloudalchemy/ansible-grafana](https://github.com/cloudalchemy/ansible-grafana) +Ansible | [https://github.com/picotrading/ansible-grafana](https://github.com/picotrading/ansible-grafana) +Chef | [https://github.com/JonathanTron/chef-grafana](https://github.com/JonathanTron/chef-grafana) +Saltstack | [https://github.com/salt-formulas/salt-formula-grafana](https://github.com/salt-formulas/salt-formula-grafana) + +## Datasources + +> This feature is available from v5.0 + +It's possible to manage datasources in Grafana by adding one or more yaml config files in the [`conf/datasources`](/installation/configuration/#datasources) directory. Each config file can contain a list of `datasources` that will be added or updated during start up. If the datasource already exists, Grafana will update it to match the configuration file. The config file can also contain a list of datasources that should be deleted. That list is called `delete_datasources`. Grafana will delete datasources listed in `delete_datasources` before inserting/updating those in the `datasource` list. + +### Running multiple grafana instances. +If you are running multiple instances of Grafana you might run into problems if they have different versions of the datasource.yaml configuration file. The best way to solve this problem is to add a version number to each datasource in the configuration and increase it when you update the config. Grafana will only update datasources with the same or lower version number than specified in the config. That way old configs cannot overwrite newer configs if they restart at the same time. + +### Example datasource config file +```yaml +# list of datasources that should be deleted from the database +delete_datasources: + - name: Graphite + org_id: 1 + +# list of datasources to insert/update depending +# whats available in the datbase +datasources: + # name of the datasource. Required +- name: Graphite + # datasource type. Required + type: graphite + # access mode. direct or proxy. Required + access: proxy + # org id. will default to org_id 1 if not specified + org_id: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basic_auth: + # basic auth username + basic_auth_user: + # basic auth password + basic_auth_password: + # enable/disable with credentials headers + with_credentials: + # mark as default datasource. Max one per org + is_default: + # fields that will be converted to json and stored in json_data + json_data: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secure_json_data: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +#### Json data + +Since all datasources dont have the same configuration settings we only have the most common ones as fields. The rest should be stored as a json blob in the `json_data` field. Here are the most common settings that the core datasources use. + +| Name | Type | Datasource |Description | +| ----| ---- | ---- | --- | +| tlsAuth | boolean | *All* | Enable TLS authentication using client cert configured in secure json data | +| tlsAuthWithCACert | boolean | *All* | Enable TLS authtication using CA cert | +| graphiteVersion | string | Graphite | Graphite version | +| timeInterval | string | Elastic, Influxdb & Prometheus | Lowest interval/step value that should be used for this data source | +| esVersion | string | Elastic | Elasticsearch version | +| timeField | string | Elastic | Which field that should be used as timestamp | +| interval | string | Elastic | Index date time format | +| authType | string | Cloudwatch | Auth provider. keys/credentials/arn | +| assumeRoleArn | string | Cloudwatch | ARN of Assume Role | +| defaultRegion | string | Cloudwatch | AWS region | +| customMetricsNamespaces | string | Cloudwatch | Namespaces of Custom Metrics | +| tsdbVersion | string | OpenTsdb | Version | +| tsdbResolution | string | OpenTsdb | Resolution | +| sslmode | string | Postgre | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | + + +#### Secure Json data + +{"authType":"keys","defaultRegion":"us-west-2","timeField":"@timestamp"} + +Secure json data is a map of settings that will be encrypted with [secret key](/installation/configuration/#secret-key) from the grafana config. The purpose of this is only to hide content from the users of the application. This should be used for storing TLS Cert and password that Grafana will append to request on the server side. All these settings are optional. + +| Name | Type | Datasource | Description | +| ----| ---- | ---- | --- | +| tlsCACert | string | *All* |CA cert for out going requests | +| tlsClientCert | string | *All* |TLS Client cert for outgoing requests | +| tlsClientKey | string | *All* |TLS Client key for outgoing requests | +| password | string | Postgre | password | +| user | string | Postgre | user | diff --git a/docs/sources/features/datasources/mysql.md b/docs/sources/features/datasources/mysql.md index 69c6f667062..d9f048e0371 100644 --- a/docs/sources/features/datasources/mysql.md +++ b/docs/sources/features/datasources/mysql.md @@ -127,6 +127,12 @@ A query can returns multiple columns and Grafana will automatically create a lis SELECT my_host.hostname, my_other_host.hostname2 FROM my_host JOIN my_other_host ON my_host.city = my_other_host.city ``` +To use time range dependent macros like `$__timeFilter(column)` in your query the refresh mode of the template variable needs to be set to *On Time Range Change*. + +```sql +SELECT event_name FROM event_log WHERE $__timeFilter(time_column) +``` + Another option is a query that can create a key/value variable. The query should return two columns that are named `__text` and `__value`. The `__text` column value should be unique (if it is not unique then the first value is used). The options in the dropdown will have a text and value that allows you to have a friendly name as text and an id as the value. An example query with `hostname` as the text and `id` as the value: ```sql diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index 154f822d9d0..e9d65b8f327 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -45,10 +45,10 @@ Macro example | Description ------------ | ------------- *$__time(dateColumn)* | Will be replaced by an expression to rename the column to `time`. For example, *dateColumn as time* *$__timeSec(dateColumn)* | Will be replaced by an expression to rename the column to `time` and converting the value to unix timestamp. For example, *extract(epoch from dateColumn) as time* -*$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *dateColumn > to_timestamp(1494410783) AND dateColumn < to_timestamp(1494497183)* +*$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *extract(epoch from dateColumn) BETWEEN 1494410783 AND 1494497183* *$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *to_timestamp(1494410783)* *$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *to_timestamp(1494497183)* -*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *(extract(epoch from "dateColumn")/extract(epoch from '5m'::interval))::int*extract(epoch from '5m'::interval)* +*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *(extract(epoch from "dateColumn")/300)::bigint*300* *$__unixEpochFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name with times represented as unix timestamp. For example, *dateColumn > 1494410783 AND dateColumn < 1494497183* *$__unixEpochFrom()* | Will be replaced by the start of the currently active time selection as unix timestamp. For example, *1494410783* *$__unixEpochTo()* | Will be replaced by the end of the currently active time selection as unix timestamp. For example, *1494497183* @@ -139,6 +139,12 @@ A query can return multiple columns and Grafana will automatically create a list SELECT host.hostname, other_host.hostname2 FROM host JOIN other_host ON host.city = other_host.city ``` +To use time range dependent macros like `$__timeFilter(column)` in your query the refresh mode of the template variable needs to be set to *On Time Range Change*. + +```sql +SELECT event_name FROM event_log WHERE $__timeFilter(time_column) +``` + Another option is a query that can create a key/value variable. The query should return two columns that are named `__text` and `__value`. The `__text` column value should be unique (if it is not unique then the first value is used). The options in the dropdown will have a text and value that allows you to have a friendly name as text and an id as the value. An example query with `hostname` as the text and `id` as the value: ```sql @@ -186,7 +192,7 @@ ORDER BY atimestamp ASC ## Annotations -[Annotations]({{< relref "reference/annotations.md" >}}) allows you to overlay rich event information on top of graphs. You add annotation queries via the Dashboard menu / Annotations view. +[Annotations]({{< relref "reference/annotations.md" >}}) allow you to overlay rich event information on top of graphs. You add annotation queries via the Dashboard menu / Annotations view. An example query: diff --git a/docs/sources/features/datasources/prometheus.md b/docs/sources/features/datasources/prometheus.md index dceb2254e41..15247ba5ebd 100644 --- a/docs/sources/features/datasources/prometheus.md +++ b/docs/sources/features/datasources/prometheus.md @@ -34,6 +34,7 @@ Name | Description *Basic Auth* | Enable basic authentication to the Prometheus data source. *User* | Name of your Prometheus user *Password* | Database user's password +*Scrape interval* | This will be used as a lower limit for the Prometheus step query parameter. Default value is 15s. ## Query editor @@ -95,3 +96,7 @@ Prometheus supports two ways to query annotations. - A Prometheus query for pending and firing alerts (for details see [Inspecting alerts during runtime](https://prometheus.io/docs/alerting/rules/#inspecting-alerts-during-runtime)) The step option is useful to limit the number of events returned from your query. + +## Getting Grafana metrics into Prometheus + +Since 4.6.0 Grafana exposes metrics for Prometheus on the `/metrics` endpoint. We also bundle a dashboard within Grafana so you can get started viewing your metrics faster. You can import the bundled dashboard by going to the data source edit page and click the dashboard tab. There you can find a dashboard for Grafana and one for Prometheus. Import and start viewing all the metrics! diff --git a/docs/sources/features/datasources/testdata.md b/docs/sources/features/datasources/testdata.md index 491e5b60fbb..d99f9cabe08 100644 --- a/docs/sources/features/datasources/testdata.md +++ b/docs/sources/features/datasources/testdata.md @@ -17,7 +17,7 @@ This make is much easier to verify functionally since the data can be shared ver ## Enable -`Grafana TestData` is not enabled by default. To enable it you have to go to `/plugins/testdata/edit` and click the enable button to enable. +`Grafana TestData` is not enabled by default. To enable it, first navigate to the Plugins section, found in your Grafana main menu. Click the Apps tabs in the Plugins section and select the Grafana TestData App. (Or navigate to http://your_grafana_instance/plugins/testdata/edit to go directly there). Finally click the enable button to enable. ## Create mock data. diff --git a/docs/sources/http_api/alerting.md b/docs/sources/http_api/alerting.md index c5172c64203..e66218bb066 100644 --- a/docs/sources/http_api/alerting.md +++ b/docs/sources/http_api/alerting.md @@ -133,6 +133,37 @@ Content-Type: application/json } ``` +## Pause all alerts + +`POST /api/admin/pause-all-alerts` + +```http +POST /api/admin/pause-all-alerts HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + +{ + "paused": true +} +``` + +JSON Body Schema: + +- **paused** – Can be `true` or `false`. True to pause an alert. False to unpause an alert. + +**Example Response**: + +```http +HTTP/1.1 200 +Content-Type: application/json +{ + "state": "Paused", + "message": "alert paused", + "alertsAffected": 1 +} +``` + ## Get alert notifications `GET /api/alert-notifications` diff --git a/docs/sources/http_api/annotations.md b/docs/sources/http_api/annotations.md index 7aab127cb0c..19c2a5c386c 100644 --- a/docs/sources/http_api/annotations.md +++ b/docs/sources/http_api/annotations.md @@ -89,7 +89,7 @@ Content-Type: application/json ## Create Annotation -Creates an annotation in the Grafana database. The `dashboardId` and `panelId` fields are optional. If they are not specified then a global annotation is created and can be queried in any dashboard that adds the Grafana annotations data source. +Creates an annotation in the Grafana database. The `dashboardId` and `panelId` fields are optional. If they are not specified then a global annotation is created and can be queried in any dashboard that adds the Grafana annotations data source. When creating a region annotation the response will include both `id` and `endId`, if not only `id`. `POST /api/annotations` @@ -117,7 +117,11 @@ Content-Type: application/json HTTP/1.1 200 Content-Type: application/json -{"message":"Annotation added"} +{ + "message":"Annotation added", + "id": 1, + "endId": 2 +} ``` ## Create Annotation in Graphite format @@ -148,7 +152,10 @@ Content-Type: application/json HTTP/1.1 200 Content-Type: application/json -{"message":"Graphite annotation added"} +{ + "message":"Graphite annotation added", + "id": 1 +} ``` ## Update Annotation diff --git a/docs/sources/http_api/dashboard.md b/docs/sources/http_api/dashboard.md index 300e5613db4..0538754bd96 100644 --- a/docs/sources/http_api/dashboard.md +++ b/docs/sources/http_api/dashboard.md @@ -258,7 +258,7 @@ Query parameters: **Example Request**: ```http -GET /api/search?query=MyDashboard&starred=true&tag=prod HTTP/1.1 +GET /api/search?query=Production%20Overview&starred=true&tag=prod HTTP/1.1 Accept: application/json Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk @@ -276,8 +276,8 @@ Content-Type: application/json "title":"Production Overview", "uri":"db/production-overview", "type":"dash-db", - "tags":[], - "isStarred":false + "tags":[prod], + "isStarred":true } ] -``` \ No newline at end of file +``` diff --git a/docs/sources/http_api/snapshot.md b/docs/sources/http_api/snapshot.md index 5cecdb85fc3..dce3b0a9160 100644 --- a/docs/sources/http_api/snapshot.md +++ b/docs/sources/http_api/snapshot.md @@ -140,9 +140,9 @@ Content-Type: application/json } ``` -## Delete Snapshot by Id +## Delete Snapshot by deleteKey -`GET /api/snapshots-delete/:key` +`GET /api/snapshots-delete/:deleteKey` **Example Request**: diff --git a/docs/sources/index.md b/docs/sources/index.md index 9226c842abc..7a431e29692 100644 --- a/docs/sources/index.md +++ b/docs/sources/index.md @@ -46,8 +46,8 @@ those options. - [Graphite]({{< relref "features/datasources/graphite.md" >}}) - [Elasticsearch]({{< relref "features/datasources/elasticsearch.md" >}}) - [InfluxDB]({{< relref "features/datasources/influxdb.md" >}}) -- [Prometheus]({{< relref "features/datasources/influxdb.md" >}}) -- [OpenTSDB]({{< relref "features/datasources/prometheus.md" >}}) +- [Prometheus]({{< relref "features/datasources/prometheus.md" >}}) +- [OpenTSDB]({{< relref "features/datasources/opentsdb.md" >}}) - [MySQL]({{< relref "features/datasources/mysql.md" >}}) - [Postgres]({{< relref "features/datasources/postgres.md" >}}) - [Cloudwatch]({{< relref "features/datasources/cloudwatch.md" >}}) diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 627a76a963e..d454455a26b 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -87,6 +87,17 @@ command line in the init.d script or the systemd service file. It can be overridden in the configuration file or in the default environment variable file. +### plugins + +Directory where grafana will automatically scan and look for plugins + +### datasources + +> This feature is available in 5.0+ + +Config files containing datasources that will be configured at startup. +You can read more about the config files at the [provisioning page](/administration/provisioning/#datasources). + ## [server] ### http_addr @@ -224,6 +235,9 @@ The maximum number of connections in the idle connection pool. ### max_open_conn The maximum number of open connections to the database. +### log_queries +Set to `true` to log the sql calls and execution times. +
## [security] @@ -551,7 +565,7 @@ session provider you have configured. - **file:** session file path, e.g. `data/sessions` - **mysql:** go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` -- **postgres:** ex: user=a password=b host=localhost port=5432 dbname=c sslmode=require +- **postgres:** ex: user=a password=b host=localhost port=5432 dbname=c sslmode=verify-full - **memcache:** ex: 127.0.0.1:11211 - **redis:** ex: `addr=127.0.0.1:6379,pool_size=100,prefix=grafana` @@ -580,7 +594,7 @@ CREATE TABLE session ( ); ``` -Postgres valid `sslmode` are `disable`, `require` (default), `verify-ca`, and `verify-full`. +Postgres valid `sslmode` are `disable`, `require`, `verify-ca`, and `verify-full` (default). ### cookie_name @@ -613,6 +627,12 @@ Analytics ID here. By default this feature is disabled.
+## [dashboards] + +### versions_to_keep (introduced in v5.0) + +Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1. + ## [dashboards.json] If you have a system that automatically builds dashboards as json files you can enable this feature to have the @@ -673,7 +693,7 @@ Ex `filters = sqlstore:debug` ## [metrics] ### enabled -Enable metrics reporting. defaults true. Available via HTTP API `/api/metrics`. +Enable metrics reporting. defaults true. Available via HTTP API `/metrics`. ### interval_seconds @@ -761,6 +781,9 @@ Service Account should have "Storage Object Writer" role. ### bucket name Bucket Name on Google Cloud Storage. +### path +Optional extra path inside bucket + ## [alerting] ### enabled diff --git a/docs/sources/installation/debian.md b/docs/sources/installation/debian.md index 7ecb6d14b0c..d832ea7a8ed 100644 --- a/docs/sources/installation/debian.md +++ b/docs/sources/installation/debian.md @@ -15,7 +15,7 @@ weight = 1 Description | Download ------------ | ------------- -Stable for Debian-based Linux | [grafana_4.6.0_amd64.deb](https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_4.6.0_amd64.deb) +Stable for Debian-based Linux | [grafana_4.6.2_amd64.deb](https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_4.6.2_amd64.deb) @@ -26,9 +26,9 @@ installation. ```bash -wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_4.6.0_amd64.deb +wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_4.6.2_amd64.deb sudo apt-get install -y adduser libfontconfig -sudo dpkg -i grafana_4.6.0_amd64.deb +sudo dpkg -i grafana_4.6.2_amd64.deb ``` @@ -27,7 +27,7 @@ installation. You can install Grafana using Yum directly. ```bash -$ sudo yum install https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.6.0-1.x86_64.rpm +$ sudo yum install https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.6.2-1.x86_64.rpm ``` Or install manually using `rpm`. @@ -35,15 +35,15 @@ Or install manually using `rpm`. #### On CentOS / Fedora / Redhat: ```bash -$ wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.6.0-1.x86_64.rpm +$ wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.6.2-1.x86_64.rpm $ sudo yum install initscripts fontconfig -$ sudo rpm -Uvh grafana-4.6.0-1.x86_64.rpm +$ sudo rpm -Uvh grafana-4.6.2-1.x86_64.rpm ``` #### On OpenSuse: ```bash -$ sudo rpm -i --nodeps grafana-4.6.0-1.x86_64.rpm +$ sudo rpm -i --nodeps grafana-4.6.2-1.x86_64.rpm ``` ## Install via YUM Repository diff --git a/docs/sources/installation/windows.md b/docs/sources/installation/windows.md index 181d26d694c..9cfd689fb43 100644 --- a/docs/sources/installation/windows.md +++ b/docs/sources/installation/windows.md @@ -13,7 +13,7 @@ weight = 3 Description | Download ------------ | ------------- -Latest stable package for Windows | [grafana.4.6.0.windows-x64.zip](https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.6.0.windows-x64.zip) +Latest stable package for Windows | [grafana.4.6.2.windows-x64.zip](https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.6.2.windows-x64.zip) Read [Upgrading Grafana]({{< relref "installation/upgrading.md" >}}) for tips and guidance on updating an existing installation. diff --git a/docs/sources/project/building_from_source.md b/docs/sources/project/building_from_source.md index e4ccedb7299..dba04164d00 100644 --- a/docs/sources/project/building_from_source.md +++ b/docs/sources/project/building_from_source.md @@ -13,9 +13,10 @@ dev environment. Grafana ships with its own required backend server; also comple ## Dependencies -- [Go 1.9.1](https://golang.org/dl/) -- [NodeJS LTS](https://nodejs.org/download/) +- [Go 1.9.2](https://golang.org/dl/) - [Git](https://git-scm.com/downloads) +- [NodeJS LTS](https://nodejs.org/download/) +- node-gyp is the Node.js native addon build tool and it requires extra dependencies: python 2.7, make and GCC. These are already installed for most Linux distros and MacOS. See the Building On Windows section or the [node-gyp installation instructions](https://github.com/nodejs/node-gyp#installation) for more details. ## Get Code Create a directory for the project and set your path accordingly (or use the [default Go workspace directory](https://golang.org/doc/code.html#GOPATH)). Then download and install Grafana into your $GOPATH directory: @@ -40,8 +41,8 @@ go run build.go build # (or 'go build ./pkg/cmd/grafana-server') ``` #### Building on Windows -The Grafana backend includes Sqlite3 which requires GCC to compile. So in order to compile Grafana on windows you need -to install GCC. We recommend [TDM-GCC](http://tdm-gcc.tdragon.net/download). + +The Grafana backend includes Sqlite3 which requires GCC to compile. So in order to compile Grafana on windows you need to install GCC. We recommend [TDM-GCC](http://tdm-gcc.tdragon.net/download). [node-gyp](https://github.com/nodejs/node-gyp#installation) is the Node.js native addon build tool and it requires extra dependencies to be installed on Windows. In a command prompt which is run as administrator, run: diff --git a/docs/sources/tutorials/authproxy.md b/docs/sources/tutorials/authproxy.md index d4d2b9926fc..8003be20644 100644 --- a/docs/sources/tutorials/authproxy.md +++ b/docs/sources/tutorials/authproxy.md @@ -25,12 +25,16 @@ enabled = true header_name = X-WEBAUTH-USER header_property = username auto_sign_up = true +ldap_sync_ttl = 60 +whitelist = ``` * **enabled**: this is to toggle the feature on or off * **header_name**: this is the HTTP header name that passes the username or email address of the authenticated user to Grafana. Grafana will trust what ever username is contained in this header and automatically log the user in. * **header_property**: this tells Grafana whether the value in the header_name is a username or an email address. (In Grafana you can log in using your account username or account email) * **auto_sign_up**: If set to true, Grafana will automatically create user accounts in the Grafana DB if one does not exist. If set to false, users who do not exist in the GrafanaDB won’t be able to log in, even though their username and password are valid. +* **ldap_sync_ttl**: When both auth.proxy and auth.ldap are enabled, user's organisation and role are synchronised from ldap after the http proxy authentication. You can force ldap re-synchronisation after `ldap_sync_ttl` minutes. +* **whitelist**: Comma separated list of trusted authentication proxies IP. With a fresh install of Grafana, using the above configuration for the authProxy feature, we can send a simple API call to list all users. The only user that will be present is the default “Admin” user that is added the first time Grafana starts up. As you can see all we need to do to authenticate the request is to provide the “X-WEBAUTH-USER” header. diff --git a/jest.config.js b/jest.config.js index cbf4928f926..ead97e39dad 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,10 +1,15 @@ module.exports = { verbose: false, + "globals": { + "ts-jest": { + "tsConfigFile": "tsconfig.json" + } + }, "transform": { "^.+\\.tsx?$": "/node_modules/ts-jest/preprocessor.js" }, - "moduleDirectories": ["/node_modules", "/public"], + "moduleDirectories": ["node_modules", "public"], "roots": [ "/public" ], diff --git a/latest.json b/latest.json index fa72bc4fadf..a746e92c3b3 100644 --- a/latest.json +++ b/latest.json @@ -1,4 +1,4 @@ { - "stable": "4.5.2", - "testing": "4.5.2" + "stable": "4.6.2", + "testing": "4.6.2" } diff --git a/package.json b/package.json index fa6788a5c62..59608e80403 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,8 @@ "@types/enzyme": "^2.8.9", "@types/jest": "^21.1.4", "@types/node": "^8.0.31", - "@types/react": "^16.0.5", - "@types/react-dom": "^15.5.4", + "@types/react": "^16.0.25", + "@types/react-dom": "^16.0.3", "angular-mocks": "^1.6.6", "autoprefixer": "^6.4.0", "awesome-typescript-loader": "^3.2.3", @@ -95,12 +95,12 @@ "zone.js": "^0.7.2" }, "scripts": { - "dev": "node ./node_modules/.bin/webpack --progress --colors --config scripts/webpack/webpack.dev.js", - "watch": "node ./node_modules/.bin/webpack --progress --colors --watch --config scripts/webpack/webpack.dev.js", - "build": "node ./node_modules/.bin/grunt build", - "test": "node ./node_modules/.bin/grunt test", - "test:coverage": "node ./node_modules/.bin/grunt test --coverage=true", - "lint": "node ./node_modules/.bin/tslint -c tslint.json --project tsconfig.json --type-check", + "dev": "webpack --progress --colors --config scripts/webpack/webpack.dev.js", + "watch": "webpack --progress --colors --watch --config scripts/webpack/webpack.dev.js", + "build": "grunt build", + "test": "grunt test", + "test:coverage": "grunt test --coverage=true", + "lint": "tslint -c tslint.json --project tsconfig.json --type-check", "karma": "node ./node_modules/grunt-cli/bin/grunt karma:dev", "jest": "node ./node_modules/jest-cli/bin/jest.js --notify --watch", "precommit": "node ./node_modules/grunt-cli/bin/grunt precommit" @@ -117,25 +117,24 @@ "brace": "^0.10.0", "classnames": "^2.2.5", "clipboard": "^1.7.1", + "d3": "^4.11.0", + "d3-scale-chromatic": "^1.1.1", "eventemitter3": "^2.0.2", "file-saver": "^1.3.3", - "gemini-scrollbar": "https://github.com/grafana/gemini-scrollbar#grafana", "jquery": "^3.2.1", "lodash": "^4.17.4", "moment": "^2.18.1", "mousetrap": "^1.6.0", - "ngreact": "^0.4.1", + "perfect-scrollbar": "^1.2.0", "prop-types": "^15.6.0", - "react": "^16.0.0", - "react-dom": "^16.0.0", + "react": "^16.1.1", + "react-dom": "^16.1.1", "react-grid-layout": "^0.16.0", "react-sizeme": "^2.3.6", "remarkable": "^1.7.1", "rxjs": "^5.4.3", "tether": "^1.4.0", "tether-drop": "https://github.com/torkelo/drop", - "tinycolor2": "^1.4.1", - "d3": "^4.11.0", - "d3-scale-chromatic": "^1.1.1" + "tinycolor2": "^1.4.1" } } diff --git a/packaging/publish/publish_both.sh b/packaging/publish/publish_both.sh index 0a76851f6fa..6c4f5a5c29a 100755 --- a/packaging/publish/publish_both.sh +++ b/packaging/publish/publish_both.sh @@ -1,5 +1,5 @@ #! /usr/bin/env bash -version=4.5.2 +version=4.6.2 wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana_${version}_amd64.deb diff --git a/packaging/rpm/systemd/grafana-server.service b/packaging/rpm/systemd/grafana-server.service index 3e018e8b176..b23e5196e17 100644 --- a/packaging/rpm/systemd/grafana-server.service +++ b/packaging/rpm/systemd/grafana-server.service @@ -9,7 +9,7 @@ After=postgresql.service mariadb.service mysql.service EnvironmentFile=/etc/sysconfig/grafana-server User=grafana Group=grafana -Type=simple +Type=notify Restart=on-failure WorkingDirectory=/usr/share/grafana RuntimeDirectory=grafana diff --git a/pkg/api/annotations.go b/pkg/api/annotations.go index e6454e9cf86..0bf95557abc 100644 --- a/pkg/api/annotations.go +++ b/pkg/api/annotations.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/services/annotations" + "github.com/grafana/grafana/pkg/util" ) func GetAnnotations(c *middleware.Context) Response { @@ -21,6 +22,7 @@ func GetAnnotations(c *middleware.Context) Response { PanelId: c.QueryInt64("panelId"), Limit: c.QueryInt64("limit"), Tags: c.QueryStrings("tags"), + Type: c.Query("type"), } repo := annotations.GetRepository() @@ -75,9 +77,11 @@ func PostAnnotation(c *middleware.Context, cmd dtos.PostAnnotationsCmd) Response return ApiError(500, "Failed to save annotation", err) } + startID := item.Id + // handle regions if cmd.IsRegion { - item.RegionId = item.Id + item.RegionId = startID if item.Data == nil { item.Data = simplejson.New() @@ -93,9 +97,18 @@ func PostAnnotation(c *middleware.Context, cmd dtos.PostAnnotationsCmd) Response if err := repo.Save(&item); err != nil { return ApiError(500, "Failed save annotation for region end time", err) } + + return Json(200, util.DynMap{ + "message": "Annotation added", + "id": startID, + "endId": item.Id, + }) } - return ApiSuccess("Annotation added") + return Json(200, util.DynMap{ + "message": "Annotation added", + "id": startID, + }) } func formatGraphiteAnnotation(what string, data string) string { @@ -154,7 +167,10 @@ func PostGraphiteAnnotation(c *middleware.Context, cmd dtos.PostGraphiteAnnotati return ApiError(500, "Failed to save Graphite annotation", err) } - return ApiSuccess("Graphite annotation added") + return Json(200, util.DynMap{ + "message": "Graphite annotation added", + "id": item.Id, + }) } func UpdateAnnotation(c *middleware.Context, cmd dtos.UpdateAnnotationsCmd) Response { diff --git a/pkg/api/api.go b/pkg/api/api.go index 957ee1e23e6..b707dc17e21 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -212,10 +212,10 @@ func (hs *HttpServer) registerRoutes() { // Data sources apiRoute.Group("/datasources", func(datasourceRoute RouteRegister) { datasourceRoute.Get("/", wrap(GetDataSources)) - datasourceRoute.Post("/", quota("data_source"), bind(m.AddDataSourceCommand{}), AddDataSource) + datasourceRoute.Post("/", quota("data_source"), bind(m.AddDataSourceCommand{}), wrap(AddDataSource)) datasourceRoute.Put("/:id", bind(m.UpdateDataSourceCommand{}), wrap(UpdateDataSource)) - datasourceRoute.Delete("/:id", DeleteDataSourceById) - datasourceRoute.Delete("/name/:name", DeleteDataSourceByName) + datasourceRoute.Delete("/:id", wrap(DeleteDataSourceById)) + datasourceRoute.Delete("/name/:name", wrap(DeleteDataSourceByName)) datasourceRoute.Get("/:id", wrap(GetDataSourceById)) datasourceRoute.Get("/name/:name", wrap(GetDataSourceByName)) }, reqOrgAdmin) @@ -340,8 +340,8 @@ func (hs *HttpServer) registerRoutes() { r.Any("/api/gnet/*", reqSignedIn, ProxyGnetRequest) // Gravatar service. - avt := avatar.CacheServer() - r.Get("/avatar/:hash", avt.ServeHTTP) + avatarCacheServer := avatar.NewCacheServer() + r.Get("/avatar/:hash", avatarCacheServer.Handler) // Websocket r.Any("/ws", hs.streamManager.Serve) diff --git a/pkg/api/avatar/avatar.go b/pkg/api/avatar/avatar.go index 80280fd3cc9..fdf93d06b5d 100644 --- a/pkg/api/avatar/avatar.go +++ b/pkg/api/avatar/avatar.go @@ -24,6 +24,7 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/setting" + "gopkg.in/macaron.v1" ) var gravatarSource string @@ -89,12 +90,12 @@ func (this *Avatar) Update() (err error) { return err } -type service struct { +type CacheServer struct { notFound *Avatar cache map[string]*Avatar } -func (this *service) mustInt(r *http.Request, defaultValue int, keys ...string) (v int) { +func (this *CacheServer) mustInt(r *http.Request, defaultValue int, keys ...string) (v int) { for _, k := range keys { if _, err := fmt.Sscanf(r.FormValue(k), "%d", &v); err == nil { defaultValue = v @@ -103,8 +104,8 @@ func (this *service) mustInt(r *http.Request, defaultValue int, keys ...string) return defaultValue } -func (this *service) ServeHTTP(w http.ResponseWriter, r *http.Request) { - urlPath := r.URL.Path +func (this *CacheServer) Handler(ctx *macaron.Context) { + urlPath := ctx.Req.URL.Path hash := urlPath[strings.LastIndex(urlPath, "/")+1:] var avatar *Avatar @@ -126,20 +127,24 @@ func (this *service) ServeHTTP(w http.ResponseWriter, r *http.Request) { this.cache[hash] = avatar } - w.Header().Set("Content-Type", "image/jpeg") - w.Header().Set("Content-Length", strconv.Itoa(len(avatar.data.Bytes()))) - w.Header().Set("Cache-Control", "private, max-age=3600") + ctx.Resp.Header().Add("Content-Type", "image/jpeg") - if err := avatar.Encode(w); err != nil { + if !setting.EnableGzip { + ctx.Resp.Header().Add("Content-Length", strconv.Itoa(len(avatar.data.Bytes()))) + } + + ctx.Resp.Header().Add("Cache-Control", "private, max-age=3600") + + if err := avatar.Encode(ctx.Resp); err != nil { log.Warn("avatar encode error: %v", err) - w.WriteHeader(500) + ctx.WriteHeader(500) } } -func CacheServer() http.Handler { +func NewCacheServer() *CacheServer { UpdateGravatarSource() - return &service{ + return &CacheServer{ notFound: newNotFound(), cache: make(map[string]*Avatar), } diff --git a/pkg/api/datasources.go b/pkg/api/datasources.go index 747ef8f25e6..b5c5f9cb834 100644 --- a/pkg/api/datasources.go +++ b/pkg/api/datasources.go @@ -33,6 +33,7 @@ func GetDataSources(c *middleware.Context) Response { BasicAuth: ds.BasicAuth, IsDefault: ds.IsDefault, JsonData: ds.JsonData, + ReadOnly: ds.ReadOnly, } if plugin, exists := plugins.DataSources[ds.Type]; exists { @@ -68,59 +69,70 @@ func GetDataSourceById(c *middleware.Context) Response { return Json(200, &dtos) } -func DeleteDataSourceById(c *middleware.Context) { +func DeleteDataSourceById(c *middleware.Context) Response { id := c.ParamsInt64(":id") if id <= 0 { - c.JsonApiErr(400, "Missing valid datasource id", nil) - return + return ApiError(400, "Missing valid datasource id", nil) + } + + ds, err := getRawDataSourceById(id, c.OrgId) + if err != nil { + return ApiError(400, "Failed to delete datasource", nil) + } + + if ds.ReadOnly { + return ApiError(403, "Cannot delete read-only data source", nil) } cmd := &m.DeleteDataSourceByIdCommand{Id: id, OrgId: c.OrgId} - err := bus.Dispatch(cmd) + err = bus.Dispatch(cmd) if err != nil { - c.JsonApiErr(500, "Failed to delete datasource", err) - return + return ApiError(500, "Failed to delete datasource", err) } - c.JsonOK("Data source deleted") + return ApiSuccess("Data source deleted") } -func DeleteDataSourceByName(c *middleware.Context) { +func DeleteDataSourceByName(c *middleware.Context) Response { name := c.Params(":name") if name == "" { - c.JsonApiErr(400, "Missing valid datasource name", nil) - return + return ApiError(400, "Missing valid datasource name", nil) + } + + getCmd := &m.GetDataSourceByNameQuery{Name: name, OrgId: c.OrgId} + if err := bus.Dispatch(getCmd); err != nil { + return ApiError(500, "Failed to delete datasource", err) + } + + if getCmd.Result.ReadOnly { + return ApiError(403, "Cannot delete read-only data source", nil) } cmd := &m.DeleteDataSourceByNameCommand{Name: name, OrgId: c.OrgId} - err := bus.Dispatch(cmd) if err != nil { - c.JsonApiErr(500, "Failed to delete datasource", err) - return + return ApiError(500, "Failed to delete datasource", err) } - c.JsonOK("Data source deleted") + return ApiSuccess("Data source deleted") } -func AddDataSource(c *middleware.Context, cmd m.AddDataSourceCommand) { +func AddDataSource(c *middleware.Context, cmd m.AddDataSourceCommand) Response { cmd.OrgId = c.OrgId if err := bus.Dispatch(&cmd); err != nil { if err == m.ErrDataSourceNameExists { - c.JsonApiErr(409, err.Error(), err) - return + return ApiError(409, err.Error(), err) } - c.JsonApiErr(500, "Failed to add datasource", err) - return + return ApiError(500, "Failed to add datasource", err) } ds := convertModelToDtos(cmd.Result) - c.JSON(200, util.DynMap{ + return Json(200, util.DynMap{ "message": "Datasource added", "id": cmd.Result.Id, "name": cmd.Result.Name, @@ -160,11 +172,14 @@ func fillWithSecureJsonData(cmd *m.UpdateDataSourceCommand) error { } ds, err := getRawDataSourceById(cmd.Id, cmd.OrgId) - if err != nil { return err } + if ds.ReadOnly { + return m.ErrDatasourceIsReadOnly + } + secureJsonData := ds.SecureJsonData.Decrypt() for k, v := range secureJsonData { @@ -201,6 +216,7 @@ func GetDataSourceByName(c *middleware.Context) Response { } dtos := convertModelToDtos(query.Result) + dtos.ReadOnly = true return Json(200, &dtos) } @@ -242,6 +258,7 @@ func convertModelToDtos(ds *m.DataSource) dtos.DataSource { JsonData: ds.JsonData, SecureJsonFields: map[string]bool{}, Version: ds.Version, + ReadOnly: ds.ReadOnly, } for k, v := range ds.SecureJsonData { diff --git a/pkg/api/dtos/datasource.go b/pkg/api/dtos/datasource.go index 7cb36e61ab4..f760486c561 100644 --- a/pkg/api/dtos/datasource.go +++ b/pkg/api/dtos/datasource.go @@ -26,6 +26,7 @@ type DataSource struct { JsonData *simplejson.Json `json:"jsonData,omitempty"` SecureJsonFields map[string]bool `json:"secureJsonFields"` Version int `json:"version"` + ReadOnly bool `json:"readOnly"` } type DataSourceListItemDTO struct { @@ -42,6 +43,7 @@ type DataSourceListItemDTO struct { BasicAuth bool `json:"basicAuth"` IsDefault bool `json:"isDefault"` JsonData *simplejson.Json `json:"jsonData,omitempty"` + ReadOnly bool `json:"readOnly"` } type DataSourceList []DataSourceListItemDTO diff --git a/pkg/api/dtos/index.go b/pkg/api/dtos/index.go index 5661ae1df91..8c7f505277d 100644 --- a/pkg/api/dtos/index.go +++ b/pkg/api/dtos/index.go @@ -24,11 +24,13 @@ type NavLink struct { Id string `json:"id,omitempty"` Text string `json:"text,omitempty"` Description string `json:"description,omitempty"` + SubTitle string `json:"subTitle,omitempty"` Icon string `json:"icon,omitempty"` Img string `json:"img,omitempty"` Url string `json:"url,omitempty"` Target string `json:"target,omitempty"` Divider bool `json:"divider,omitempty"` HideFromMenu bool `json:"hideFromMenu,omitempty"` + HideFromTabs bool `json:"hideFromTabs,omitempty"` Children []*NavLink `json:"children,omitempty"` } diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 037d2ae98e5..89456d20d8c 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -146,12 +146,13 @@ func (hs *HttpServer) newMacaron() *macaron.Macaron { m := macaron.New() m.Use(middleware.Logger()) - m.Use(middleware.Recovery()) if setting.EnableGzip { m.Use(middleware.Gziper()) } + m.Use(middleware.Recovery()) + for _, route := range plugins.StaticRoutes { pluginRoute := path.Join("/public/plugins/", route.PluginId) hs.log.Debug("Plugins: Adding route", "route", pluginRoute, "dir", route.Directory) @@ -193,7 +194,8 @@ func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) { } func (hs *HttpServer) healthHandler(ctx *macaron.Context) { - if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/api/health" { + notHeadOrGet := ctx.Req.Method != http.MethodGet && ctx.Req.Method != http.MethodHead + if notHeadOrGet || ctx.Req.URL.Path != "/api/health" { return } diff --git a/pkg/api/index.go b/pkg/api/index.go index bf5a128a03f..e179828d2f7 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -90,26 +90,29 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { if c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR { data.NavTree = append(data.NavTree, &dtos.NavLink{ Text: "Create", + Id: "create", Icon: "fa fa-fw fa-plus", Url: "#", Children: []*dtos.NavLink{ {Text: "Dashboard", Icon: "gicon gicon-dashboard-new", Url: setting.AppSubUrl + "/dashboard/new"}, - {Text: "Folder", Icon: "gicon gicon-folder-new", Url: setting.AppSubUrl + "/dashboard/new/?editview=new-folder"}, - {Text: "Import", Icon: "gicon gicon-dashboard-import", Url: setting.AppSubUrl + "/dashboard/new/?editview=import"}, + {Text: "Folder", SubTitle: "Create a new folder to organize your dashboards", Id: "folder", Icon: "gicon gicon-folder-new", Url: setting.AppSubUrl + "/dashboards/folder/new"}, + {Text: "Import", SubTitle: "Import dashboard from file or Grafana.com", Id: "import", Icon: "gicon gicon-dashboard-import", Url: setting.AppSubUrl + "/dashboard/import"}, }, }) } dashboardChildNavs := []*dtos.NavLink{ - {Text: "Home", Url: setting.AppSubUrl + "/", Icon: "fa fa-fw fa-home"}, + {Text: "Home", Url: setting.AppSubUrl + "/", Icon: "fa fa-fw fa-home", HideFromTabs: true}, + {Divider: true, HideFromTabs: true}, + {Text: "Manage", Id: "manage-dashboards", Url: setting.AppSubUrl + "/dashboards", Icon: "fa fa-fw fa-sitemap"}, {Text: "Playlists", Id: "playlists", Url: setting.AppSubUrl + "/playlists", Icon: "fa fa-fw fa-film"}, {Text: "Snapshots", Id: "snapshots", Url: setting.AppSubUrl + "/dashboard/snapshots", Icon: "icon-gf icon-gf-fw icon-gf-snapshot"}, - {Text: "Dashboard List", Description: "Manage Dashboards And Folders", Id: "dashboards", Url: setting.AppSubUrl + "/dashboards", Icon: "fa fa-fw fa-bars"}, } data.NavTree = append(data.NavTree, &dtos.NavLink{ Text: "Dashboards", Id: "dashboards", + SubTitle: "Manage dashboards & folders", Icon: "gicon gicon-dashboard", Url: setting.AppSubUrl + "/", Children: dashboardChildNavs, @@ -117,22 +120,23 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { if c.IsSignedIn { profileNode := &dtos.NavLink{ - Text: c.SignedInUser.Login, + Text: c.SignedInUser.Name, + SubTitle: c.SignedInUser.Login, Id: "profile", Img: data.User.GravatarUrl, Url: setting.AppSubUrl + "/profile", HideFromMenu: true, Children: []*dtos.NavLink{ - {Text: "Your profile", Url: setting.AppSubUrl + "/profile", Icon: "fa fa-fw fa-sliders"}, + {Text: "Preferences", Id: "profile-settings", Url: setting.AppSubUrl + "/profile", Icon: "fa fa-fw fa-sliders"}, {Text: "Change Password", Id: "change-password", Url: setting.AppSubUrl + "/profile/password", Icon: "fa fa-fw fa-lock", HideFromMenu: true}, }, } if !setting.DisableSignoutMenu { // add sign out first - profileNode.Children = append([]*dtos.NavLink{ - {Text: "Sign out", Url: setting.AppSubUrl + "/logout", Icon: "fa fa-fw fa-sign-out", Target: "_self"}, - }, profileNode.Children...) + profileNode.Children = append(profileNode.Children, &dtos.NavLink{ + Text: "Sign out", Id: "sign-out", Url: setting.AppSubUrl + "/logout", Icon: "fa fa-fw fa-sign-out", Target: "_self", + }) } data.NavTree = append(data.NavTree, profileNode) @@ -140,12 +144,13 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { if setting.AlertingEnabled && (c.OrgRole == m.ROLE_ADMIN || c.OrgRole == m.ROLE_EDITOR) { alertChildNavs := []*dtos.NavLink{ - {Text: "Alert List", Id: "alert-list", Url: setting.AppSubUrl + "/alerting/list", Icon: "fa fa-fw fa-list-ul"}, - {Text: "Notification channels", Id: "channels", Url: setting.AppSubUrl + "/alerting/notifications", Icon: "fa fa-fw fa-bell-o"}, + {Text: "Alert Rules", Id: "alert-list", Url: setting.AppSubUrl + "/alerting/list", Icon: "fa fa-fw fa-list-ul"}, + {Text: "Notification channels", Id: "channels", Url: setting.AppSubUrl + "/alerting/notifications", Icon: "gicon gicon-alert-notification-channel"}, } data.NavTree = append(data.NavTree, &dtos.NavLink{ Text: "Alerting", + SubTitle: "Alert rules & notifications", Id: "alerting", Icon: "gicon gicon-alert", Url: setting.AppSubUrl + "/alerting/list", @@ -202,10 +207,11 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { if c.OrgRole == m.ROLE_ADMIN { cfgNode := &dtos.NavLink{ - Id: "cfg", - Text: "Configuration", - Icon: "fa fa-fw fa-cogs", - Url: setting.AppSubUrl + "/configuration", + Id: "cfg", + Text: "Configuration", + SubTitle: "Organization: " + c.OrgName, + Icon: "fa fa-fw fa-cog", + Url: setting.AppSubUrl + "/datasources", Children: []*dtos.NavLink{ { Text: "Data Sources", @@ -213,29 +219,6 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { Description: "Add and configure data sources", Id: "datasources", Url: setting.AppSubUrl + "/datasources", - Children: []*dtos.NavLink{ - {Text: "List", Url: setting.AppSubUrl + "/datasources", Icon: "gicon gicon-datasources"}, - {Text: "New", Url: setting.AppSubUrl + "/datasources", Icon: "fa fa-fw fa-plus"}, - }, - }, - { - Text: "Preferences", - Id: "org", - Description: "Organization preferences", - Icon: "fa fa-fw fa-sliders", - Url: setting.AppSubUrl + "/org", - }, - { - Text: "Plugins", - Id: "plugins", - Description: "View and configure plugins", - Icon: "icon-gf icon-gf-fw icon-gf-apps", - Url: setting.AppSubUrl + "/plugins", - Children: []*dtos.NavLink{ - {Text: "Panels", Url: setting.AppSubUrl + "/plugins?type=panel", Icon: "fa fa-fw fa-stop"}, - {Text: "Data sources", Url: setting.AppSubUrl + "/plugins?type=datasource", Icon: "icon-gf icon-gf-datasources"}, - {Text: "Apps", Url: setting.AppSubUrl + "/plugins?type=app", Icon: "icon-gf icon-gf-apps"}, - }, }, { Text: "Members", @@ -245,12 +228,27 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { Url: setting.AppSubUrl + "/org/users", }, { - Text: "Groups", - Id: "users", + Text: "Teams", + Id: "teams", Description: "Manage org groups", - Icon: "fa fa-fw fa-users", + Icon: "gicon gicon-user-group", Url: setting.AppSubUrl + "/org/user-groups", }, + { + Text: "Plugins", + Id: "plugins", + Description: "View and configure plugins", + Icon: "icon-gf icon-gf-fw icon-gf-apps", + Url: setting.AppSubUrl + "/plugins", + }, + { + Text: "Preferences", + Id: "org-settings", + Description: "Organization preferences", + Icon: "fa fa-fw fa-sliders", + Url: setting.AppSubUrl + "/org", + }, + { Text: "API Keys", Id: "apikeys", @@ -263,16 +261,21 @@ func setIndexViewData(c *middleware.Context) (*dtos.IndexViewData, error) { if c.IsGrafanaAdmin { cfgNode.Children = append(cfgNode.Children, &dtos.NavLink{ - Text: "Server Admin", - Id: "admin", - Icon: "fa fa-fw fa-shield", - Url: setting.AppSubUrl + "/admin", + Divider: true, HideFromTabs: true, + }) + cfgNode.Children = append(cfgNode.Children, &dtos.NavLink{ + Text: "Server Admin", + HideFromTabs: true, + SubTitle: "Manage all users & orgs", + Id: "admin", + Icon: "fa fa-fw fa-shield", + Url: setting.AppSubUrl + "/admin/users", Children: []*dtos.NavLink{ - {Text: "Users", Id: "global-users", Url: setting.AppSubUrl + "/admin/users"}, - {Text: "Orgs", Id: "global-orgs", Url: setting.AppSubUrl + "/admin/orgs"}, - {Text: "Server Settings", Id: "server-settings", Url: setting.AppSubUrl + "/admin/settings"}, - {Text: "Server Stats", Id: "server-stats", Url: setting.AppSubUrl + "/admin/stats"}, - {Text: "Style Guide", Id: "styleguide", Url: setting.AppSubUrl + "/styleguide"}, + {Text: "Users", Id: "global-users", Url: setting.AppSubUrl + "/admin/users", Icon: "icon-gf icon-gf-fw icon-gf-users"}, + {Text: "Orgs", Id: "global-orgs", Url: setting.AppSubUrl + "/admin/orgs", Icon: "gicon gicon-org"}, + {Text: "Settings", Id: "server-settings", Url: setting.AppSubUrl + "/admin/settings", Icon: "fa fa-fw fa-sliders"}, + {Text: "Stats", Id: "server-stats", Url: setting.AppSubUrl + "/admin/stats", Icon: "fa fa-fw fa-bar-chart"}, + {Text: "Style Guide", Id: "styleguide", Url: setting.AppSubUrl + "/styleguide", Icon: "fa fa-fw fa-eyedropper"}, }, }) } diff --git a/pkg/api/render.go b/pkg/api/render.go index 2a5ce27c210..6d1b49e4462 100644 --- a/pkg/api/render.go +++ b/pkg/api/render.go @@ -10,7 +10,11 @@ import ( ) func RenderToPng(c *middleware.Context) { - queryReader := util.NewUrlQueryReader(c.Req.URL) + queryReader, err := util.NewUrlQueryReader(c.Req.URL) + if err != nil { + c.Handle(400, "Render parameters error", err) + return + } queryParams := fmt.Sprintf("?%s", c.Req.URL.RawQuery) renderOpts := &renderer.RenderOpts{ diff --git a/pkg/api/route_register.go b/pkg/api/route_register.go index daa6f35e52c..76ebb633ca1 100644 --- a/pkg/api/route_register.go +++ b/pkg/api/route_register.go @@ -81,8 +81,6 @@ func (rr *routeRegister) Register(router Router) *macaron.Router { } func (rr *routeRegister) route(pattern, method string, handlers ...macaron.Handler) { - //inject tracing - h := make([]macaron.Handler, 0) for _, fn := range rr.namedMiddleware { h = append(h, fn(pattern)) diff --git a/pkg/api/search.go b/pkg/api/search.go index 3998fb0298c..fee062a5599 100644 --- a/pkg/api/search.go +++ b/pkg/api/search.go @@ -15,7 +15,6 @@ func Search(c *middleware.Context) { starred := c.Query("starred") limit := c.QueryInt("limit") dashboardType := c.Query("type") - folderId := c.QueryInt64("folderId") if limit == 0 { limit = 1000 @@ -29,6 +28,14 @@ func Search(c *middleware.Context) { } } + folderIds := make([]int64, 0) + for _, id := range c.QueryStrings("folderIds") { + folderId, err := strconv.ParseInt(id, 10, 64) + if err == nil { + folderIds = append(folderIds, folderId) + } + } + searchQuery := search.Query{ Title: query, Tags: tags, @@ -38,7 +45,7 @@ func Search(c *middleware.Context) { OrgId: c.OrgId, DashboardIds: dbids, Type: dashboardType, - FolderId: folderId, + FolderIds: folderIds, } err := bus.Dispatch(&searchQuery) diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index fa63f05efba..183e4b047cd 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -16,7 +16,6 @@ import ( "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" _ "github.com/grafana/grafana/pkg/services/alerting/conditions" @@ -88,11 +87,6 @@ func main() { server.Start() } -func initSql() { - sqlstore.NewEngine() - sqlstore.EnsureAdminUser() -} - func listenToSystemSignals(server models.GrafanaServer) { signalChan := make(chan os.Signal, 1) ignoreChan := make(chan os.Signal, 1) diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go index 4bbabbe3273..f5c6b0d1cee 100644 --- a/pkg/cmd/grafana-server/server.go +++ b/pkg/cmd/grafana-server/server.go @@ -3,12 +3,17 @@ package main import ( "context" "flag" + "fmt" "io/ioutil" + "net" "os" "path/filepath" "strconv" "time" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "github.com/grafana/grafana/pkg/services/provisioning" + "golang.org/x/sync/errgroup" "github.com/grafana/grafana/pkg/api" @@ -21,7 +26,9 @@ import ( "github.com/grafana/grafana/pkg/services/cleanup" "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/services/search" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/social" "github.com/grafana/grafana/pkg/tracing" ) @@ -54,12 +61,19 @@ func (g *GrafanaServerImpl) Start() { g.writePIDFile() initSql() + metrics.Init(setting.Cfg) search.Init() login.Init() social.NewOAuthService() plugins.Init() + if err := provisioning.StartUp(setting.DatasourcesPath); err != nil { + logger.Error("Failed to provision Grafana from config", "error", err) + g.Shutdown(1, "Startup failed") + return + } + closer, err := tracing.Init(setting.Cfg) if err != nil { g.log.Error("Tracing settings is not valid", "error", err) @@ -84,9 +98,15 @@ func (g *GrafanaServerImpl) Start() { return } + SendSystemdNotification("READY=1") g.startHttpServer() } +func initSql() { + sqlstore.NewEngine() + sqlstore.EnsureAdminUser() +} + func (g *GrafanaServerImpl) initLogging() { err := setting.NewConfigContext(&setting.CommandLineArgs{ Config: *configFile, @@ -152,3 +172,28 @@ func (g *GrafanaServerImpl) writePIDFile() { g.log.Info("Writing PID file", "path", *pidFile, "pid", pid) } + +func SendSystemdNotification(state string) error { + notifySocket := os.Getenv("NOTIFY_SOCKET") + + if notifySocket == "" { + return fmt.Errorf("NOTIFY_SOCKET environment variable empty or unset.") + } + + socketAddr := &net.UnixAddr{ + Name: notifySocket, + Net: "unixgram", + } + + conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr) + + if err != nil { + return err + } + + _, err = conn.Write([]byte(state)) + + conn.Close() + + return err +} diff --git a/pkg/components/imguploader/gcsuploader.go b/pkg/components/imguploader/gcsuploader.go index 2271cec7db0..cb1ae19059d 100644 --- a/pkg/components/imguploader/gcsuploader.go +++ b/pkg/components/imguploader/gcsuploader.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "net/http" "os" + "path" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/util" @@ -20,19 +21,22 @@ const ( type GCSUploader struct { keyFile string bucket string + path string log log.Logger } -func NewGCSUploader(keyFile, bucket string) *GCSUploader { +func NewGCSUploader(keyFile, bucket, path string) *GCSUploader { return &GCSUploader{ keyFile: keyFile, bucket: bucket, + path: path, log: log.New("gcsuploader"), } } func (u *GCSUploader) Upload(ctx context.Context, imageDiskPath string) (string, error) { - key := util.GetRandomString(20) + ".png" + fileName := util.GetRandomString(20) + ".png" + key := path.Join(u.path, fileName) u.log.Debug("Opening key file ", u.keyFile) data, err := ioutil.ReadFile(u.keyFile) diff --git a/pkg/components/imguploader/imguploader.go b/pkg/components/imguploader/imguploader.go index 728614735d0..fd14b5d6739 100644 --- a/pkg/components/imguploader/imguploader.go +++ b/pkg/components/imguploader/imguploader.go @@ -73,8 +73,9 @@ func NewImageUploader() (ImageUploader, error) { keyFile := gcssec.Key("key_file").MustString("") bucketName := gcssec.Key("bucket").MustString("") + path := gcssec.Key("path").MustString("") - return NewGCSUploader(keyFile, bucketName), nil + return NewGCSUploader(keyFile, bucketName, path), nil } return NopImageUploader{}, nil diff --git a/pkg/log/file.go b/pkg/log/file.go index a8d35ba6b81..721db1e55b3 100644 --- a/pkg/log/file.go +++ b/pkg/log/file.go @@ -5,9 +5,10 @@ package log import ( + "bytes" "errors" "fmt" - "io/ioutil" + "io" "os" "path/filepath" "strings" @@ -124,6 +125,30 @@ func (w *FileLogWriter) createLogFile() (*os.File, error) { return os.OpenFile(w.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) } +func (w *FileLogWriter) lineCounter() (int, error) { + r, err := os.OpenFile(w.Filename, os.O_RDONLY, 0644) + if err != nil { + return 0, fmt.Errorf("lineCounter Open File : %s", err) + } + buf := make([]byte, 32*1024) + count := 0 + + for { + c, err := r.Read(buf) + count += bytes.Count(buf[:c], []byte{'\n'}) + switch { + case err == io.EOF: + if err := r.Close(); err != nil { + return count, err + } + return count, nil + + case err != nil: + return count, err + } + } +} + func (w *FileLogWriter) initFd() error { fd := w.mw.fd finfo, err := fd.Stat() @@ -133,11 +158,11 @@ func (w *FileLogWriter) initFd() error { w.maxsize_cursize = int(finfo.Size()) w.daily_opendate = time.Now().Day() if finfo.Size() > 0 { - content, err := ioutil.ReadFile(w.Filename) + count, err := w.lineCounter() if err != nil { return err } - w.maxlines_curlines = len(strings.Split(string(content), "\n")) + w.maxlines_curlines = count } else { w.maxlines_curlines = 0 } diff --git a/pkg/log/file_test.go b/pkg/log/file_test.go new file mode 100644 index 00000000000..3e98e0786cc --- /dev/null +++ b/pkg/log/file_test.go @@ -0,0 +1,45 @@ +package log + +import ( + "os" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func (w *FileLogWriter) WriteLine(line string) error { + n, err := w.mw.Write([]byte(line)) + if err != nil { + return err + } + w.docheck(n) + return nil +} + +func TestLogFile(t *testing.T) { + + Convey("When logging to file", t, func() { + fileLogWrite := NewFileWriter() + So(fileLogWrite, ShouldNotBeNil) + + fileLogWrite.Filename = "grafana_test.log" + err := fileLogWrite.Init() + So(err, ShouldBeNil) + + Convey("Log file is empty", func() { + So(fileLogWrite.maxlines_curlines, ShouldEqual, 0) + }) + + Convey("Logging should add lines", func() { + err := fileLogWrite.WriteLine("test1\n") + err = fileLogWrite.WriteLine("test2\n") + err = fileLogWrite.WriteLine("test3\n") + So(err, ShouldBeNil) + So(fileLogWrite.maxlines_curlines, ShouldEqual, 3) + }) + + fileLogWrite.Close() + err = os.Remove(fileLogWrite.Filename) + So(err, ShouldBeNil) + }) +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4b155ae3208..4d7de98f2ea 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -225,7 +225,7 @@ func init() { M_DataSource_ProxyReq_Timer = prometheus.NewSummary(prometheus.SummaryOpts{ Name: "api_dataproxy_request_all_milliseconds", - Help: "summary for dashboard search duration", + Help: "summary for dataproxy request duration", Namespace: exporterName, }) diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index a279743aed3..0d9e0e5b973 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -363,6 +363,7 @@ type scenarioContext struct { respJson map[string]interface{} handlerFunc handlerFunc defaultHandler macaron.Handler + url string req *http.Request } diff --git a/pkg/middleware/recovery.go b/pkg/middleware/recovery.go index b63bc623549..0c9dc4670e2 100644 --- a/pkg/middleware/recovery.go +++ b/pkg/middleware/recovery.go @@ -123,23 +123,22 @@ func Recovery() macaron.Handler { c.Data["ErrorMsg"] = string(stack) } - c.HTML(500, "500") + ctx, ok := c.Data["ctx"].(*Context) - // // Lookup the current responsewriter - // val := c.GetVal(inject.InterfaceOf((*http.ResponseWriter)(nil))) - // res := val.Interface().(http.ResponseWriter) - // - // // respond with panic message while in development mode - // var body []byte - // if setting.Env == setting.DEV { - // res.Header().Set("Content-Type", "text/html") - // body = []byte(fmt.Sprintf(panicHtml, err, err, stack)) - // } - // - // res.WriteHeader(http.StatusInternalServerError) - // if nil != body { - // res.Write(body) - // } + if ok && ctx.IsApiRequest() { + resp := make(map[string]interface{}) + resp["message"] = "Internal Server Error - Check the Grafana server logs for the detailed error message." + + if c.Data["ErrorMsg"] != nil { + resp["error"] = fmt.Sprintf("%v - %v", c.Data["Title"], c.Data["ErrorMsg"]) + } else { + resp["error"] = c.Data["Title"] + } + + c.JSON(500, resp) + } else { + c.HTML(500, "500") + } } }() diff --git a/pkg/middleware/recovery_test.go b/pkg/middleware/recovery_test.go new file mode 100644 index 00000000000..299186945ee --- /dev/null +++ b/pkg/middleware/recovery_test.go @@ -0,0 +1,79 @@ +package middleware + +import ( + "path/filepath" + "testing" + + "github.com/go-macaron/session" + "github.com/grafana/grafana/pkg/bus" + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/macaron.v1" +) + +func TestRecoveryMiddleware(t *testing.T) { + Convey("Given an api route that panics", t, func() { + apiUrl := "/api/whatever" + recoveryScenario("recovery middleware should return json", apiUrl, func(sc *scenarioContext) { + sc.handlerFunc = PanicHandler + sc.fakeReq("GET", apiUrl).exec() + sc.req.Header.Add("content-type", "application/json") + + So(sc.resp.Code, ShouldEqual, 500) + So(sc.respJson["message"], ShouldStartWith, "Internal Server Error - Check the Grafana server logs for the detailed error message.") + So(sc.respJson["error"], ShouldStartWith, "Server Error") + }) + }) + + Convey("Given a non-api route that panics", t, func() { + apiUrl := "/whatever" + recoveryScenario("recovery middleware should return html", apiUrl, func(sc *scenarioContext) { + sc.handlerFunc = PanicHandler + sc.fakeReq("GET", apiUrl).exec() + + So(sc.resp.Code, ShouldEqual, 500) + So(sc.resp.Header().Get("content-type"), ShouldEqual, "text/html; charset=UTF-8") + So(sc.resp.Body.String(), ShouldContainSubstring, "Grafana - Error") + }) + }) +} + +func PanicHandler(c *Context) { + panic("Handler has panicked") +} + +func recoveryScenario(desc string, url string, fn scenarioFunc) { + Convey(desc, func() { + defer bus.ClearBusHandlers() + + sc := &scenarioContext{ + url: url, + } + viewsPath, _ := filepath.Abs("../../public/views") + + sc.m = macaron.New() + sc.m.Use(Recovery()) + + sc.m.Use(macaron.Renderer(macaron.RenderOptions{ + Directory: viewsPath, + Delims: macaron.Delims{Left: "[[", Right: "]]"}, + })) + + sc.m.Use(GetContextHandler()) + // mock out gc goroutine + startSessionGC = func() {} + sc.m.Use(Sessioner(&session.Options{})) + sc.m.Use(OrgRedirect()) + sc.m.Use(AddDefaultResponseHeaders()) + + sc.defaultHandler = func(c *Context) { + sc.context = c + if sc.handlerFunc != nil { + sc.handlerFunc(sc.context) + } + } + + sc.m.Get(url, sc.defaultHandler) + + fn(sc) + }) +} diff --git a/pkg/models/dashboard_version.go b/pkg/models/dashboard_version.go index 06b5797e57c..4acb4282a58 100644 --- a/pkg/models/dashboard_version.go +++ b/pkg/models/dashboard_version.go @@ -69,3 +69,10 @@ type GetDashboardVersionsQuery struct { Result []*DashboardVersionDTO } + +// +// Commands +// + +type DeleteExpiredVersionsCommand struct { +} diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index eadf4cd15e5..1d84d8250a6 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -68,6 +68,15 @@ func NewDashboard(title string) *Dashboard { return dash } +// NewDashboardFolder creates a new dashboard folder +func NewDashboardFolder(title string) *Dashboard { + folder := NewDashboard(title) + folder.Data.Set("schemaVersion", 16) + folder.Data.Set("editable", true) + folder.Data.Set("hideControls", true) + return folder +} + // GetTags turns the tags in data json into go string array func (dash *Dashboard) GetTags() []string { return dash.Data.Get("tags").MustStringArray() diff --git a/pkg/models/datasource.go b/pkg/models/datasource.go index 7fc8f935124..9c1cb6fe9e2 100644 --- a/pkg/models/datasource.go +++ b/pkg/models/datasource.go @@ -27,6 +27,7 @@ var ( ErrDataSourceNotFound = errors.New("Data source not found") ErrDataSourceNameExists = errors.New("Data source with same name already exists") ErrDataSourceUpdatingOldVersion = errors.New("Trying to update old version of datasource") + ErrDatasourceIsReadOnly = errors.New("Data source is readonly. Can only be updated from configuration.") ) type DsAccess string @@ -50,6 +51,7 @@ type DataSource struct { IsDefault bool JsonData *simplejson.Json SecureJsonData securejsondata.SecureJsonData + ReadOnly bool Created time.Time Updated time.Time @@ -109,6 +111,7 @@ type AddDataSourceCommand struct { IsDefault bool `json:"isDefault"` JsonData *simplejson.Json `json:"jsonData"` SecureJsonData map[string]string `json:"secureJsonData"` + ReadOnly bool `json:"readOnly"` OrgId int64 `json:"-"` @@ -132,6 +135,7 @@ type UpdateDataSourceCommand struct { JsonData *simplejson.Json `json:"jsonData"` SecureJsonData map[string]string `json:"secureJsonData"` Version int `json:"version"` + ReadOnly bool `json:"readOnly"` OrgId int64 `json:"-"` Id int64 `json:"-"` @@ -142,11 +146,15 @@ type UpdateDataSourceCommand struct { type DeleteDataSourceByIdCommand struct { Id int64 OrgId int64 + + DeletedDatasourcesCount int64 } type DeleteDataSourceByNameCommand struct { Name string OrgId int64 + + DeletedDatasourcesCount int64 } // --------------------- @@ -157,6 +165,10 @@ type GetDataSourcesQuery struct { Result []*DataSource } +type GetAllDataSourcesQuery struct { + Result []*DataSource +} + type GetDataSourceByIdQuery struct { Id int64 OrgId int64 diff --git a/pkg/plugins/dashboard_importer.go b/pkg/plugins/dashboard_importer.go index 1b3e4bac182..bf516818e3c 100644 --- a/pkg/plugins/dashboard_importer.go +++ b/pkg/plugins/dashboard_importer.go @@ -69,6 +69,7 @@ func ImportDashboard(cmd *ImportDashboardCommand) error { UserId: cmd.UserId, Overwrite: cmd.Overwrite, PluginId: cmd.PluginId, + FolderId: dashboard.FolderId, } if err := bus.Dispatch(&saveCmd); err != nil { diff --git a/pkg/plugins/dashboard_importer_test.go b/pkg/plugins/dashboard_importer_test.go index d2897fad1cd..78df94309f8 100644 --- a/pkg/plugins/dashboard_importer_test.go +++ b/pkg/plugins/dashboard_importer_test.go @@ -13,16 +13,9 @@ import ( ) func TestDashboardImport(t *testing.T) { - - Convey("When importing plugin dashboard", t, func() { - setting.Cfg = ini.Empty() - sec, _ := setting.Cfg.NewSection("plugin.test-app") - sec.NewKey("path", "../../tests/test-app") - err := Init() - - So(err, ShouldBeNil) - + pluginScenario("When importing a plugin dashboard", t, func() { var importedDash *m.Dashboard + bus.AddHandler("test", func(cmd *m.SaveDashboardCommand) error { importedDash = cmd.GetDashboardModel() cmd.Result = importedDash @@ -39,7 +32,7 @@ func TestDashboardImport(t *testing.T) { }, } - err = ImportDashboard(&cmd) + err := ImportDashboard(&cmd) So(err, ShouldBeNil) Convey("should install dashboard", func() { @@ -59,16 +52,16 @@ func TestDashboardImport(t *testing.T) { Convey("When evaling dashboard template", t, func() { template, _ := simplejson.NewJson([]byte(`{ - "__inputs": [ - { - "name": "DS_NAME", - "type": "datasource" - } - ], - "test": { - "prop": "${DS_NAME}" - } - }`)) + "__inputs": [ + { + "name": "DS_NAME", + "type": "datasource" + } + ], + "test": { + "prop": "${DS_NAME}" + } + }`)) evaluator := &DashTemplateEvaluator{ template: template, @@ -92,3 +85,16 @@ func TestDashboardImport(t *testing.T) { }) } + +func pluginScenario(desc string, t *testing.T, fn func()) { + Convey("Given a plugin", t, func() { + setting.Cfg = ini.Empty() + sec, _ := setting.Cfg.NewSection("plugin.test-app") + sec.NewKey("path", "../../tests/test-app") + err := Init() + + So(err, ShouldBeNil) + + Convey(desc, fn) + }) +} diff --git a/pkg/services/alerting/extractor_test.go b/pkg/services/alerting/extractor_test.go index b7f83404452..68054fab5ef 100644 --- a/pkg/services/alerting/extractor_test.go +++ b/pkg/services/alerting/extractor_test.go @@ -366,7 +366,6 @@ func TestAlertRuleExtraction(t *testing.T) { "steppedLine": false, "targets": [ { - "dsType": "influxdb", "groupBy": [ { "params": [ @@ -411,7 +410,6 @@ func TestAlertRuleExtraction(t *testing.T) { "tags": [] }, { - "dsType": "influxdb", "groupBy": [ { "params": [ diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go index ed1451da419..e051a71740a 100644 --- a/pkg/services/alerting/notifiers/slack.go +++ b/pkg/services/alerting/notifiers/slack.go @@ -6,6 +6,7 @@ import ( "io" "mime/multipart" "os" + "path/filepath" "time" "github.com/grafana/grafana/pkg/bus" @@ -176,7 +177,7 @@ func (this *SlackNotifier) Notify(evalContext *alerting.EvalContext) error { func SlackFileUpload(evalContext *alerting.EvalContext, log log.Logger, url string, recipient string, token string) error { if evalContext.ImageOnDiskPath == "" { - evalContext.ImageOnDiskPath = "public/img/mixed_styles.png" + evalContext.ImageOnDiskPath = filepath.Join(setting.HomePath, "public/img/mixed_styles.png") } log.Info("Uploading to slack via file.upload API") headers, uploadBody, err := GenerateSlackBody(evalContext.ImageOnDiskPath, token, recipient) diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go new file mode 100644 index 00000000000..200a8594428 --- /dev/null +++ b/pkg/services/alerting/notifiers/teams.go @@ -0,0 +1,122 @@ +package notifiers + +import ( + "encoding/json" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" +) + +func init() { + alerting.RegisterNotifier(&alerting.NotifierPlugin{ + Type: "teams", + Name: "Microsoft Teams", + Description: "Sends notifications using Incomming Webhook connector to Microsoft Teams", + Factory: NewTeamsNotifier, + OptionsTemplate: ` +

Teams settings

+
+ Url + +
+ `, + }) + +} + +func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) { + url := model.Settings.Get("url").MustString() + if url == "" { + return nil, alerting.ValidationError{Reason: "Could not find url property in settings"} + } + + return &TeamsNotifier{ + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + Url: url, + log: log.New("alerting.notifier.teams"), + }, nil +} + +type TeamsNotifier struct { + NotifierBase + Url string + Recipient string + Mention string + log log.Logger +} + +func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error { + this.log.Info("Executing teams notification", "ruleId", evalContext.Rule.Id, "notification", this.Name) + + ruleUrl, err := evalContext.GetRuleUrl() + if err != nil { + this.log.Error("Failed get rule link", "error", err) + return err + } + + fields := make([]map[string]interface{}, 0) + fieldLimitCount := 4 + for index, evt := range evalContext.EvalMatches { + fields = append(fields, map[string]interface{}{ + "name": evt.Metric, + "value": evt.Value, + }) + if index > fieldLimitCount { + break + } + } + + if evalContext.Error != nil { + fields = append(fields, map[string]interface{}{ + "name": "Error message", + "value": evalContext.Error.Error(), + }) + } + + message := this.Mention + if evalContext.Rule.State != m.AlertStateOK { //dont add message when going back to alert state ok. + message += " " + evalContext.Rule.Message + } + + body := map[string]interface{}{ + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "summary": message, + "title": evalContext.GetNotificationTitle(), + "themeColor": evalContext.GetStateModel().Color, + "sections": []map[string]interface{}{ + { + "title": "Details", + "facts": fields, + "images": []map[string]interface{}{ + { + "image": evalContext.ImagePublicUrl, + }, + }, + "text": message, + "potentialAction": []map[string]interface{}{ + { + "@context": "http://schema.org", + "@type": "ViewAction", + "name": "View Rule", + "target": []string{ + ruleUrl, + }, + }, + }, + }, + }, + } + + data, _ := json.Marshal(&body) + cmd := &m.SendWebhookSync{Url: this.Url, Body: string(data)} + + if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { + this.log.Error("Failed to send teams notification", "error", err, "webhook", this.Name) + return err + } + + return nil +} diff --git a/pkg/services/alerting/notifiers/teams_test.go b/pkg/services/alerting/notifiers/teams_test.go new file mode 100644 index 00000000000..a9647350736 --- /dev/null +++ b/pkg/services/alerting/notifiers/teams_test.go @@ -0,0 +1,75 @@ +package notifiers + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestTeamsNotifier(t *testing.T) { + Convey("Teams notifier tests", t, func() { + + Convey("Parsing alert notification from settings", func() { + Convey("empty settings should return error", func() { + json := `{ }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "teams", + Settings: settingsJSON, + } + + _, err := NewTeamsNotifier(model) + So(err, ShouldNotBeNil) + }) + + Convey("from settings", func() { + json := ` + { + "url": "http://google.com" + }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "teams", + Settings: settingsJSON, + } + + not, err := NewTeamsNotifier(model) + teamsNotifier := not.(*TeamsNotifier) + + So(err, ShouldBeNil) + So(teamsNotifier.Name, ShouldEqual, "ops") + So(teamsNotifier.Type, ShouldEqual, "teams") + So(teamsNotifier.Url, ShouldEqual, "http://google.com") + }) + + Convey("from settings with Recipient and Mention", func() { + json := ` + { + "url": "http://google.com" + }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "ops", + Type: "teams", + Settings: settingsJSON, + } + + not, err := NewTeamsNotifier(model) + teamsNotifier := not.(*TeamsNotifier) + + So(err, ShouldBeNil) + So(teamsNotifier.Name, ShouldEqual, "ops") + So(teamsNotifier.Type, ShouldEqual, "teams") + So(teamsNotifier.Url, ShouldEqual, "http://google.com") + }) + + }) + }) +} diff --git a/pkg/services/annotations/annotations.go b/pkg/services/annotations/annotations.go index 2fdc824f172..02f927a76ba 100644 --- a/pkg/services/annotations/annotations.go +++ b/pkg/services/annotations/annotations.go @@ -17,6 +17,7 @@ type ItemQuery struct { DashboardId int64 `json:"dashboardId"` PanelId int64 `json:"panelId"` Tags []string `json:"tags"` + Type string `json:"type"` Limit int64 `json:"limit"` } diff --git a/pkg/services/cleanup/cleanup.go b/pkg/services/cleanup/cleanup.go index ffaa75de9cc..6e5e7684100 100644 --- a/pkg/services/cleanup/cleanup.go +++ b/pkg/services/cleanup/cleanup.go @@ -39,12 +39,13 @@ func (service *CleanUpService) Run(ctx context.Context) error { func (service *CleanUpService) start(ctx context.Context) error { service.cleanUpTmpFiles() - ticker := time.NewTicker(time.Hour * 1) + ticker := time.NewTicker(time.Minute * 10) for { select { case <-ticker.C: service.cleanUpTmpFiles() service.deleteExpiredSnapshots() + service.deleteExpiredDashboardVersions() case <-ctx.Done(): return ctx.Err() } @@ -83,3 +84,7 @@ func (service *CleanUpService) cleanUpTmpFiles() { func (service *CleanUpService) deleteExpiredSnapshots() { bus.Dispatch(&m.DeleteExpiredSnapshotsCommand{}) } + +func (service *CleanUpService) deleteExpiredDashboardVersions() { + bus.Dispatch(&m.DeleteExpiredVersionsCommand{}) +} diff --git a/pkg/services/provisioning/datasources/datasources.go b/pkg/services/provisioning/datasources/datasources.go new file mode 100644 index 00000000000..325dbbbd757 --- /dev/null +++ b/pkg/services/provisioning/datasources/datasources.go @@ -0,0 +1,148 @@ +package datasources + +import ( + "errors" + "io/ioutil" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/bus" + + "github.com/grafana/grafana/pkg/log" + + "github.com/grafana/grafana/pkg/models" + yaml "gopkg.in/yaml.v2" +) + +var ( + ErrInvalidConfigToManyDefault = errors.New("datasource.yaml config is invalid. Only one datasource can be marked as default") +) + +func Provision(configDirectory string) error { + dc := newDatasourceProvisioner(log.New("provisioning.datasources")) + return dc.applyChanges(configDirectory) +} + +type DatasourceProvisioner struct { + log log.Logger + cfgProvider configReader +} + +func newDatasourceProvisioner(log log.Logger) DatasourceProvisioner { + return DatasourceProvisioner{ + log: log, + cfgProvider: configReader{}, + } +} + +func (dc *DatasourceProvisioner) apply(cfg *DatasourcesAsConfig) error { + if err := dc.deleteDatasources(cfg.DeleteDatasources); err != nil { + return err + } + + for _, ds := range cfg.Datasources { + cmd := &models.GetDataSourceByNameQuery{OrgId: ds.OrgId, Name: ds.Name} + err := bus.Dispatch(cmd) + if err != nil && err != models.ErrDataSourceNotFound { + return err + } + + if err == models.ErrDataSourceNotFound { + dc.log.Info("inserting datasource from configuration ", "name", ds.Name) + insertCmd := createInsertCommand(ds) + if err := bus.Dispatch(insertCmd); err != nil { + return err + } + } else { + dc.log.Debug("updating datasource from configuration", "name", ds.Name) + updateCmd := createUpdateCommand(ds, cmd.Result.Id) + if err := bus.Dispatch(updateCmd); err != nil { + return err + } + } + } + + return nil +} + +func (dc *DatasourceProvisioner) applyChanges(configPath string) error { + configs, err := dc.cfgProvider.readConfig(configPath) + if err != nil { + return err + } + + for _, cfg := range configs { + if err := dc.apply(cfg); err != nil { + return err + } + } + + return nil +} + +func (dc *DatasourceProvisioner) deleteDatasources(dsToDelete []*DeleteDatasourceConfig) error { + for _, ds := range dsToDelete { + cmd := &models.DeleteDataSourceByNameCommand{OrgId: ds.OrgId, Name: ds.Name} + if err := bus.Dispatch(cmd); err != nil { + return err + } + + if cmd.DeletedDatasourcesCount > 0 { + dc.log.Info("deleted datasource based on configuration", "name", ds.Name) + } + } + + return nil +} + +type configReader struct{} + +func (configReader) readConfig(path string) ([]*DatasourcesAsConfig, error) { + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + + var datasources []*DatasourcesAsConfig + for _, file := range files { + if strings.HasSuffix(file.Name(), ".yaml") || strings.HasSuffix(file.Name(), ".yml") { + filename, _ := filepath.Abs(filepath.Join(path, file.Name())) + yamlFile, err := ioutil.ReadFile(filename) + + if err != nil { + return nil, err + } + var datasource *DatasourcesAsConfig + err = yaml.Unmarshal(yamlFile, &datasource) + if err != nil { + return nil, err + } + + datasources = append(datasources, datasource) + } + } + + defaultCount := 0 + for _, cfg := range datasources { + for _, ds := range cfg.Datasources { + if ds.OrgId == 0 { + ds.OrgId = 1 + } + + if ds.IsDefault { + defaultCount++ + if defaultCount > 1 { + return nil, ErrInvalidConfigToManyDefault + } + } + } + + for _, ds := range cfg.DeleteDatasources { + if ds.OrgId == 0 { + ds.OrgId = 1 + } + } + } + + return datasources, nil +} diff --git a/pkg/services/provisioning/datasources/datasources_test.go b/pkg/services/provisioning/datasources/datasources_test.go new file mode 100644 index 00000000000..f3252c28d9d --- /dev/null +++ b/pkg/services/provisioning/datasources/datasources_test.go @@ -0,0 +1,202 @@ +package datasources + +import ( + "testing" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" + + . "github.com/smartystreets/goconvey/convey" +) + +var ( + logger log.Logger = log.New("fake.logger") + oneDatasourcesConfig string = "" + twoDatasourcesConfig string = "./test-configs/two-datasources" + twoDatasourcesConfigPurgeOthers string = "./test-configs/insert-two-delete-two" + doubleDatasourcesConfig string = "./test-configs/double-default" + allProperties string = "./test-configs/all-properties" + brokenYaml string = "./test-configs/broken-yaml" + + fakeRepo *fakeRepository +) + +func TestDatasourceAsConfig(t *testing.T) { + Convey("Testing datasource as configuration", t, func() { + fakeRepo = &fakeRepository{} + bus.ClearBusHandlers() + bus.AddHandler("test", mockDelete) + bus.AddHandler("test", mockInsert) + bus.AddHandler("test", mockUpdate) + bus.AddHandler("test", mockGet) + bus.AddHandler("test", mockGetAll) + + Convey("One configured datasource", func() { + Convey("no datasource in database", func() { + dc := newDatasourceProvisioner(logger) + err := dc.applyChanges(twoDatasourcesConfig) + if err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + So(len(fakeRepo.deleted), ShouldEqual, 0) + So(len(fakeRepo.inserted), ShouldEqual, 2) + So(len(fakeRepo.updated), ShouldEqual, 0) + }) + + Convey("One datasource in database with same name", func() { + fakeRepo.loadAll = []*models.DataSource{ + {Name: "Graphite", OrgId: 1, Id: 1}, + } + + Convey("should update one datasource", func() { + dc := newDatasourceProvisioner(logger) + err := dc.applyChanges(twoDatasourcesConfig) + if err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + So(len(fakeRepo.deleted), ShouldEqual, 0) + So(len(fakeRepo.inserted), ShouldEqual, 1) + So(len(fakeRepo.updated), ShouldEqual, 1) + }) + }) + + Convey("Two datasources with is_default", func() { + dc := newDatasourceProvisioner(logger) + err := dc.applyChanges(doubleDatasourcesConfig) + Convey("should raise error", func() { + So(err, ShouldEqual, ErrInvalidConfigToManyDefault) + }) + }) + }) + + Convey("Two configured datasource and purge others ", func() { + Convey("two other datasources in database", func() { + fakeRepo.loadAll = []*models.DataSource{ + {Name: "old-graphite", OrgId: 1, Id: 1}, + {Name: "old-graphite2", OrgId: 1, Id: 2}, + } + + Convey("should have two new datasources", func() { + dc := newDatasourceProvisioner(logger) + err := dc.applyChanges(twoDatasourcesConfigPurgeOthers) + if err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + So(len(fakeRepo.deleted), ShouldEqual, 2) + So(len(fakeRepo.inserted), ShouldEqual, 2) + So(len(fakeRepo.updated), ShouldEqual, 0) + }) + }) + }) + + Convey("Two configured datasource and purge others = false", func() { + Convey("two other datasources in database", func() { + fakeRepo.loadAll = []*models.DataSource{ + {Name: "Graphite", OrgId: 1, Id: 1}, + {Name: "old-graphite2", OrgId: 1, Id: 2}, + } + + Convey("should have two new datasources", func() { + dc := newDatasourceProvisioner(logger) + err := dc.applyChanges(twoDatasourcesConfig) + if err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + So(len(fakeRepo.deleted), ShouldEqual, 0) + So(len(fakeRepo.inserted), ShouldEqual, 1) + So(len(fakeRepo.updated), ShouldEqual, 1) + }) + }) + }) + + Convey("broken yaml should return error", func() { + _, err := configReader{}.readConfig(brokenYaml) + So(err, ShouldNotBeNil) + }) + + Convey("can read all properties", func() { + cfgProvifer := configReader{} + cfg, err := cfgProvifer.readConfig(allProperties) + if err != nil { + t.Fatalf("readConfig return an error %v", err) + } + + So(len(cfg), ShouldEqual, 2) + + dsCfg := cfg[0] + ds := dsCfg.Datasources[0] + + So(ds.Name, ShouldEqual, "name") + So(ds.Type, ShouldEqual, "type") + So(ds.Access, ShouldEqual, models.DS_ACCESS_PROXY) + So(ds.OrgId, ShouldEqual, 2) + So(ds.Url, ShouldEqual, "url") + So(ds.User, ShouldEqual, "user") + So(ds.Password, ShouldEqual, "password") + So(ds.Database, ShouldEqual, "database") + So(ds.BasicAuth, ShouldBeTrue) + So(ds.BasicAuthUser, ShouldEqual, "basic_auth_user") + So(ds.BasicAuthPassword, ShouldEqual, "basic_auth_password") + So(ds.WithCredentials, ShouldBeTrue) + So(ds.IsDefault, ShouldBeTrue) + So(ds.Editable, ShouldBeTrue) + + So(len(ds.JsonData), ShouldBeGreaterThan, 2) + So(ds.JsonData["graphiteVersion"], ShouldEqual, "1.1") + So(ds.JsonData["tlsAuth"], ShouldEqual, true) + So(ds.JsonData["tlsAuthWithCACert"], ShouldEqual, true) + + So(len(ds.SecureJsonData), ShouldBeGreaterThan, 2) + So(ds.SecureJsonData["tlsCACert"], ShouldEqual, "MjNOcW9RdkbUDHZmpco2HCYzVq9dE+i6Yi+gmUJotq5CDA==") + So(ds.SecureJsonData["tlsClientCert"], ShouldEqual, "ckN0dGlyMXN503YNfjTcf9CV+GGQneN+xmAclQ==") + So(ds.SecureJsonData["tlsClientKey"], ShouldEqual, "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A==") + + dstwo := cfg[1].Datasources[0] + So(dstwo.Name, ShouldEqual, "name2") + }) + }) +} + +type fakeRepository struct { + inserted []*models.AddDataSourceCommand + deleted []*models.DeleteDataSourceByNameCommand + updated []*models.UpdateDataSourceCommand + + loadAll []*models.DataSource +} + +func mockDelete(cmd *models.DeleteDataSourceByNameCommand) error { + fakeRepo.deleted = append(fakeRepo.deleted, cmd) + return nil +} + +func mockUpdate(cmd *models.UpdateDataSourceCommand) error { + fakeRepo.updated = append(fakeRepo.updated, cmd) + return nil +} + +func mockInsert(cmd *models.AddDataSourceCommand) error { + fakeRepo.inserted = append(fakeRepo.inserted, cmd) + return nil +} + +func mockGetAll(cmd *models.GetAllDataSourcesQuery) error { + cmd.Result = fakeRepo.loadAll + return nil +} + +func mockGet(cmd *models.GetDataSourceByNameQuery) error { + for _, v := range fakeRepo.loadAll { + if cmd.Name == v.Name && cmd.OrgId == v.OrgId { + cmd.Result = v + return nil + } + } + + return models.ErrDataSourceNotFound +} diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml b/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml new file mode 100644 index 00000000000..af0d3009a4c --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml @@ -0,0 +1,23 @@ +datasources: + - name: name + type: type + access: proxy + org_id: 2 + url: url + password: password + user: user + database: database + basic_auth: true + basic_auth_user: basic_auth_user + basic_auth_password: basic_auth_password + with_credentials: true + is_default: true + json_data: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + secure_json_data: + tlsCACert: "MjNOcW9RdkbUDHZmpco2HCYzVq9dE+i6Yi+gmUJotq5CDA==" + tlsClientCert: "ckN0dGlyMXN503YNfjTcf9CV+GGQneN+xmAclQ==" + tlsClientKey: "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A==" + editable: true diff --git a/public/sass/components/_dash_list.scss b/pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt similarity index 100% rename from public/sass/components/_dash_list.scss rename to pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml b/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml new file mode 100644 index 00000000000..43c41ee9b3b --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml @@ -0,0 +1,7 @@ +purge_other_datasources: true +datasources: + - name: name2 + type: type2 + access: proxy + org_id: 2 + url: url2 diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml b/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml new file mode 100644 index 00000000000..9050f543cef --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml @@ -0,0 +1,6 @@ +#sfxzgnsxzcvnbzcvn +cvbn +cvbn +c +vbn +cvbncvbn \ No newline at end of file diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml b/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml new file mode 100644 index 00000000000..f202d617fc9 --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml @@ -0,0 +1,7 @@ +datasources: + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + is_default: true + diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml b/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml new file mode 100644 index 00000000000..f202d617fc9 --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml @@ -0,0 +1,7 @@ +datasources: + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + is_default: true + diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml b/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml new file mode 100644 index 00000000000..a1a58dc6621 --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml @@ -0,0 +1,7 @@ +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://localhost:9090 +delete_datasources: + - name: old-graphite diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml b/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml new file mode 100644 index 00000000000..d9d151227c9 --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml @@ -0,0 +1,7 @@ +datasources: + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 +delete_datasources: + - name: old-graphite3 diff --git a/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml b/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml new file mode 100644 index 00000000000..d555db3365b --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml @@ -0,0 +1,9 @@ +datasources: + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + - name: Prometheus + type: prometheus + access: proxy + url: http://localhost:9090 diff --git a/pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/services/provisioning/datasources/types.go b/pkg/services/provisioning/datasources/types.go new file mode 100644 index 00000000000..ee2175d6a90 --- /dev/null +++ b/pkg/services/provisioning/datasources/types.go @@ -0,0 +1,92 @@ +package datasources + +import "github.com/grafana/grafana/pkg/models" +import "github.com/grafana/grafana/pkg/components/simplejson" + +type DatasourcesAsConfig struct { + Datasources []*DataSourceFromConfig `json:"datasources" yaml:"datasources"` + DeleteDatasources []*DeleteDatasourceConfig `json:"delete_datasources" yaml:"delete_datasources"` +} + +type DeleteDatasourceConfig struct { + OrgId int64 `json:"org_id" yaml:"org_id"` + Name string `json:"name" yaml:"name"` +} + +type DataSourceFromConfig struct { + OrgId int64 `json:"org_id" yaml:"org_id"` + Version int `json:"version" yaml:"version"` + + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + Access string `json:"access" yaml:"access"` + Url string `json:"url" yaml:"url"` + Password string `json:"password" yaml:"password"` + User string `json:"user" yaml:"user"` + Database string `json:"database" yaml:"database"` + BasicAuth bool `json:"basic_auth" yaml:"basic_auth"` + BasicAuthUser string `json:"basic_auth_user" yaml:"basic_auth_user"` + BasicAuthPassword string `json:"basic_auth_password" yaml:"basic_auth_password"` + WithCredentials bool `json:"with_credentials" yaml:"with_credentials"` + IsDefault bool `json:"is_default" yaml:"is_default"` + JsonData map[string]interface{} `json:"json_data" yaml:"json_data"` + SecureJsonData map[string]string `json:"secure_json_data" yaml:"secure_json_data"` + Editable bool `json:"editable" yaml:"editable"` +} + +func createInsertCommand(ds *DataSourceFromConfig) *models.AddDataSourceCommand { + jsonData := simplejson.New() + if len(ds.JsonData) > 0 { + for k, v := range ds.JsonData { + jsonData.Set(k, v) + } + } + + return &models.AddDataSourceCommand{ + OrgId: ds.OrgId, + Name: ds.Name, + Type: ds.Type, + Access: models.DsAccess(ds.Access), + Url: ds.Url, + Password: ds.Password, + User: ds.User, + Database: ds.Database, + BasicAuth: ds.BasicAuth, + BasicAuthUser: ds.BasicAuthUser, + BasicAuthPassword: ds.BasicAuthPassword, + WithCredentials: ds.WithCredentials, + IsDefault: ds.IsDefault, + JsonData: jsonData, + SecureJsonData: ds.SecureJsonData, + ReadOnly: !ds.Editable, + } +} + +func createUpdateCommand(ds *DataSourceFromConfig, id int64) *models.UpdateDataSourceCommand { + jsonData := simplejson.New() + if len(ds.JsonData) > 0 { + for k, v := range ds.JsonData { + jsonData.Set(k, v) + } + } + + return &models.UpdateDataSourceCommand{ + Id: id, + OrgId: ds.OrgId, + Name: ds.Name, + Type: ds.Type, + Access: models.DsAccess(ds.Access), + Url: ds.Url, + Password: ds.Password, + User: ds.User, + Database: ds.Database, + BasicAuth: ds.BasicAuth, + BasicAuthUser: ds.BasicAuthUser, + BasicAuthPassword: ds.BasicAuthPassword, + WithCredentials: ds.WithCredentials, + IsDefault: ds.IsDefault, + JsonData: jsonData, + SecureJsonData: ds.SecureJsonData, + ReadOnly: !ds.Editable, + } +} diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go new file mode 100644 index 00000000000..1bea60f03e4 --- /dev/null +++ b/pkg/services/provisioning/provisioning.go @@ -0,0 +1,14 @@ +package provisioning + +import ( + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/provisioning/datasources" +) + +var ( + logger log.Logger = log.New("services.provisioning") +) + +func StartUp(datasourcePath string) error { + return datasources.Provision(datasourcePath) +} diff --git a/pkg/services/search/handlers.go b/pkg/services/search/handlers.go index 9197c8f77e6..247585402ef 100644 --- a/pkg/services/search/handlers.go +++ b/pkg/services/search/handlers.go @@ -18,7 +18,7 @@ func searchHandler(query *Query) error { IsStarred: query.IsStarred, DashboardIds: query.DashboardIds, Type: query.Type, - FolderId: query.FolderId, + FolderIds: query.FolderIds, Tags: query.Tags, Limit: query.Limit, } diff --git a/pkg/services/search/models.go b/pkg/services/search/models.go index a95a5dc3d4a..dbd4f21a71e 100644 --- a/pkg/services/search/models.go +++ b/pkg/services/search/models.go @@ -48,7 +48,7 @@ type Query struct { IsStarred bool Type string DashboardIds []int64 - FolderId int64 + FolderIds []int64 Result HitList } @@ -60,9 +60,10 @@ type FindPersistedDashboardsQuery struct { IsStarred bool DashboardIds []int64 Type string - FolderId int64 + FolderIds []int64 Tags []string Limit int + IsBrowse bool Result HitList } diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 33a4cae53c2..73be7d774fd 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -94,7 +94,12 @@ func HandleAlertsQuery(query *m.GetAlertsQuery) error { if i > 0 { sql.WriteString(" OR ") } - sql.WriteString("state = ? ") + if strings.HasPrefix(v, "not_") { + sql.WriteString("state <> ? ") + v = strings.TrimPrefix(v, "not_") + } else { + sql.WriteString("state = ? ") + } params = append(params, v) } sql.WriteString(")") diff --git a/pkg/services/sqlstore/annotation.go b/pkg/services/sqlstore/annotation.go index d97db10f630..effffb8bab4 100644 --- a/pkg/services/sqlstore/annotation.go +++ b/pkg/services/sqlstore/annotation.go @@ -158,6 +158,10 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I params = append(params, query.From, query.To) } + if query.Type == "alert" { + sql.WriteString(` AND annotation.alert_id > 0`) + } + if len(query.Tags) > 0 { keyValueFilters := []string{} diff --git a/pkg/services/sqlstore/annotation_test.go b/pkg/services/sqlstore/annotation_test.go index 3f7415a952b..2afd4479b66 100644 --- a/pkg/services/sqlstore/annotation_test.go +++ b/pkg/services/sqlstore/annotation_test.go @@ -37,16 +37,19 @@ func TestAnnotations(t *testing.T) { repo := SqlAnnotationRepo{} Convey("Can save annotation", func() { - err := repo.Save(&annotations.Item{ + annotation := &annotations.Item{ OrgId: 1, UserId: 1, DashboardId: 1, Text: "hello", + Type: "alert", Epoch: 10, Tags: []string{"outage", "error", "type:outage", "server:server-1"}, - }) + } + err := repo.Save(annotation) So(err, ShouldBeNil) + So(annotation.Id, ShouldBeGreaterThan, 0) Convey("Can query for annotation", func() { items, err := repo.Find(&annotations.ItemQuery{ @@ -89,6 +92,19 @@ func TestAnnotations(t *testing.T) { So(items, ShouldHaveLength, 0) }) + Convey("Should not find one when type filter does not match", func() { + items, err := repo.Find(&annotations.ItemQuery{ + OrgId: 1, + DashboardId: 1, + From: 1, + To: 15, + Type: "alert", + }) + + So(err, ShouldBeNil) + So(items, ShouldHaveLength, 0) + }) + Convey("Should find one when all tag filters does match", func() { items, err := repo.Find(&annotations.ItemQuery{ OrgId: 1, diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index fcc49799def..8ab0d50b85f 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -1,9 +1,6 @@ package sqlstore import ( - "bytes" - "fmt" - "strings" "time" "github.com/grafana/grafana/pkg/bus" @@ -189,77 +186,35 @@ type DashboardSearchProjection struct { } func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSearchProjection, error) { - var sql bytes.Buffer - params := make([]interface{}, 0) limit := query.Limit if limit == 0 { limit = 1000 } - sql.WriteString(` - SELECT - dashboard.id, - dashboard.title, - dashboard.slug, - dashboard_tag.term, - dashboard.is_folder, - dashboard.folder_id, - folder.slug as folder_slug, - folder.title as folder_title - FROM `) + sb := NewSearchBuilder(query.SignedInUser, limit). + WithTags(query.Tags). + WithDashboardIdsIn(query.DashboardIds) - // add tags filter - if len(query.Tags) > 0 { - sql.WriteString( - `( - SELECT - dashboard.id FROM dashboard - LEFT OUTER JOIN dashboard_tag ON dashboard_tag.dashboard_id = dashboard.id - `) - if query.IsStarred { - sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id") - } - - sql.WriteString(` WHERE dashboard_tag.term IN (?` + strings.Repeat(",?", len(query.Tags)-1) + `) AND `) - for _, tag := range query.Tags { - params = append(params, tag) - } - params = createSearchWhereClause(query, &sql, params) - fmt.Printf("params2 %v", params) - - // this ends the inner select (tag filtered part) - sql.WriteString(` - GROUP BY dashboard.id HAVING COUNT(dashboard.id) >= ? - LIMIT ?) as ids - INNER JOIN dashboard on ids.id = dashboard.id - `) - - params = append(params, len(query.Tags)) - params = append(params, limit) - } else { - sql.WriteString(`( SELECT dashboard.id FROM dashboard `) - if query.IsStarred { - sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id") - } - sql.WriteString(` WHERE `) - params = createSearchWhereClause(query, &sql, params) - - sql.WriteString(` - LIMIT ?) as ids - INNER JOIN dashboard on ids.id = dashboard.id - `) - params = append(params, limit) + if query.IsStarred { + sb.IsStarred() } - sql.WriteString(` - LEFT OUTER JOIN dashboard folder on folder.id = dashboard.folder_id - LEFT OUTER JOIN dashboard_tag on dashboard.id = dashboard_tag.dashboard_id`) + if len(query.Title) > 0 { + sb.WithTitle(query.Title) + } - sql.WriteString(fmt.Sprintf(" ORDER BY dashboard.title ASC LIMIT 5000")) + if len(query.Type) > 0 { + sb.WithType(query.Type) + } + + if len(query.FolderIds) > 0 { + sb.WithFolderIds(query.FolderIds) + } var res []DashboardSearchProjection - err := x.Sql(sql.String(), params...).Find(&res) + sql, params := sb.ToSql() + err := x.Sql(sql, params...).Find(&res) if err != nil { return nil, err } @@ -267,61 +222,6 @@ func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSear return res, nil } -func createSearchWhereClause(query *search.FindPersistedDashboardsQuery, sql *bytes.Buffer, params []interface{}) []interface{} { - sql.WriteString(` dashboard.org_id=?`) - params = append(params, query.SignedInUser.OrgId) - - if query.IsStarred { - sql.WriteString(` AND star.user_id=?`) - params = append(params, query.SignedInUser.UserId) - } - - if len(query.DashboardIds) > 0 { - sql.WriteString(` AND dashboard.id IN (?` + strings.Repeat(",?", len(query.DashboardIds)-1) + `)`) - for _, dashboardId := range query.DashboardIds { - params = append(params, dashboardId) - } - } - - if query.SignedInUser.OrgRole != m.ROLE_ADMIN { - allowedDashboardsSubQuery := ` AND (dashboard.has_acl = 0 OR dashboard.id in ( - SELECT distinct d.id AS DashboardId - FROM dashboard AS d - LEFT JOIN dashboard_acl as da on d.folder_id = da.dashboard_id or d.id = da.dashboard_id - LEFT JOIN user_group_member as ugm on ugm.user_group_id = da.user_group_id - LEFT JOIN org_user ou on ou.role = da.role - WHERE - d.has_acl = 1 and - (da.user_id = ? or ugm.user_id = ? or ou.id is not null) - and d.org_id = ? - ) - )` - - sql.WriteString(allowedDashboardsSubQuery) - params = append(params, query.SignedInUser.UserId, query.SignedInUser.UserId, query.SignedInUser.OrgId) - } - - if len(query.Title) > 0 { - sql.WriteString(" AND dashboard.title " + dialect.LikeStr() + " ?") - params = append(params, "%"+query.Title+"%") - } - - if len(query.Type) > 0 && query.Type == "dash-folder" { - sql.WriteString(" AND dashboard.is_folder = 1") - } - - if len(query.Type) > 0 && query.Type == "dash-db" { - sql.WriteString(" AND dashboard.is_folder = 0") - } - - if query.FolderId > 0 { - sql.WriteString(" AND dashboard.folder_id = ?") - params = append(params, query.FolderId) - } - - return params -} - func SearchDashboards(query *search.FindPersistedDashboardsQuery) error { res, err := findDashboards(query) if err != nil { @@ -440,7 +340,7 @@ func GetDashboards(query *m.GetDashboardsQuery) error { func GetDashboardsByPluginId(query *m.GetDashboardsByPluginIdQuery) error { var dashboards = make([]*m.Dashboard, 0) - err := x.Where("org_id=? AND plugin_id=?", query.OrgId, query.PluginId).Find(&dashboards) + err := x.Where("org_id=? AND plugin_id=? AND is_folder=0", query.OrgId, query.PluginId).Find(&dashboards) query.Result = dashboards if err != nil { diff --git a/pkg/services/sqlstore/dashboard_test.go b/pkg/services/sqlstore/dashboard_test.go index e0473b0e38f..a552bd0546a 100644 --- a/pkg/services/sqlstore/dashboard_test.go +++ b/pkg/services/sqlstore/dashboard_test.go @@ -114,7 +114,7 @@ func TestDashboardDataAccess(t *testing.T) { Convey("Should be able to search for a dashboard folder's children", func() { query := search.FindPersistedDashboardsQuery{ OrgId: 1, - FolderId: savedFolder.Id, + FolderIds: []int64{savedFolder.Id}, SignedInUser: &m.SignedInUser{OrgId: 1}, } @@ -218,7 +218,7 @@ func TestDashboardDataAccess(t *testing.T) { query := search.FindPersistedDashboardsQuery{ OrgId: 1, - FolderId: savedFolder.Id, + FolderIds: []int64{savedFolder.Id}, SignedInUser: &m.SignedInUser{}, } @@ -381,6 +381,20 @@ func TestDashboardDataAccess(t *testing.T) { childDash2 := insertTestDashboard("child dash 2", 1, folder2.Id, false, "prod") currentUser := createUser("viewer", "Viewer", false) + var rootFolderId int64 = 0 + + Convey("and one folder is expanded, the other collapsed", func() { + Convey("should return dashboards in root and expanded folder", func() { + query := &search.FindPersistedDashboardsQuery{FolderIds: []int64{rootFolderId, folder1.Id}, SignedInUser: &m.SignedInUser{UserId: currentUser.Id, OrgId: 1}, OrgId: 1} + err := SearchDashboards(query) + So(err, ShouldBeNil) + So(len(query.Result), ShouldEqual, 4) + So(query.Result[0].Id, ShouldEqual, folder1.Id) + So(query.Result[1].Id, ShouldEqual, folder2.Id) + So(query.Result[2].Id, ShouldEqual, childDash1.Id) + So(query.Result[3].Id, ShouldEqual, dashInRoot.Id) + }) + }) Convey("and acl is set for one dashboard folder", func() { var otherUser int64 = 999 @@ -444,6 +458,25 @@ func TestDashboardDataAccess(t *testing.T) { }) }) }) + + Convey("Given a plugin with imported dashboards", func() { + pluginId := "test-app" + + appFolder := insertTestDashboardForPlugin("app-test", 1, 0, true, pluginId) + insertTestDashboardForPlugin("app-dash1", 1, appFolder.Id, false, pluginId) + insertTestDashboardForPlugin("app-dash2", 1, appFolder.Id, false, pluginId) + + Convey("Should return imported dashboard", func() { + query := m.GetDashboardsByPluginIdQuery{ + PluginId: pluginId, + OrgId: 1, + } + + err := GetDashboardsByPluginId(&query) + So(err, ShouldBeNil) + So(len(query.Result), ShouldEqual, 2) + }) + }) }) } @@ -465,6 +498,24 @@ func insertTestDashboard(title string, orgId int64, folderId int64, isFolder boo return cmd.Result } +func insertTestDashboardForPlugin(title string, orgId int64, folderId int64, isFolder bool, pluginId string) *m.Dashboard { + cmd := m.SaveDashboardCommand{ + OrgId: orgId, + FolderId: folderId, + IsFolder: isFolder, + Dashboard: simplejson.NewFromAny(map[string]interface{}{ + "id": nil, + "title": title, + }), + PluginId: pluginId, + } + + err := SaveDashboard(&cmd) + So(err, ShouldBeNil) + + return cmd.Result +} + func createUser(name string, role string, isAdmin bool) m.User { setting.AutoAssignOrg = true setting.AutoAssignOrgRole = role diff --git a/pkg/services/sqlstore/dashboard_version.go b/pkg/services/sqlstore/dashboard_version.go index c5ef7d374f0..49c35397094 100644 --- a/pkg/services/sqlstore/dashboard_version.go +++ b/pkg/services/sqlstore/dashboard_version.go @@ -1,13 +1,17 @@ package sqlstore import ( + "strings" + "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) func init() { bus.AddHandler("sql", GetDashboardVersion) bus.AddHandler("sql", GetDashboardVersions) + bus.AddHandler("sql", DeleteExpiredVersions) } // GetDashboardVersion gets the dashboard version for the given dashboard ID and version number. @@ -62,3 +66,73 @@ func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error { } return nil } + +func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error { + return inTransaction(func(sess *DBSession) error { + expiredCount := int64(0) + versions := []DashboardVersionExp{} + versionsToKeep := setting.DashboardVersionsToKeep + + if versionsToKeep < 1 { + versionsToKeep = 1 + } + + err := sess.Table("dashboard_version"). + Select("dashboard_version.id, dashboard_version.version, dashboard_version.dashboard_id"). + Where(`dashboard_id IN ( + SELECT dashboard_id FROM dashboard_version + GROUP BY dashboard_id HAVING COUNT(dashboard_version.id) > ? + )`, versionsToKeep). + Desc("dashboard_version.dashboard_id", "dashboard_version.version"). + Find(&versions) + + if err != nil { + return err + } + + // Keep last versionsToKeep versions and delete other + versionIdsToDelete := getVersionIDsToDelete(versions, versionsToKeep) + if len(versionIdsToDelete) > 0 { + deleteExpiredSql := `DELETE FROM dashboard_version WHERE id IN (?` + strings.Repeat(",?", len(versionIdsToDelete)-1) + `)` + expiredResponse, err := sess.Exec(deleteExpiredSql, versionIdsToDelete...) + if err != nil { + return err + } + expiredCount, _ = expiredResponse.RowsAffected() + sqlog.Debug("Deleted old/expired dashboard versions", "expired", expiredCount) + } + + return nil + }) +} + +// Short version of DashboardVersion for getting expired versions +type DashboardVersionExp struct { + Id int64 `json:"id"` + DashboardId int64 `json:"dashboardId"` + Version int `json:"version"` +} + +func getVersionIDsToDelete(versions []DashboardVersionExp, versionsToKeep int) []interface{} { + versionIds := make([]interface{}, 0) + + if len(versions) == 0 { + return versionIds + } + + currentDashboard := versions[0].DashboardId + count := 0 + for _, v := range versions { + if v.DashboardId == currentDashboard { + count++ + } else { + count = 1 + currentDashboard = v.DashboardId + } + if count > versionsToKeep { + versionIds = append(versionIds, v.Id) + } + } + + return versionIds +} diff --git a/pkg/services/sqlstore/dashboard_version_test.go b/pkg/services/sqlstore/dashboard_version_test.go index 919efcd0004..6ed37cd6904 100644 --- a/pkg/services/sqlstore/dashboard_version_test.go +++ b/pkg/services/sqlstore/dashboard_version_test.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) func updateTestDashboard(dashboard *m.Dashboard, data map[string]interface{}) { @@ -101,3 +102,44 @@ func TestGetDashboardVersions(t *testing.T) { }) }) } + +func TestDeleteExpiredVersions(t *testing.T) { + Convey("Testing dashboard versions clean up", t, func() { + InitTestDB(t) + versionsToKeep := 5 + versionsToWrite := 10 + setting.DashboardVersionsToKeep = versionsToKeep + + savedDash := insertTestDashboard("test dash 53", 1, 0, false, "diff-all") + for i := 0; i < versionsToWrite-1; i++ { + updateTestDashboard(savedDash, map[string]interface{}{ + "tags": "different-tag", + }) + } + + Convey("Clean up old dashboard versions", func() { + err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) + So(err, ShouldBeNil) + + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1} + GetDashboardVersions(&query) + + So(len(query.Result), ShouldEqual, versionsToKeep) + // Ensure latest versions were kept + So(query.Result[versionsToKeep-1].Version, ShouldEqual, versionsToWrite-versionsToKeep+1) + So(query.Result[0].Version, ShouldEqual, versionsToWrite) + }) + + Convey("Don't delete anything if there're no expired versions", func() { + setting.DashboardVersionsToKeep = versionsToWrite + + err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) + So(err, ShouldBeNil) + + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1} + GetDashboardVersions(&query) + + So(len(query.Result), ShouldEqual, versionsToWrite) + }) + }) +} diff --git a/pkg/services/sqlstore/datasource.go b/pkg/services/sqlstore/datasource.go index 7069990c476..e9b400a1772 100644 --- a/pkg/services/sqlstore/datasource.go +++ b/pkg/services/sqlstore/datasource.go @@ -13,6 +13,7 @@ import ( func init() { bus.AddHandler("sql", GetDataSources) + bus.AddHandler("sql", GetAllDataSources) bus.AddHandler("sql", AddDataSource) bus.AddHandler("sql", DeleteDataSourceById) bus.AddHandler("sql", DeleteDataSourceByName) @@ -54,10 +55,19 @@ func GetDataSources(query *m.GetDataSourcesQuery) error { return sess.Find(&query.Result) } +func GetAllDataSources(query *m.GetAllDataSourcesQuery) error { + sess := x.Limit(1000, 0).Asc("name") + + query.Result = make([]*m.DataSource, 0) + return sess.Find(&query.Result) +} + func DeleteDataSourceById(cmd *m.DeleteDataSourceByIdCommand) error { return inTransaction(func(sess *DBSession) error { var rawSql = "DELETE FROM data_source WHERE id=? and org_id=?" - _, err := sess.Exec(rawSql, cmd.Id, cmd.OrgId) + result, err := sess.Exec(rawSql, cmd.Id, cmd.OrgId) + affected, _ := result.RowsAffected() + cmd.DeletedDatasourcesCount = affected return err }) } @@ -65,7 +75,9 @@ func DeleteDataSourceById(cmd *m.DeleteDataSourceByIdCommand) error { func DeleteDataSourceByName(cmd *m.DeleteDataSourceByNameCommand) error { return inTransaction(func(sess *DBSession) error { var rawSql = "DELETE FROM data_source WHERE name=? and org_id=?" - _, err := sess.Exec(rawSql, cmd.Name, cmd.OrgId) + result, err := sess.Exec(rawSql, cmd.Name, cmd.OrgId) + affected, _ := result.RowsAffected() + cmd.DeletedDatasourcesCount = affected return err }) } @@ -98,6 +110,7 @@ func AddDataSource(cmd *m.AddDataSourceCommand) error { Created: time.Now(), Updated: time.Now(), Version: 1, + ReadOnly: cmd.ReadOnly, } if _, err := sess.Insert(ds); err != nil { @@ -143,12 +156,14 @@ func UpdateDataSource(cmd *m.UpdateDataSourceCommand) error { JsonData: cmd.JsonData, SecureJsonData: securejsondata.GetEncryptedJsonData(cmd.SecureJsonData), Updated: time.Now(), + ReadOnly: cmd.ReadOnly, Version: cmd.Version + 1, } sess.UseBool("is_default") sess.UseBool("basic_auth") sess.UseBool("with_credentials") + sess.UseBool("read_only") var updateSession *xorm.Session if cmd.Version != 0 { diff --git a/pkg/services/sqlstore/datasource_test.go b/pkg/services/sqlstore/datasource_test.go index de16b17f960..e6f0114ab4d 100644 --- a/pkg/services/sqlstore/datasource_test.go +++ b/pkg/services/sqlstore/datasource_test.go @@ -47,6 +47,7 @@ func TestDataAccess(t *testing.T) { Access: m.DS_ACCESS_DIRECT, Url: "http://test", Database: "site", + ReadOnly: true, }) So(err, ShouldBeNil) @@ -61,6 +62,7 @@ func TestDataAccess(t *testing.T) { So(ds.OrgId, ShouldEqual, 10) So(ds.Database, ShouldEqual, "site") + So(ds.ReadOnly, ShouldBeTrue) }) Convey("Given a datasource", func() { diff --git a/pkg/services/sqlstore/migrations/dashboard_mig.go b/pkg/services/sqlstore/migrations/dashboard_mig.go index ed8d2c73a5d..4f1602be931 100644 --- a/pkg/services/sqlstore/migrations/dashboard_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_mig.go @@ -139,7 +139,7 @@ func addDashboardMigration(mg *Migrator) { // add column to store folder_id for dashboard folder structure mg.AddMigration("Add column folder_id in dashboard", NewAddColumnMigration(dashboardV2, &Column{ - Name: "folder_id", Type: DB_BigInt, Nullable: true, + Name: "folder_id", Type: DB_BigInt, Nullable: false, Default: "0", })) mg.AddMigration("Add column isFolder in dashboard", NewAddColumnMigration(dashboardV2, &Column{ diff --git a/pkg/services/sqlstore/migrations/datasource_mig.go b/pkg/services/sqlstore/migrations/datasource_mig.go index cc8a7f05177..919881adaba 100644 --- a/pkg/services/sqlstore/migrations/datasource_mig.go +++ b/pkg/services/sqlstore/migrations/datasource_mig.go @@ -126,4 +126,8 @@ func addDataSourceMigration(mg *Migrator) { Sqlite(setVersionToOneWhereZero). Postgres(setVersionToOneWhereZero). Mysql(setVersionToOneWhereZero)) + + mg.AddMigration("Add read_only data column", NewAddColumnMigration(tableV2, &Column{ + Name: "read_only", Type: DB_Bool, Nullable: true, + })) } diff --git a/pkg/services/sqlstore/search_builder.go b/pkg/services/sqlstore/search_builder.go new file mode 100644 index 00000000000..99bde21d171 --- /dev/null +++ b/pkg/services/sqlstore/search_builder.go @@ -0,0 +1,214 @@ +package sqlstore + +import ( + "bytes" + "strings" + + m "github.com/grafana/grafana/pkg/models" +) + +// SearchBuilder is a builder/object mother that builds a dashboard search query +type SearchBuilder struct { + tags []string + isStarred bool + limit int + signedInUser *m.SignedInUser + whereDashboardIdsIn []int64 + whereTitle string + whereTypeFolder bool + whereTypeDash bool + whereFolderIds []int64 + sql bytes.Buffer + params []interface{} +} + +func NewSearchBuilder(signedInUser *m.SignedInUser, limit int) *SearchBuilder { + searchBuilder := &SearchBuilder{ + signedInUser: signedInUser, + limit: limit, + } + + return searchBuilder +} + +func (sb *SearchBuilder) WithTags(tags []string) *SearchBuilder { + if len(tags) > 0 { + sb.tags = tags + } + + return sb +} + +func (sb *SearchBuilder) IsStarred() *SearchBuilder { + sb.isStarred = true + + return sb +} + +func (sb *SearchBuilder) WithDashboardIdsIn(ids []int64) *SearchBuilder { + if len(ids) > 0 { + sb.whereDashboardIdsIn = ids + } + + return sb +} + +func (sb *SearchBuilder) WithTitle(title string) *SearchBuilder { + sb.whereTitle = title + + return sb +} + +func (sb *SearchBuilder) WithType(queryType string) *SearchBuilder { + if len(queryType) > 0 && queryType == "dash-folder" { + sb.whereTypeFolder = true + } + + if len(queryType) > 0 && queryType == "dash-db" { + sb.whereTypeDash = true + } + + return sb +} + +func (sb *SearchBuilder) WithFolderIds(folderIds []int64) *SearchBuilder { + sb.whereFolderIds = folderIds + return sb +} + +// ToSql builds the sql and returns it as a string, together with the params. +func (sb *SearchBuilder) ToSql() (string, []interface{}) { + sb.params = make([]interface{}, 0) + + sb.buildSelect() + + if len(sb.tags) > 0 { + sb.buildTagQuery() + } else { + sb.buildMainQuery() + } + + sb.sql.WriteString(` + LEFT OUTER JOIN dashboard folder on folder.id = dashboard.folder_id + LEFT OUTER JOIN dashboard_tag on dashboard.id = dashboard_tag.dashboard_id`) + + sb.sql.WriteString(" ORDER BY dashboard.title ASC LIMIT 5000") + + return sb.sql.String(), sb.params +} + +func (sb *SearchBuilder) buildSelect() { + sb.sql.WriteString( + `SELECT + dashboard.id, + dashboard.title, + dashboard.slug, + dashboard_tag.term, + dashboard.is_folder, + dashboard.folder_id, + folder.slug as folder_slug, + folder.title as folder_title + FROM `) +} + +func (sb *SearchBuilder) buildTagQuery() { + sb.sql.WriteString( + `( + SELECT + dashboard.id FROM dashboard + LEFT OUTER JOIN dashboard_tag ON dashboard_tag.dashboard_id = dashboard.id + `) + + if sb.isStarred { + sb.sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id") + } + + sb.sql.WriteString(` WHERE dashboard_tag.term IN (?` + strings.Repeat(",?", len(sb.tags)-1) + `) AND `) + for _, tag := range sb.tags { + sb.params = append(sb.params, tag) + } + + sb.buildSearchWhereClause() + + // this ends the inner select (tag filtered part) + sb.sql.WriteString(` + GROUP BY dashboard.id HAVING COUNT(dashboard.id) >= ? + LIMIT ?) as ids + INNER JOIN dashboard on ids.id = dashboard.id + `) + + sb.params = append(sb.params, len(sb.tags)) + sb.params = append(sb.params, sb.limit) +} + +func (sb *SearchBuilder) buildMainQuery() { + sb.sql.WriteString(`( SELECT dashboard.id FROM dashboard `) + + if sb.isStarred { + sb.sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id") + } + + sb.sql.WriteString(` WHERE `) + sb.buildSearchWhereClause() + + sb.sql.WriteString(` + LIMIT ?) as ids + INNER JOIN dashboard on ids.id = dashboard.id + `) + sb.params = append(sb.params, sb.limit) +} + +func (sb *SearchBuilder) buildSearchWhereClause() { + sb.sql.WriteString(` dashboard.org_id=?`) + sb.params = append(sb.params, sb.signedInUser.OrgId) + + if sb.isStarred { + sb.sql.WriteString(` AND star.user_id=?`) + sb.params = append(sb.params, sb.signedInUser.UserId) + } + + if len(sb.whereDashboardIdsIn) > 0 { + sb.sql.WriteString(` AND dashboard.id IN (?` + strings.Repeat(",?", len(sb.whereDashboardIdsIn)-1) + `)`) + for _, dashboardId := range sb.whereDashboardIdsIn { + sb.params = append(sb.params, dashboardId) + } + } + + if sb.signedInUser.OrgRole != m.ROLE_ADMIN { + allowedDashboardsSubQuery := ` AND (dashboard.has_acl = 0 OR dashboard.id in ( + SELECT distinct d.id AS DashboardId + FROM dashboard AS d + LEFT JOIN dashboard_acl as da on d.folder_id = da.dashboard_id or d.id = da.dashboard_id + LEFT JOIN user_group_member as ugm on ugm.user_group_id = da.user_group_id + LEFT JOIN org_user ou on ou.role = da.role + WHERE + d.has_acl = 1 and + (da.user_id = ? or ugm.user_id = ? or ou.id is not null) + and d.org_id = ? + ) + )` + + sb.sql.WriteString(allowedDashboardsSubQuery) + sb.params = append(sb.params, sb.signedInUser.UserId, sb.signedInUser.UserId, sb.signedInUser.OrgId) + } + + if len(sb.whereTitle) > 0 { + sb.sql.WriteString(" AND dashboard.title " + dialect.LikeStr() + " ?") + sb.params = append(sb.params, "%"+sb.whereTitle+"%") + } + + if sb.whereTypeFolder { + sb.sql.WriteString(" AND dashboard.is_folder = 1") + } + + if sb.whereTypeDash { + sb.sql.WriteString(" AND dashboard.is_folder = 0") + } + + if len(sb.whereFolderIds) > 0 { + sb.sql.WriteString(` AND dashboard.folder_id IN (?` + strings.Repeat(",?", len(sb.whereFolderIds)-1) + `) `) + for _, id := range sb.whereFolderIds { + sb.params = append(sb.params, id) + } + } +} diff --git a/pkg/services/sqlstore/search_builder_test.go b/pkg/services/sqlstore/search_builder_test.go new file mode 100644 index 00000000000..32ccbc583f5 --- /dev/null +++ b/pkg/services/sqlstore/search_builder_test.go @@ -0,0 +1,37 @@ +package sqlstore + +import ( + "testing" + + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + . "github.com/smartystreets/goconvey/convey" +) + +func TestSearchBuilder(t *testing.T) { + dialect = migrator.NewDialect("sqlite3") + + Convey("Testing building a search", t, func() { + signedInUser := &m.SignedInUser{ + OrgId: 1, + UserId: 1, + } + sb := NewSearchBuilder(signedInUser, 1000) + + Convey("When building a normal search", func() { + sql, params := sb.IsStarred().WithTitle("test").ToSql() + So(sql, ShouldStartWith, "SELECT") + So(sql, ShouldContainSubstring, "INNER JOIN dashboard on ids.id = dashboard.id") + So(sql, ShouldEndWith, "ORDER BY dashboard.title ASC LIMIT 5000") + So(len(params), ShouldBeGreaterThan, 0) + }) + + Convey("When building a search with tag filter", func() { + sql, params := sb.WithTags([]string{"tag1", "tag2"}).ToSql() + So(sql, ShouldStartWith, "SELECT") + So(sql, ShouldContainSubstring, "LEFT OUTER JOIN dashboard_tag") + So(sql, ShouldEndWith, "ORDER BY dashboard.title ASC LIMIT 5000") + So(len(params), ShouldBeGreaterThan, 0) + }) + }) +} diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index e2fc1f29063..f37499bd60f 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -158,10 +158,14 @@ func getEngine() (*xorm.Engine, error) { } else { engine.SetMaxOpenConns(DbCfg.MaxOpenConn) engine.SetMaxIdleConns(DbCfg.MaxIdleConn) - engine.SetLogger(&xorm.DiscardLogger{}) - // engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm"))) - // engine.ShowSQL = true - // engine.ShowInfo = true + debugSql := setting.Cfg.Section("database").Key("log_queries").MustBool(false) + if !debugSql { + engine.SetLogger(&xorm.DiscardLogger{}) + } else { + engine.SetLogger(NewXormLogger(log.LvlInfo, log.New("sqlstore.xorm"))) + engine.ShowSQL(true) + engine.ShowExecTime(true) + } } return engine, nil } @@ -190,12 +194,12 @@ func LoadConfig() { DbCfg.Host = sec.Key("host").String() DbCfg.Name = sec.Key("name").String() DbCfg.User = sec.Key("user").String() - DbCfg.MaxOpenConn = sec.Key("max_open_conn").MustInt(0) - DbCfg.MaxIdleConn = sec.Key("max_idle_conn").MustInt(0) if len(DbCfg.Pwd) == 0 { DbCfg.Pwd = sec.Key("password").String() } } + DbCfg.MaxOpenConn = sec.Key("max_open_conn").MustInt(0) + DbCfg.MaxIdleConn = sec.Key("max_idle_conn").MustInt(0) if DbCfg.Type == "sqlite3" { UseSQLite3 = true diff --git a/pkg/services/sqlstore/user.go b/pkg/services/sqlstore/user.go index 110553d3b05..669f655a159 100644 --- a/pkg/services/sqlstore/user.go +++ b/pkg/services/sqlstore/user.go @@ -401,7 +401,7 @@ func SearchUsers(query *m.SearchUsersQuery) error { } if query.Query != "" { - whereConditions = append(whereConditions, "(email LIKE ? OR name LIKE ? OR login like ?)") + whereConditions = append(whereConditions, "(email "+dialect.LikeStr()+" ? OR name "+dialect.LikeStr()+" ? OR login "+dialect.LikeStr()+" ?)") whereParams = append(whereParams, queryWithWildcards, queryWithWildcards, queryWithWildcards) } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index ca65fe581af..2caf7366727 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -50,11 +50,12 @@ var ( BuildStamp int64 // Paths - LogsPath string - HomePath string - DataPath string - PluginsPath string - CustomInitPath = "conf/custom.ini" + LogsPath string + HomePath string + DataPath string + PluginsPath string + DatasourcesPath string + CustomInitPath = "conf/custom.ini" // Log settings. LogModes []string @@ -89,6 +90,9 @@ var ( SnapShotTTLDays int SnapShotRemoveExpired bool + // Dashboard history + DashboardVersionsToKeep int + // User settings AllowUserSignUp bool AllowUserOrgCreate bool @@ -470,6 +474,7 @@ func NewConfigContext(args *CommandLineArgs) error { Env = Cfg.Section("").Key("app_mode").MustString("development") InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name") PluginsPath = makeAbsolute(Cfg.Section("paths").Key("plugins").String(), HomePath) + DatasourcesPath = makeAbsolute(Cfg.Section("paths").Key("datasources").String(), HomePath) server := Cfg.Section("server") AppUrl, AppSubUrl = parseAppUrlAndSubUrl(server) @@ -518,6 +523,10 @@ func NewConfigContext(args *CommandLineArgs) error { SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true) SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90) + // read dashboard settings + dashboards := Cfg.Section("dashboards") + DashboardVersionsToKeep = dashboards.Key("versions_to_keep").MustInt(20) + // read data source proxy white list DataProxyWhiteList = make(map[string]bool) for _, hostAndIp := range util.SplitString(security.Key("data_source_proxy_whitelist").String()) { @@ -661,5 +670,6 @@ func LogConfigurationInfo() { logger.Info("Path Data", "path", DataPath) logger.Info("Path Logs", "path", LogsPath) logger.Info("Path Plugins", "path", PluginsPath) + logger.Info("Path Datasources", "path", DatasourcesPath) logger.Info("App mode " + Env) } diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 266b71ec14e..d5bdd010269 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/metrics" @@ -24,6 +25,7 @@ import ( type CloudWatchExecutor struct { *models.DataSource + ec2Svc ec2iface.EC2API } type DatasourceInfo struct { @@ -267,7 +269,10 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { period = int(d.Seconds()) } - alias := model.Get("alias").MustString("{{metric}}_{{stat}}") + alias := model.Get("alias").MustString() + if alias == "" { + alias = "{{metric}}_{{stat}}" + } return &CloudWatchQuery{ Region: region, @@ -287,6 +292,7 @@ func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]stri data["namespace"] = query.Namespace data["metric"] = query.MetricName data["stat"] = stat + data["period"] = strconv.Itoa(query.Period) for k, v := range dimensions { data[k] = v } @@ -311,7 +317,8 @@ func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatch var value float64 for _, s := range append(query.Statistics, query.ExtendedStatistics...) { series := tsdb.TimeSeries{ - Tags: map[string]string{}, + Tags: map[string]string{}, + Points: make([]tsdb.TimePoint, 0), } for _, d := range query.Dimensions { series.Tags[*d.Name] = *d.Value diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index b1ce507d27c..b9d4d5b6a80 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -87,6 +87,7 @@ func init() { "AWS/Logs": {"IncomingBytes", "IncomingLogEvents", "ForwardedBytes", "ForwardedLogEvents", "DeliveryErrors", "DeliveryThrottling"}, "AWS/ML": {"PredictCount", "PredictFailureCount"}, "AWS/NATGateway": {"PacketsOutToDestination", "PacketsOutToSource", "PacketsInFromSource", "PacketsInFromDestination", "BytesOutToDestination", "BytesOutToSource", "BytesInFromSource", "BytesInFromDestination", "ErrorPortAllocation", "ActiveConnectionCount", "ConnectionAttemptCount", "ConnectionEstablishedCount", "IdleTimeoutCount", "PacketsDropCount"}, + "AWS/NetworkELB": {"ActiveFlowCount", "ConsumedLCUs", "HealthyHostCount", "NewFlowCount", "ProcessedBytes", "TCP_Client_Reset_Count", "TCP_ELB_Reset_Count", "TCP_Target_Reset_Count", "UnHealthyHostCount"}, "AWS/OpsWorks": {"cpu_idle", "cpu_nice", "cpu_system", "cpu_user", "cpu_waitio", "load_1", "load_5", "load_15", "memory_buffers", "memory_cached", "memory_free", "memory_swap", "memory_total", "memory_used", "procs"}, "AWS/Redshift": {"CPUUtilization", "DatabaseConnections", "HealthStatus", "MaintenanceMode", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "PercentageDiskSpaceUsed", "ReadIOPS", "ReadLatency", "ReadThroughput", "WriteIOPS", "WriteLatency", "WriteThroughput"}, "AWS/RDS": {"ActiveTransactions", "AuroraBinlogReplicaLag", "AuroraReplicaLag", "AuroraReplicaLagMaximum", "AuroraReplicaLagMinimum", "BinLogDiskUsage", "BlockedTransactions", "BufferCacheHitRatio", "CommitLatency", "CommitThroughput", "BinLogDiskUsage", "CPUCreditBalance", "CPUCreditUsage", "CPUUtilization", "DatabaseConnections", "DDLLatency", "DDLThroughput", "Deadlocks", "DeleteLatency", "DeleteThroughput", "DiskQueueDepth", "DMLLatency", "DMLThroughput", "EngineUptime", "FailedSqlStatements", "FreeableMemory", "FreeLocalStorage", "FreeStorageSpace", "InsertLatency", "InsertThroughput", "LoginFailures", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "NetworkThroughput", "Queries", "ReadIOPS", "ReadLatency", "ReadThroughput", "ReplicaLag", "ResultSetCacheHitRatio", "SelectLatency", "SelectThroughput", "SwapUsage", "TotalConnections", "UpdateLatency", "UpdateThroughput", "VolumeBytesUsed", "VolumeReadIOPS", "VolumeWriteIOPS", "WriteIOPS", "WriteLatency", "WriteThroughput"}, @@ -132,6 +133,7 @@ func init() { "AWS/Logs": {"LogGroupName", "DestinationType", "FilterName"}, "AWS/ML": {"MLModelId", "RequestMode"}, "AWS/NATGateway": {"NatGatewayId"}, + "AWS/NetworkELB": {"LoadBalancer", "TargetGroup", "AvailabilityZone"}, "AWS/OpsWorks": {"StackId", "LayerId", "InstanceId"}, "AWS/Redshift": {"NodeID", "ClusterIdentifier"}, "AWS/RDS": {"DBInstanceIdentifier", "DBClusterIdentifier", "DbClusterIdentifier", "DatabaseClass", "EngineName", "Role"}, @@ -183,6 +185,18 @@ func (e *CloudWatchExecutor) executeMetricFindQuery(ctx context.Context, queryCo data, err = e.handleGetEbsVolumeIds(ctx, parameters, queryContext) break case "ec2_instance_attribute": + region := parameters.Get("region").MustString() + dsInfo := e.getDsInfo(region) + cfg, err := e.getAwsConfig(dsInfo) + if err != nil { + return nil, errors.New("Failed to call ec2:DescribeInstances") + } + sess, err := session.NewSession(cfg) + if err != nil { + return nil, errors.New("Failed to call ec2:DescribeInstances") + } + e.ec2Svc = ec2.New(sess, cfg) + data, err = e.handleGetEc2InstanceAttribute(ctx, parameters, queryContext) break } @@ -373,14 +387,16 @@ func (e *CloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context, var filters []*ec2.Filter for k, v := range filterJson { - if vv, ok := v.([]string); ok { - var vvvv []*string + if vv, ok := v.([]interface{}); ok { + var vvvvv []*string for _, vvv := range vv { - vvvv = append(vvvv, &vvv) + if vvvv, ok := vvv.(string); ok { + vvvvv = append(vvvvv, &vvvv) + } } filters = append(filters, &ec2.Filter{ Name: aws.String(k), - Values: vvvv, + Values: vvvvv, }) } } @@ -467,24 +483,13 @@ func (e *CloudWatchExecutor) cloudwatchListMetrics(region string, namespace stri } func (e *CloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.Filter, instanceIds []*string) (*ec2.DescribeInstancesOutput, error) { - dsInfo := e.getDsInfo(region) - cfg, err := e.getAwsConfig(dsInfo) - if err != nil { - return nil, errors.New("Failed to call ec2:DescribeInstances") - } - sess, err := session.NewSession(cfg) - if err != nil { - return nil, errors.New("Failed to call ec2:DescribeInstances") - } - svc := ec2.New(sess, cfg) - params := &ec2.DescribeInstancesInput{ Filters: filters, InstanceIds: instanceIds, } var resp ec2.DescribeInstancesOutput - err = svc.DescribeInstancesPages(params, + err := e.ec2Svc.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { reservations, _ := awsutil.ValuesAtPath(page, "Reservations") for _, reservation := range reservations { diff --git a/pkg/tsdb/cloudwatch/metric_find_query_test.go b/pkg/tsdb/cloudwatch/metric_find_query_test.go index 238e815fac1..255b343a33a 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query_test.go +++ b/pkg/tsdb/cloudwatch/metric_find_query_test.go @@ -1,13 +1,28 @@ package cloudwatch import ( + "context" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) +type mockedEc2 struct { + ec2iface.EC2API + Resp ec2.DescribeInstancesOutput +} + +func (m mockedEc2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error { + fn(&m.Resp, true) + return nil +} + func TestCloudWatchMetrics(t *testing.T) { Convey("When calling getMetricsForCustomMetrics", t, func() { @@ -66,4 +81,37 @@ func TestCloudWatchMetrics(t *testing.T) { }) }) + Convey("When calling handleGetEc2InstanceAttribute", t, func() { + executor := &CloudWatchExecutor{ + ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + InstanceId: aws.String("i-12345678"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Environment"), + Value: aws.String("production"), + }, + }, + }, + }, + }, + }, + }}, + } + + json := simplejson.New() + json.Set("region", "us-east-1") + json.Set("attributeName", "InstanceId") + filters := make(map[string]interface{}) + filters["tag:Environment"] = []string{"production"} + json.Set("filters", filters) + result, _ := executor.handleGetEc2InstanceAttribute(context.Background(), json, &tsdb.TsdbQuery{}) + + Convey("Should equal production InstanceId", func() { + So(result[0].Text, ShouldEqual, "i-12345678") + }) + }) } diff --git a/pkg/tsdb/graphite/graphite.go b/pkg/tsdb/graphite/graphite.go index 7cadf055ff6..73b173813af 100644 --- a/pkg/tsdb/graphite/graphite.go +++ b/pkg/tsdb/graphite/graphite.go @@ -17,7 +17,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" ) type GraphiteExecutor struct { @@ -158,7 +158,7 @@ func formatTimeRange(input string) string { if input == "now" { return input } - return strings.Replace(strings.Replace(input, "m", "min", -1), "M", "mon", -1) + return strings.Replace(strings.Replace(strings.Replace(input, "now", "", -1), "m", "min", -1), "M", "mon", -1) } func fixIntervalFormat(target string) string { diff --git a/pkg/tsdb/graphite/graphite_test.go b/pkg/tsdb/graphite/graphite_test.go index c1a2736293b..1704a9b5f55 100644 --- a/pkg/tsdb/graphite/graphite_test.go +++ b/pkg/tsdb/graphite/graphite_test.go @@ -18,14 +18,14 @@ func TestGraphiteFunctions(t *testing.T) { Convey("formatting time range for now-1m", func() { timeRange := formatTimeRange("now-1m") - So(timeRange, ShouldEqual, "now-1min") + So(timeRange, ShouldEqual, "-1min") }) Convey("formatting time range for now-1M", func() { timeRange := formatTimeRange("now-1M") - So(timeRange, ShouldEqual, "now-1mon") + So(timeRange, ShouldEqual, "-1mon") }) diff --git a/pkg/tsdb/influxdb/model_parser.go b/pkg/tsdb/influxdb/model_parser.go index f3d87739e5b..deb2f15e3ce 100644 --- a/pkg/tsdb/influxdb/model_parser.go +++ b/pkg/tsdb/influxdb/model_parser.go @@ -2,9 +2,11 @@ package influxdb import ( "strconv" + "time" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" ) type InfluxdbQueryParser struct{} @@ -37,13 +39,7 @@ func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.Data return nil, err } - interval := model.Get("interval").MustString("") - if interval == "" && dsInfo.JsonData != nil { - dsInterval := dsInfo.JsonData.Get("timeInterval").MustString("") - if dsInterval != "" { - interval = dsInterval - } - } + parsedInterval, err := tsdb.GetIntervalFrom(dsInfo, model, time.Millisecond*1) return &Query{ Measurement: measurement, @@ -53,7 +49,7 @@ func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.Data Tags: tags, Selects: selects, RawQuery: rawQuery, - Interval: interval, + Interval: parsedInterval, Alias: alias, UseRawQuery: useRawQuery, }, nil diff --git a/pkg/tsdb/influxdb/model_parser_test.go b/pkg/tsdb/influxdb/model_parser_test.go index f7049efb9a7..7be9cae9702 100644 --- a/pkg/tsdb/influxdb/model_parser_test.go +++ b/pkg/tsdb/influxdb/model_parser_test.go @@ -2,6 +2,7 @@ package influxdb import ( "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" @@ -19,7 +20,6 @@ func TestInfluxdbQueryParser(t *testing.T) { Convey("can parse influxdb json model", func() { json := ` { - "dsType": "influxdb", "groupBy": [ { "params": [ @@ -115,14 +115,13 @@ func TestInfluxdbQueryParser(t *testing.T) { So(len(res.GroupBy), ShouldEqual, 3) So(len(res.Selects), ShouldEqual, 3) So(len(res.Tags), ShouldEqual, 2) - So(res.Interval, ShouldEqual, ">20s") + So(res.Interval, ShouldEqual, time.Second*20) So(res.Alias, ShouldEqual, "serie alias") }) Convey("can part raw query json model", func() { json := ` { - "dsType": "influxdb", "groupBy": [ { "params": [ @@ -174,7 +173,7 @@ func TestInfluxdbQueryParser(t *testing.T) { So(len(res.GroupBy), ShouldEqual, 2) So(len(res.Selects), ShouldEqual, 1) So(len(res.Tags), ShouldEqual, 0) - So(res.Interval, ShouldEqual, ">10s") + So(res.Interval, ShouldEqual, time.Second*10) }) }) } diff --git a/pkg/tsdb/influxdb/models.go b/pkg/tsdb/influxdb/models.go index 44e05608290..82ed72c2a18 100644 --- a/pkg/tsdb/influxdb/models.go +++ b/pkg/tsdb/influxdb/models.go @@ -1,5 +1,7 @@ package influxdb +import "time" + type Query struct { Measurement string Policy string @@ -10,8 +12,7 @@ type Query struct { RawQuery string UseRawQuery bool Alias string - - Interval string + Interval time.Duration } type Tag struct { diff --git a/pkg/tsdb/influxdb/query.go b/pkg/tsdb/influxdb/query.go index 3a796e6db63..499f446e9f0 100644 --- a/pkg/tsdb/influxdb/query.go +++ b/pkg/tsdb/influxdb/query.go @@ -29,10 +29,8 @@ func (query *Query) Build(queryContext *tsdb.TsdbQuery) (string, error) { res += query.renderGroupBy(queryContext) } - interval, err := getDefinedInterval(query, queryContext) - if err != nil { - return "", err - } + calculator := tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{}) + interval := calculator.Calculate(queryContext.TimeRange, query.Interval) res = strings.Replace(res, "$timeFilter", query.renderTimeFilter(queryContext), -1) res = strings.Replace(res, "$interval", interval.Text, -1) @@ -41,29 +39,6 @@ func (query *Query) Build(queryContext *tsdb.TsdbQuery) (string, error) { return res, nil } -func getDefinedInterval(query *Query, queryContext *tsdb.TsdbQuery) (*tsdb.Interval, error) { - defaultInterval := tsdb.CalculateInterval(queryContext.TimeRange) - - if query.Interval == "" { - return &defaultInterval, nil - } - - setInterval := strings.Replace(strings.Replace(query.Interval, "<", "", 1), ">", "", 1) - parsedSetInterval, err := time.ParseDuration(setInterval) - - if err != nil { - return nil, err - } - - if strings.Contains(query.Interval, ">") { - if defaultInterval.Value > parsedSetInterval { - return &defaultInterval, nil - } - } - - return &tsdb.Interval{Value: parsedSetInterval, Text: setInterval}, nil -} - func (query *Query) renderTags() []string { var res []string for i, tag := range query.Tags { diff --git a/pkg/tsdb/influxdb/query_test.go b/pkg/tsdb/influxdb/query_test.go index 5c8dc1eaf69..4a620539a26 100644 --- a/pkg/tsdb/influxdb/query_test.go +++ b/pkg/tsdb/influxdb/query_test.go @@ -2,6 +2,7 @@ package influxdb import ( "testing" + "time" "strings" @@ -38,7 +39,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { Measurement: "cpu", Policy: "policy", GroupBy: []*QueryPart{groupBy1, groupBy3}, - Interval: "10s", + Interval: time.Second * 10, } rawQuery, err := query.Build(queryContext) @@ -52,7 +53,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { Measurement: "cpu", GroupBy: []*QueryPart{groupBy1, groupBy2, groupBy3}, Tags: []*Tag{tag1, tag2}, - Interval: "5s", + Interval: time.Second * 5, } rawQuery, err := query.Build(queryContext) @@ -64,7 +65,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { query := &Query{ Selects: []*Select{{*qp1, *qp2, *mathPartDivideBy100}}, Measurement: "cpu", - Interval: "5s", + Interval: time.Second * 5, } rawQuery, err := query.Build(queryContext) @@ -76,7 +77,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { query := &Query{ Selects: []*Select{{*qp1, *qp2, *mathPartDivideByIntervalMs}}, Measurement: "cpu", - Interval: "5s", + Interval: time.Second * 5, } rawQuery, err := query.Build(queryContext) @@ -117,7 +118,7 @@ func TestInfluxdbQueryBuilder(t *testing.T) { Measurement: "cpu", Policy: "policy", GroupBy: []*QueryPart{groupBy1, groupBy3}, - Interval: "10s", + Interval: time.Second * 10, RawQuery: "Raw query", UseRawQuery: true, } diff --git a/pkg/tsdb/influxdb/response_parser.go b/pkg/tsdb/influxdb/response_parser.go index b7db6182241..8de8dcbb464 100644 --- a/pkg/tsdb/influxdb/response_parser.go +++ b/pkg/tsdb/influxdb/response_parser.go @@ -50,6 +50,7 @@ func (rp *ResponseParser) transformRows(rows []Row, queryResult *tsdb.QueryResul result = append(result, &tsdb.TimeSeries{ Name: rp.formatSerieName(row, column, query), Points: points, + Tags: row.Tags, }) } } diff --git a/pkg/tsdb/interval.go b/pkg/tsdb/interval.go index aef6cc4f47b..e26d39f3986 100644 --- a/pkg/tsdb/interval.go +++ b/pkg/tsdb/interval.go @@ -2,14 +2,18 @@ package tsdb import ( "fmt" + "strings" "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" ) var ( - defaultRes int64 = 1500 - minInterval time.Duration = 1 * time.Millisecond - year time.Duration = time.Hour * 24 * 365 - day time.Duration = time.Hour * 24 * 365 + defaultRes int64 = 1500 + defaultMinInterval time.Duration = 1 * time.Millisecond + year time.Duration = time.Hour * 24 * 365 + day time.Duration = time.Hour * 24 ) type Interval struct { @@ -17,14 +21,68 @@ type Interval struct { Value time.Duration } -func CalculateInterval(timerange *TimeRange) Interval { - interval := time.Duration((timerange.MustGetTo().UnixNano() - timerange.MustGetFrom().UnixNano()) / defaultRes) +type intervalCalculator struct { + minInterval time.Duration +} - if interval < minInterval { - return Interval{Text: formatDuration(minInterval), Value: interval} +type IntervalCalculator interface { + Calculate(timeRange *TimeRange, minInterval time.Duration) Interval +} + +type IntervalOptions struct { + MinInterval time.Duration +} + +func NewIntervalCalculator(opt *IntervalOptions) *intervalCalculator { + if opt == nil { + opt = &IntervalOptions{} } - return Interval{Text: formatDuration(roundInterval(interval)), Value: interval} + calc := &intervalCalculator{} + + if opt.MinInterval == 0 { + calc.minInterval = defaultMinInterval + } else { + calc.minInterval = opt.MinInterval + } + + return calc +} + +func (ic *intervalCalculator) Calculate(timerange *TimeRange, minInterval time.Duration) Interval { + to := timerange.MustGetTo().UnixNano() + from := timerange.MustGetFrom().UnixNano() + interval := time.Duration((to - from) / defaultRes) + + if interval < minInterval { + return Interval{Text: formatDuration(minInterval), Value: minInterval} + } + + rounded := roundInterval(interval) + return Interval{Text: formatDuration(rounded), Value: rounded} +} + +func GetIntervalFrom(dsInfo *models.DataSource, queryModel *simplejson.Json, defaultInterval time.Duration) (time.Duration, error) { + interval := queryModel.Get("interval").MustString("") + + if interval == "" && dsInfo.JsonData != nil { + dsInterval := dsInfo.JsonData.Get("timeInterval").MustString("") + if dsInterval != "" { + interval = dsInterval + } + } + + if interval == "" { + return defaultInterval, nil + } + + interval = strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1) + parsedInterval, err := time.ParseDuration(interval) + if err != nil { + return time.Duration(0), err + } + + return parsedInterval, nil } func formatDuration(inter time.Duration) string { diff --git a/pkg/tsdb/interval_test.go b/pkg/tsdb/interval_test.go index 7b243b4e3ba..1e36e5428fe 100644 --- a/pkg/tsdb/interval_test.go +++ b/pkg/tsdb/interval_test.go @@ -14,31 +14,33 @@ func TestInterval(t *testing.T) { HomePath: "../../", }) + calculator := NewIntervalCalculator(&IntervalOptions{}) + Convey("for 5min", func() { tr := NewTimeRange("5m", "now") - interval := CalculateInterval(tr) + interval := calculator.Calculate(tr, time.Millisecond*1) So(interval.Text, ShouldEqual, "200ms") }) Convey("for 15min", func() { tr := NewTimeRange("15m", "now") - interval := CalculateInterval(tr) + interval := calculator.Calculate(tr, time.Millisecond*1) So(interval.Text, ShouldEqual, "500ms") }) Convey("for 30min", func() { tr := NewTimeRange("30m", "now") - interval := CalculateInterval(tr) + interval := calculator.Calculate(tr, time.Millisecond*1) So(interval.Text, ShouldEqual, "1s") }) Convey("for 1h", func() { tr := NewTimeRange("1h", "now") - interval := CalculateInterval(tr) + interval := calculator.Calculate(tr, time.Millisecond*1) So(interval.Text, ShouldEqual, "2s") }) @@ -51,6 +53,7 @@ func TestInterval(t *testing.T) { So(formatDuration(time.Second*61), ShouldEqual, "1m") So(formatDuration(time.Millisecond*30), ShouldEqual, "30ms") So(formatDuration(time.Hour*23), ShouldEqual, "23h") + So(formatDuration(time.Hour*24), ShouldEqual, "1d") So(formatDuration(time.Hour*24*367), ShouldEqual, "1y") }) }) diff --git a/pkg/tsdb/mysql/macros.go b/pkg/tsdb/mysql/macros.go index 36c38804a01..108b81fc5f3 100644 --- a/pkg/tsdb/mysql/macros.go +++ b/pkg/tsdb/mysql/macros.go @@ -3,6 +3,8 @@ package mysql import ( "fmt" "regexp" + "strings" + "time" "github.com/grafana/grafana/pkg/tsdb" ) @@ -25,7 +27,7 @@ func (m *MySqlMacroEngine) Interpolate(timeRange *tsdb.TimeRange, sql string) (s var macroError error sql = replaceAllStringSubmatchFunc(rExp, sql, func(groups []string) string { - res, err := m.evaluateMacro(groups[1], groups[2:]) + res, err := m.evaluateMacro(groups[1], strings.Split(groups[2], ",")) if err != nil && macroError == nil { macroError = err return "macro_error()" @@ -73,6 +75,15 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er return fmt.Sprintf("FROM_UNIXTIME(%d)", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__timeTo": return fmt.Sprintf("FROM_UNIXTIME(%d)", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + case "__timeGroup": + if len(args) != 2 { + return "", fmt.Errorf("macro %v needs time column and interval", name) + } + interval, err := time.ParseDuration(strings.Trim(args[1], `'" `)) + if err != nil { + return "", fmt.Errorf("error parsing interval %v", args[1]) + } + return fmt.Sprintf("cast(cast(UNIX_TIMESTAMP(%s)/(%.0f) as signed)*%.0f as signed)", args[0], interval.Seconds(), interval.Seconds()), nil case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) diff --git a/pkg/tsdb/mysql/macros_test.go b/pkg/tsdb/mysql/macros_test.go index c92020d0aae..988612fb287 100644 --- a/pkg/tsdb/mysql/macros_test.go +++ b/pkg/tsdb/mysql/macros_test.go @@ -40,6 +40,14 @@ func TestMacroEngine(t *testing.T) { So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914186738)") }) + Convey("interpolate __timeGroup function", func() { + + sql, err := engine.Interpolate(timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY cast(cast(UNIX_TIMESTAMP(time_column)/(300) as signed)*300 as signed)") + }) + Convey("interpolate __timeTo function", func() { sql, err := engine.Interpolate(timeRange, "select $__timeTo(time_column)") So(err, ShouldBeNil) diff --git a/pkg/tsdb/mysql/mysql.go b/pkg/tsdb/mysql/mysql.go index bdb48867b6e..e5c6b92f245 100644 --- a/pkg/tsdb/mysql/mysql.go +++ b/pkg/tsdb/mysql/mysql.go @@ -35,7 +35,7 @@ func NewMysqlQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoin MacroEngine: NewMysqlMacroEngine(), } - cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC", + cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true", datasource.User, datasource.Password, "tcp", diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index 21400b03dfd..288787589ce 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "strings" + "time" "github.com/grafana/grafana/pkg/tsdb" ) @@ -71,6 +72,7 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, } return fmt.Sprintf("extract(epoch from %s) as \"time\"", args[0]), nil case "__timeFilter": + // dont use to_timestamp in this macro for redshift compatibility #9566 if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } @@ -80,10 +82,14 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, case "__timeTo": return fmt.Sprintf("to_timestamp(%d)", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeGroup": - if len(args) < 2 { + if len(args) != 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) } - return fmt.Sprintf("(extract(epoch from \"%s\")/extract(epoch from %s::interval))::int*extract(epoch from %s::interval)", args[0], args[1], args[1]), nil + interval, err := time.ParseDuration(strings.Trim(args[1], `' `)) + if err != nil { + return "", fmt.Errorf("error parsing interval %v", args[1]) + } + return fmt.Sprintf("(extract(epoch from \"%s\")/%v)::bigint*%v", args[0], interval.Seconds(), interval.Seconds()), nil case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index ba991e6f2d5..ff268805259 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -45,7 +45,7 @@ func TestMacroEngine(t *testing.T) { sql, err := engine.Interpolate(timeRange, "GROUP BY $__timeGroup(time_column,'5m')") So(err, ShouldBeNil) - So(sql, ShouldEqual, "GROUP BY (extract(epoch from \"time_column\")/extract(epoch from '5m'::interval))::int*extract(epoch from '5m'::interval)") + So(sql, ShouldEqual, "GROUP BY (extract(epoch from \"time_column\")/300)::bigint*300") }) Convey("interpolate __timeTo function", func() { diff --git a/pkg/tsdb/postgres/postgres.go b/pkg/tsdb/postgres/postgres.go index 6fc9c89e7be..a8c96d8119c 100644 --- a/pkg/tsdb/postgres/postgres.go +++ b/pkg/tsdb/postgres/postgres.go @@ -4,6 +4,7 @@ import ( "container/list" "context" "fmt" + "net/url" "strconv" "time" @@ -51,8 +52,8 @@ func generateConnectionString(datasource *models.DataSource) string { } } - sslmode := datasource.JsonData.Get("sslmode").MustString("require") - return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", datasource.User, password, datasource.Url, datasource.Database, sslmode) + sslmode := datasource.JsonData.Get("sslmode").MustString("verify-full") + return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", url.PathEscape(datasource.User), url.PathEscape(password), url.PathEscape(datasource.Url), url.PathEscape(datasource.Database), url.QueryEscape(sslmode)) } func (e *PostgresQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { @@ -77,6 +78,15 @@ func (e PostgresQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Ro rowLimit := 1000000 rowCount := 0 + timeIndex := -1 + + // check if there is a column named time + for i, col := range columnNames { + switch col { + case "time": + timeIndex = i + } + } for ; rows.Next(); rowCount++ { if rowCount > rowLimit { @@ -88,6 +98,15 @@ func (e PostgresQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Ro return err } + // convert column named time to unix timestamp to make + // native datetime postgres types work in annotation queries + if timeIndex != -1 { + switch value := values[timeIndex].(type) { + case time.Time: + values[timeIndex] = float64(value.UnixNano() / 1e9) + } + } + table.Rows = append(table.Rows, values) } @@ -141,8 +160,13 @@ func (e PostgresQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error { pointsBySeries := make(map[string]*tsdb.TimeSeries) seriesByQueryOrder := list.New() - columnNames, err := rows.Columns() + columnNames, err := rows.Columns() + if err != nil { + return err + } + + columnTypes, err := rows.ColumnTypes() if err != nil { return err } @@ -152,13 +176,21 @@ func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *co timeIndex := -1 metricIndex := -1 - // check columns of resultset + // check columns of resultset: a column named time is mandatory + // the first text column is treated as metric name unless a column named metric is present for i, col := range columnNames { switch col { case "time": timeIndex = i case "metric": metricIndex = i + default: + if metricIndex == -1 { + switch columnTypes[i].DatabaseTypeName() { + case "UNKNOWN", "TEXT", "VARCHAR", "CHAR": + metricIndex = i + } + } } } @@ -186,7 +218,7 @@ func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *co case float64: timestamp = columnValue * 1000 case time.Time: - timestamp = float64(columnValue.Unix() * 1000) + timestamp = float64(columnValue.UnixNano() / 1e6) default: return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp") } diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go index 33219837281..e798b92c6fe 100644 --- a/pkg/tsdb/prometheus/prometheus.go +++ b/pkg/tsdb/prometheus/prometheus.go @@ -48,14 +48,16 @@ func NewPrometheusExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, e } var ( - plog log.Logger - legendFormat *regexp.Regexp + plog log.Logger + legendFormat *regexp.Regexp + intervalCalculator tsdb.IntervalCalculator ) func init() { plog = log.New("tsdb.prometheus") tsdb.RegisterTsdbQueryEndpoint("prometheus", NewPrometheusExecutor) legendFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) + intervalCalculator = tsdb.NewIntervalCalculator(&tsdb.IntervalOptions{MinInterval: time.Second * 1}) } func (e *PrometheusExecutor) getClient(dsInfo *models.DataSource) (apiv1.API, error) { @@ -88,7 +90,7 @@ func (e *PrometheusExecutor) Query(ctx context.Context, dsInfo *models.DataSourc return nil, err } - query, err := parseQuery(tsdbQuery.Queries, tsdbQuery) + query, err := parseQuery(dsInfo, tsdbQuery.Queries, tsdbQuery) if err != nil { return nil, err } @@ -138,7 +140,7 @@ func formatLegend(metric model.Metric, query *PrometheusQuery) string { return string(result) } -func parseQuery(queries []*tsdb.Query, queryContext *tsdb.TsdbQuery) (*PrometheusQuery, error) { +func parseQuery(dsInfo *models.DataSource, queries []*tsdb.Query, queryContext *tsdb.TsdbQuery) (*PrometheusQuery, error) { queryModel := queries[0] expr, err := queryModel.Model.Get("expr").String() @@ -146,11 +148,6 @@ func parseQuery(queries []*tsdb.Query, queryContext *tsdb.TsdbQuery) (*Prometheu return nil, err } - step, err := queryModel.Model.Get("step").Int64() - if err != nil { - return nil, err - } - format := queryModel.Model.Get("legendFormat").MustString("") start, err := queryContext.TimeRange.ParseFrom() @@ -163,9 +160,18 @@ func parseQuery(queries []*tsdb.Query, queryContext *tsdb.TsdbQuery) (*Prometheu return nil, err } + dsInterval, err := tsdb.GetIntervalFrom(dsInfo, queryModel.Model, time.Second*15) + if err != nil { + return nil, err + } + + intervalFactor := queryModel.Model.Get("intervalFactor").MustInt64(1) + interval := intervalCalculator.Calculate(queryContext.TimeRange, dsInterval) + step := time.Duration(int64(interval.Value) * intervalFactor) + return &PrometheusQuery{ Expr: expr, - Step: time.Second * time.Duration(step), + Step: step, LegendFormat: format, Start: start, End: end, diff --git a/pkg/tsdb/prometheus/prometheus_test.go b/pkg/tsdb/prometheus/prometheus_test.go index d66ef75e479..c551ab98112 100644 --- a/pkg/tsdb/prometheus/prometheus_test.go +++ b/pkg/tsdb/prometheus/prometheus_test.go @@ -2,13 +2,21 @@ package prometheus import ( "testing" + "time" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" + + "github.com/grafana/grafana/pkg/components/simplejson" p "github.com/prometheus/common/model" . "github.com/smartystreets/goconvey/convey" ) func TestPrometheus(t *testing.T) { Convey("Prometheus", t, func() { + dsInfo := &models.DataSource{ + JsonData: simplejson.New(), + } Convey("converting metric name", func() { metric := map[p.LabelName]p.LabelValue{ @@ -36,5 +44,108 @@ func TestPrometheus(t *testing.T) { So(formatLegend(metric, query), ShouldEqual, `http_request_total{app="backend", device="mobile"}`) }) + + Convey("parsing query model with step", func() { + json := `{ + "expr": "go_goroutines", + "format": "time_series", + "refId": "A" + }` + jsonModel, _ := simplejson.NewJson([]byte(json)) + queryContext := &tsdb.TsdbQuery{} + queryModels := []*tsdb.Query{ + {Model: jsonModel}, + } + + Convey("with 48h time range", func() { + queryContext.TimeRange = tsdb.NewTimeRange("12h", "now") + + model, err := parseQuery(dsInfo, queryModels, queryContext) + + So(err, ShouldBeNil) + So(model.Step, ShouldEqual, time.Second*30) + }) + }) + + Convey("parsing query model without step parameter", func() { + json := `{ + "expr": "go_goroutines", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }` + jsonModel, _ := simplejson.NewJson([]byte(json)) + queryContext := &tsdb.TsdbQuery{} + queryModels := []*tsdb.Query{ + {Model: jsonModel}, + } + + Convey("with 48h time range", func() { + queryContext.TimeRange = tsdb.NewTimeRange("48h", "now") + + model, err := parseQuery(dsInfo, queryModels, queryContext) + + So(err, ShouldBeNil) + So(model.Step, ShouldEqual, time.Minute*2) + }) + + Convey("with 1h time range", func() { + queryContext.TimeRange = tsdb.NewTimeRange("1h", "now") + + model, err := parseQuery(dsInfo, queryModels, queryContext) + + So(err, ShouldBeNil) + So(model.Step, ShouldEqual, time.Second*15) + }) + }) + + Convey("parsing query model with intervalFactor", func() { + Convey("high intervalFactor", func() { + json := `{ + "expr": "go_goroutines", + "format": "time_series", + "intervalFactor": 10, + "refId": "A" + }` + jsonModel, _ := simplejson.NewJson([]byte(json)) + queryContext := &tsdb.TsdbQuery{} + queryModels := []*tsdb.Query{ + {Model: jsonModel}, + } + + Convey("with 48h time range", func() { + queryContext.TimeRange = tsdb.NewTimeRange("48h", "now") + + model, err := parseQuery(dsInfo, queryModels, queryContext) + + So(err, ShouldBeNil) + So(model.Step, ShouldEqual, time.Minute*20) + }) + }) + + Convey("low intervalFactor", func() { + json := `{ + "expr": "go_goroutines", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }` + jsonModel, _ := simplejson.NewJson([]byte(json)) + queryContext := &tsdb.TsdbQuery{} + queryModels := []*tsdb.Query{ + {Model: jsonModel}, + } + + Convey("with 48h time range", func() { + queryContext.TimeRange = tsdb.NewTimeRange("48h", "now") + + model, err := parseQuery(dsInfo, queryModels, queryContext) + + So(err, ShouldBeNil) + So(model.Step, ShouldEqual, time.Minute*2) + }) + }) + }) + }) } diff --git a/pkg/tsdb/sql_engine.go b/pkg/tsdb/sql_engine.go index d79ca938bb4..12778b4e1ad 100644 --- a/pkg/tsdb/sql_engine.go +++ b/pkg/tsdb/sql_engine.go @@ -57,12 +57,13 @@ func (e *DefaultSqlEngine) InitEngine(driverName string, dsInfo *models.DataSour } engine, err := xorm.NewEngine(driverName, cnnstr) - engine.SetMaxOpenConns(10) - engine.SetMaxIdleConns(10) if err != nil { return err } + engine.SetMaxOpenConns(10) + engine.SetMaxIdleConns(10) + engineCache.cache[dsInfo.Id] = engine e.XormEngine = engine diff --git a/pkg/tsdb/testdata/scenarios.go b/pkg/tsdb/testdata/scenarios.go index 0a7f1467933..e907fa8aae0 100644 --- a/pkg/tsdb/testdata/scenarios.go +++ b/pkg/tsdb/testdata/scenarios.go @@ -1,6 +1,7 @@ package testdata import ( + "encoding/json" "math/rand" "strconv" "strings" @@ -142,6 +143,45 @@ func init() { }, }) + registerScenario(&Scenario{ + Id: "manual_entry", + Name: "Manual Entry", + Handler: func(query *tsdb.Query, context *tsdb.TsdbQuery) *tsdb.QueryResult { + queryRes := tsdb.NewQueryResult() + + points := query.Model.Get("points").MustArray() + + series := newSeriesForQuery(query) + startTime := context.TimeRange.GetFromAsMsEpoch() + endTime := context.TimeRange.GetToAsMsEpoch() + + for _, val := range points { + pointValues := val.([]interface{}) + + var value null.Float + var time int64 + + if valueFloat, err := strconv.ParseFloat(string(pointValues[0].(json.Number)), 64); err == nil { + value = null.FloatFrom(valueFloat) + } + + if timeInt, err := strconv.ParseInt(string(pointValues[1].(json.Number)), 10, 64); err != nil { + continue + } else { + time = timeInt + } + + if time >= startTime && time <= endTime { + series.Points = append(series.Points, tsdb.NewTimePoint(value, float64(time))) + } + } + + queryRes.Series = append(queryRes.Series, series) + + return queryRes + }, + }) + registerScenario(&Scenario{ Id: "csv_metric_values", Name: "CSV Metric Values", diff --git a/pkg/util/encoding.go b/pkg/util/encoding.go index e87da9d3d55..0edb721e422 100644 --- a/pkg/util/encoding.go +++ b/pkg/util/encoding.go @@ -8,7 +8,6 @@ import ( "encoding/base64" "encoding/hex" "errors" - "fmt" "hash" "strings" ) @@ -30,7 +29,7 @@ func GetRandomString(n int, alphabets ...byte) string { func EncodePassword(password string, salt string) string { newPasswd := PBKDF2([]byte(password), []byte(salt), 10000, 50, sha256.New) - return fmt.Sprintf("%x", newPasswd) + return hex.EncodeToString(newPasswd) } // Encode string to md5 hex value. diff --git a/pkg/util/encoding_test.go b/pkg/util/encoding_test.go index abcf5425826..6b0e9331467 100644 --- a/pkg/util/encoding_test.go +++ b/pkg/util/encoding_test.go @@ -7,7 +7,6 @@ import ( ) func TestEncoding(t *testing.T) { - Convey("When generating base64 header", t, func() { result := GetBasicAuthHeader("grafana", "1234") @@ -23,4 +22,8 @@ func TestEncoding(t *testing.T) { So(password, ShouldEqual, "1234") }) + Convey("When encoding password", t, func() { + encodedPassword := EncodePassword("iamgod", "pepper") + So(encodedPassword, ShouldEqual, "e59c568621e57756495a468f47c74e07c911b037084dd464bb2ed72410970dc849cabd71b48c394faf08a5405dae53741ce9") + }) } diff --git a/pkg/util/url.go b/pkg/util/url.go index ba452596a2b..c82dcef67c5 100644 --- a/pkg/util/url.go +++ b/pkg/util/url.go @@ -9,10 +9,15 @@ type UrlQueryReader struct { values url.Values } -func NewUrlQueryReader(url *url.URL) *UrlQueryReader { - return &UrlQueryReader{ - values: url.Query(), +func NewUrlQueryReader(urlInfo *url.URL) (*UrlQueryReader, error) { + u, err := url.ParseQuery(urlInfo.String()) + if err != nil { + return nil, err } + + return &UrlQueryReader{ + values: u, + }, nil } func (r *UrlQueryReader) Get(name string, def string) string { diff --git a/public/app/app.ts b/public/app/app.ts index 8e345c6abed..b76dceb4943 100644 --- a/public/app/app.ts +++ b/public/app/app.ts @@ -9,7 +9,6 @@ import 'angular-native-dragdrop'; import 'angular-bindonce'; import 'react'; import 'react-dom'; -import 'ngreact'; import 'vendor/bootstrap/bootstrap'; import 'vendor/angular-ui/ui-bootstrap-tpls'; diff --git a/public/app/core/angular_wrappers.ts b/public/app/core/angular_wrappers.ts index 7acdc79d55a..4cb3ca513cc 100644 --- a/public/app/core/angular_wrappers.ts +++ b/public/app/core/angular_wrappers.ts @@ -1,8 +1,10 @@ import { react2AngularDirective } from 'app/core/utils/react2angular'; import { PasswordStrength } from './components/PasswordStrength'; +import PageHeader from './components/PageHeader/PageHeader'; +import EmptyListCTA from './components/EmptyListCTA/EmptyListCTA'; export function registerAngularDirectives() { - react2AngularDirective('passwordStrength', PasswordStrength, ['password']); - + react2AngularDirective('pageHeader', PageHeader, ['model', 'noTabs']); + react2AngularDirective('emptyListCta', EmptyListCTA, ['model']); } diff --git a/public/app/core/components/EmptyListCTA/EmptyListCTA.jest.tsx b/public/app/core/components/EmptyListCTA/EmptyListCTA.jest.tsx new file mode 100644 index 00000000000..d62ae892a0a --- /dev/null +++ b/public/app/core/components/EmptyListCTA/EmptyListCTA.jest.tsx @@ -0,0 +1,22 @@ +import React from 'react'; +import renderer from 'react-test-renderer'; +import EmptyListCTA from './EmptyListCTA'; + +const model = { + title: 'Title', + buttonIcon: 'ga css class', + buttonLink: 'http://url/to/destination', + buttonTitle: 'Click me', + proTip: 'This is a tip', + proTipLink: 'http://url/to/tip/destination', + proTipLinkTitle: 'Learn more', + proTipTarget: '_blank' +}; + +describe('CollorPalette', () => { + + it('renders correctly', () => { + const tree = renderer.create().toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/public/app/core/components/EmptyListCTA/EmptyListCTA.tsx b/public/app/core/components/EmptyListCTA/EmptyListCTA.tsx new file mode 100644 index 00000000000..a5a78a63bf1 --- /dev/null +++ b/public/app/core/components/EmptyListCTA/EmptyListCTA.tsx @@ -0,0 +1,34 @@ +import React, { Component } from 'react'; + +export interface IProps { + model: any; +} + +class EmptyListCTA extends Component { + render() { + const { + title, + buttonIcon, + buttonLink, + buttonTitle, + proTip, + proTipLink, + proTipLinkTitle, + proTipTarget + } = this.props.model; + return ( +
+
{title}
+ {buttonTitle} +
+ ProTip: {proTip} + {proTipLinkTitle} +
+
+ ); + } +} + +export default EmptyListCTA; diff --git a/public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.jest.tsx.snap b/public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.jest.tsx.snap new file mode 100644 index 00000000000..0598d7ecd4e --- /dev/null +++ b/public/app/core/components/EmptyListCTA/__snapshots__/EmptyListCTA.jest.tsx.snap @@ -0,0 +1,38 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CollorPalette renders correctly 1`] = ` +
+
+ Title +
+ + + Click me + +
+ + ProTip: + This is a tip + + Learn more + +
+
+`; diff --git a/public/app/core/components/PageHeader/PageHeader.tsx b/public/app/core/components/PageHeader/PageHeader.tsx new file mode 100644 index 00000000000..3ab6cac8143 --- /dev/null +++ b/public/app/core/components/PageHeader/PageHeader.tsx @@ -0,0 +1,127 @@ +import React from 'react'; +import { NavModel, NavModelItem } from '../../nav_model_srv'; +import classNames from 'classnames'; +import appEvents from 'app/core/app_events'; + +export interface IProps { + model: NavModel; +} + +function TabItem(tab: NavModelItem) { + if (tab.hideFromTabs) { + return (null); + } + + let tabClasses = classNames({ + 'gf-tabs-link': true, + active: tab.active, + }); + + return ( +
  • + + + {tab.text} + +
  • + ); +} + +function SelectOption(navItem: NavModelItem) { + if (navItem.hideFromTabs) { // TODO: Rename hideFromTabs => hideFromNav + return (null); + } + + return ( + + ); +} + +function Navigation({main}: {main: NavModelItem}) { + return (); +} + +function SelectNav({main, customCss}: {main: NavModelItem, customCss: string}) { + const defaultSelectedItem = main.children.find(navItem => { + return navItem.active === true; + }); + + const gotoUrl = evt => { + var element = evt.target; + var url = element.options[element.selectedIndex].value; + appEvents.emit('location-change', {href: url}); + }; + + return (); +} + +function Tabs({main, customCss}: {main: NavModelItem, customCss: string}) { + return
      {main.children.map(TabItem)}
    ; +} + +export default class PageHeader extends React.Component { + constructor(props) { + super(props); + } + + renderBreadcrumb(breadcrumbs) { + const breadcrumbsResult = []; + for (let i = 0; i < breadcrumbs.length; i++) { + const bc = breadcrumbs[i]; + if (bc.uri) { + breadcrumbsResult.push({bc.title}); + } else { + breadcrumbsResult.push( / {bc.title}); + } + } + return breadcrumbsResult; + } + + renderHeaderTitle(main) { + return ( +
    + + {main.icon && } + {main.img && } + + +
    + {main.text &&

    {main.text}

    } + {main.breadcrumbs && main.breadcrumbs.length > 0 && ( +

    + {this.renderBreadcrumb(main.breadcrumbs)} +

    ) + } + {main.subTitle &&
    {main.subTitle}
    } + {main.subType && ( +
    + + {main.subType.text} +
    + )} +
    +
    + ); + } + + render() { + return ( +
    +
    +
    + {this.renderHeaderTitle(this.props.model.main)} + {this.props.model.main.children && } +
    +
    +
    + ); + } +} diff --git a/public/app/core/components/ScrollBar/ScrollBar.tsx b/public/app/core/components/ScrollBar/ScrollBar.tsx new file mode 100644 index 00000000000..49a200b0f3b --- /dev/null +++ b/public/app/core/components/ScrollBar/ScrollBar.tsx @@ -0,0 +1,62 @@ +import React from 'react'; +import PerfectScrollbar from 'perfect-scrollbar'; + +export interface Props { + children: any; + className: string; +} + +export default class ScrollBar extends React.Component { + + private container: any; + private ps: PerfectScrollbar; + + constructor(props) { + super(props); + } + + componentDidMount() { + this.ps = new PerfectScrollbar(this.container); + } + + componentDidUpdate() { + this.ps.update(); + } + + componentWillUnmount() { + this.ps.destroy(); + } + + // methods can be invoked by outside + setScrollTop(top) { + if (this.container) { + this.container.scrollTop = top; + this.ps.update(); + + return true; + } + return false; + } + + setScrollLeft(left) { + if (this.container) { + this.container.scrollLeft = left; + this.ps.update(); + + return true; + } + return false; + } + + handleRef = ref => { + this.container = ref; + }; + + render() { + return ( +
    + {this.props.children} +
    + ); + } +} diff --git a/public/app/core/components/code_editor/code_editor.ts b/public/app/core/components/code_editor/code_editor.ts index 2615a635c7e..cc3b1e46ad4 100644 --- a/public/app/core/components/code_editor/code_editor.ts +++ b/public/app/core/components/code_editor/code_editor.ts @@ -36,6 +36,8 @@ import 'brace/mode/text'; import 'brace/snippets/text'; import 'brace/mode/sql'; import 'brace/snippets/sql'; +import 'brace/mode/markdown'; +import 'brace/snippets/markdown'; const DEFAULT_THEME_DARK = "ace/theme/grafana-dark"; const DEFAULT_THEME_LIGHT = "ace/theme/textmate"; diff --git a/public/app/core/components/colorpicker/ColorPickerPopover.tsx b/public/app/core/components/colorpicker/ColorPickerPopover.tsx index 4677eee22a7..360c3fdd5c4 100644 --- a/public/app/core/components/colorpicker/ColorPickerPopover.tsx +++ b/public/app/core/components/colorpicker/ColorPickerPopover.tsx @@ -56,10 +56,11 @@ export class ColorPickerPopover extends React.Component { let newColor = tinycolor(colorString); if (newColor.isValid()) { // Update only color state + let newColorString = newColor.toString(); this.setState({ - color: newColor.toString(), + color: newColorString, }); - this.props.onColorSelect(newColor); + this.props.onColorSelect(newColorString); } } diff --git a/public/app/core/components/colorpicker/SeriesColorPicker.tsx b/public/app/core/components/colorpicker/SeriesColorPicker.tsx index 6b6d387a2b2..3b24b9a4661 100644 --- a/public/app/core/components/colorpicker/SeriesColorPicker.tsx +++ b/public/app/core/components/colorpicker/SeriesColorPicker.tsx @@ -43,7 +43,7 @@ export class SeriesColorPicker extends React.Component { render() { return (
    - {this.props.series && this.renderAxisSelection()} + {this.props.series.yaxis && this.renderAxisSelection()}
    ); diff --git a/public/app/core/components/form_dropdown/form_dropdown.ts b/public/app/core/components/form_dropdown/form_dropdown.ts index b1cbd9b6ced..364d61cfcdb 100644 --- a/public/app/core/components/form_dropdown/form_dropdown.ts +++ b/public/app/core/components/form_dropdown/form_dropdown.ts @@ -1,5 +1,3 @@ -/// - import _ from 'lodash'; import $ from 'jquery'; import coreModule from '../../core_module'; @@ -159,6 +157,8 @@ export class FormDropdownCtrl { } updateValue(text) { + text = _.unescape(text); + if (text === '' || this.text === text) { return; } @@ -199,9 +199,9 @@ export class FormDropdownCtrl { } open() { - this.inputElement.show(); - this.inputElement.css('width', (Math.max(this.linkElement.width(), 80) + 16) + 'px'); + + this.inputElement.show(); this.inputElement.focus(); this.linkElement.hide(); diff --git a/public/app/core/components/grafana_app.ts b/public/app/core/components/grafana_app.ts index d13c6a6a00e..eff55405d6f 100644 --- a/public/app/core/components/grafana_app.ts +++ b/public/app/core/components/grafana_app.ts @@ -1,5 +1,3 @@ -/// - import config from 'app/core/config'; import _ from 'lodash'; import $ from 'jquery'; @@ -12,7 +10,7 @@ import Drop from 'tether-drop'; export class GrafanaCtrl { /** @ngInject */ - constructor($scope, alertSrv, utilSrv, $rootScope, $controller, contextSrv) { + constructor($scope, alertSrv, utilSrv, $rootScope, $controller, contextSrv, globalEventSrv) { $scope.init = function() { $scope.contextSrv = contextSrv; @@ -23,6 +21,7 @@ export class GrafanaCtrl { profiler.init(config, $rootScope); alertSrv.init(); utilSrv.init(); + globalEventSrv.init(); $scope.dashAlerts = alertSrv; }; @@ -78,11 +77,16 @@ export function grafanaAppDirective(playlistSrv, contextSrv, $timeout, $rootScop sidemenuOpen = scope.contextSrv.sidemenu; body.toggleClass('sidemenu-open', sidemenuOpen); - scope.$watch('contextSrv.sidemenu', newVal => { - if (sidemenuOpen !== scope.contextSrv.sidemenu) { - sidemenuOpen = scope.contextSrv.sidemenu; - body.toggleClass('sidemenu-open', scope.contextSrv.sidemenu); - } + appEvents.on('toggle-sidemenu', () => { + body.toggleClass('sidemenu-open'); + }); + + appEvents.on('toggle-sidemenu-mobile', () => { + body.toggleClass('sidemenu-open--xs'); + }); + + appEvents.on('toggle-sidemenu-hidden', () => { + body.toggleClass('sidemenu-hidden'); }); // tooltip removal fix @@ -100,6 +104,9 @@ export function grafanaAppDirective(playlistSrv, contextSrv, $timeout, $rootScop } } + // clear body class sidemenu states + body.removeClass('sidemenu-open--xs'); + $("#tooltip, .tooltip").remove(); // check for kiosk url param diff --git a/public/app/core/components/manage_dashboards/manage_dashboards.html b/public/app/core/components/manage_dashboards/manage_dashboards.html new file mode 100644 index 00000000000..957bca0262c --- /dev/null +++ b/public/app/core/components/manage_dashboards/manage_dashboards.html @@ -0,0 +1,98 @@ +
    +
    + + +
    +
    + + + Dashboard + + + + Folder + +
    + +
    + Filters: + + + + {{tagName}} + + +
    + +
    +
    + +
    +
    + +
    +
    + +
    + +
    + + +
    +
    +
    +
    + +
    +
    + +
    + +
    \ No newline at end of file diff --git a/public/app/core/components/manage_dashboards/manage_dashboards.ts b/public/app/core/components/manage_dashboards/manage_dashboards.ts new file mode 100644 index 00000000000..1aa8a3f014a --- /dev/null +++ b/public/app/core/components/manage_dashboards/manage_dashboards.ts @@ -0,0 +1,221 @@ +import _ from 'lodash'; +import coreModule from 'app/core/core_module'; +import appEvents from 'app/core/app_events'; +import { SearchSrv } from 'app/core/services/search_srv'; + +export class ManageDashboardsCtrl { + public sections: any[]; + tagFilterOptions: any[]; + selectedTagFilter: any; + query: any; + navModel: any; + canDelete = false; + canMove = false; + hasFilters = false; + selectAllChecked = false; + starredFilterOptions = [{ text: 'Filter by Starred', disabled: true }, { text: 'Yes' }, { text: 'No' }]; + selectedStarredFilter: any; + folderId?: number; + + /** @ngInject */ + constructor(private backendSrv, navModelSrv, private $q, private searchSrv: SearchSrv) { + this.query = { query: '', mode: 'tree', tag: [], starred: false, skipRecent: true, skipStarred: true }; + + if (this.folderId) { + this.query.folderIds = [this.folderId]; + } + + this.selectedStarredFilter = this.starredFilterOptions[0]; + + this.getDashboards().then(() => { + this.getTags(); + }); + } + + getDashboards() { + return this.searchSrv.search(this.query).then((result) => { + return this.initDashboardList(result); + }); + } + + initDashboardList(result: any) { + this.canMove = false; + this.canDelete = false; + this.selectAllChecked = false; + this.hasFilters = this.query.query.length > 0 || this.query.tag.length > 0 || this.query.starred; + + if (!result) { + this.sections = []; + return; + } + + this.sections = result; + + for (let section of this.sections) { + section.checked = false; + + for (let dashboard of section.items) { + dashboard.checked = false; + } + } + } + + selectionChanged() { + + let selectedDashboards = 0; + + for (let section of this.sections) { + selectedDashboards += _.filter(section.items, { checked: true }).length; + } + + const selectedFolders = _.filter(this.sections, { checked: true }).length; + this.canMove = selectedDashboards > 0 && selectedFolders === 0; + this.canDelete = selectedDashboards > 0 || selectedFolders > 0; + } + + getDashboardsToDelete() { + let selectedDashboards = []; + + for (const section of this.sections) { + if (section.checked) { + selectedDashboards.push(section.uri); + } else { + const selected = _.filter(section.items, { checked: true }); + selectedDashboards.push(..._.map(selected, 'uri')); + } + } + + return selectedDashboards; + } + + getFolderIds(sections) { + const ids = []; + for (let s of sections) { + if (s.checked) { + ids.push(s.id); + } + } + return ids; + } + + delete() { + const selectedDashboards = this.getDashboardsToDelete(); + + appEvents.emit('confirm-modal', { + title: 'Delete', + text: `Do you want to delete the ${selectedDashboards.length} selected dashboards?`, + icon: 'fa-trash', + yesText: 'Delete', + onConfirm: () => { + const promises = []; + for (let dash of selectedDashboards) { + promises.push(this.backendSrv.delete(`/api/dashboards/${dash}`)); + } + + this.$q.all(promises).then(() => { + this.getDashboards(); + }); + } + }); + } + + getDashboardsToMove() { + let selectedDashboards = []; + + for (const section of this.sections) { + const selected = _.filter(section.items, { checked: true }); + selectedDashboards.push(..._.map(selected, 'uri')); + } + + return selectedDashboards; + } + + moveTo() { + const selectedDashboards = this.getDashboardsToMove(); + + const template = '' + + '`'; + appEvents.emit('show-modal', { + templateHtml: template, + modalClass: 'modal--narrow', + model: { dashboards: selectedDashboards, fromFolderId: this.folderId ? Number(this.folderId) : 0, afterSave: this.getDashboards.bind(this) } + }); + } + + getTags() { + return this.searchSrv.getDashboardTags().then((results) => { + this.tagFilterOptions = [{ term: 'Filter By Tag', disabled: true }].concat(results); + this.selectedTagFilter = this.tagFilterOptions[0]; + }); + } + + filterByTag(tag) { + if (_.indexOf(this.query.tag, tag) === -1) { + this.query.tag.push(tag); + } + + return this.getDashboards(); + } + + onQueryChange() { + return this.getDashboards(); + } + + onTagFilterChange() { + var res = this.filterByTag(this.selectedTagFilter.term); + this.selectedTagFilter = this.tagFilterOptions[0]; + return res; + } + + removeTag(tag, evt) { + this.query.tag = _.without(this.query.tag, tag); + this.getDashboards(); + if (evt) { + evt.stopPropagation(); + evt.preventDefault(); + } + } + + onStarredFilterChange() { + this.query.starred = this.selectedStarredFilter.text === 'Yes'; + return this.getDashboards(); + } + + onSelectAllChanged() { + for (let section of this.sections) { + if (!section.hideHeader) { + section.checked = this.selectAllChecked; + } + + section.items = _.map(section.items, (item) => { + item.checked = this.selectAllChecked; + return item; + }); + } + + this.selectionChanged(); + } + + clearFilters() { + this.query.query = ''; + this.query.tag = []; + this.query.starred = false; + this.getDashboards(); + } +} + +export function manageDashboardsDirective() { + return { + restrict: 'E', + templateUrl: 'public/app/core/components/manage_dashboards/manage_dashboards.html', + controller: ManageDashboardsCtrl, + bindToController: true, + controllerAs: 'ctrl', + scope: { + folderId: '=' + } + }; +} + +coreModule.directive('manageDashboards', manageDashboardsDirective); diff --git a/public/app/core/components/navbar/navbar.html b/public/app/core/components/navbar/navbar.html index ab18c24e503..6d611692efc 100644 --- a/public/app/core/components/navbar/navbar.html +++ b/public/app/core/components/navbar/navbar.html @@ -1,20 +1,11 @@
    -
    -
    - +
    +
    +
    +
    No dashboards matching your query were found.
    + +
    +
    -
    -
    No dashboards matching your query were found.
    + -
    +
    + + + +
    +
    + + +
    +
    diff --git a/public/app/core/components/search/search.ts b/public/app/core/components/search/search.ts index 32ea6800106..ffbec30cc0a 100644 --- a/public/app/core/components/search/search.ts +++ b/public/app/core/components/search/search.ts @@ -1,7 +1,6 @@ -/// - import _ from 'lodash'; import coreModule from '../../core_module'; +import { SearchSrv } from 'app/core/services/search_srv'; export class SearchCtrl { isOpen: boolean; @@ -10,23 +9,22 @@ export class SearchCtrl { selectedIndex: number; results: any; currentSearchId: number; - tagsMode: boolean; showImport: boolean; dismiss: any; ignoreClose: any; - // triggers fade animation class - openCompleted: boolean; + isLoading: boolean; + initialFolderFilterTitle: string; /** @ngInject */ - constructor($scope, private $location, private $timeout, private backendSrv, public contextSrv, $rootScope) { + constructor($scope, private $location, private $timeout, private searchSrv: SearchSrv, $rootScope) { $rootScope.onAppEvent('show-dash-search', this.openSearch.bind(this), $scope); $rootScope.onAppEvent('hide-dash-search', this.closeSearch.bind(this), $scope); + + this.initialFolderFilterTitle = "All"; } closeSearch() { this.isOpen = this.ignoreClose; - this.openCompleted = false; - this.contextSrv.isSearching = this.isOpen; } openSearch(evt, payload) { @@ -36,28 +34,19 @@ export class SearchCtrl { } this.isOpen = true; - this.contextSrv.isSearching = true; this.giveSearchFocus = 0; this.selectedIndex = -1; this.results = []; - this.query = { query: '', tag: [], starred: false, mode: 'tree' }; + this.query = { query: '', tag: [], starred: false }; this.currentSearchId = 0; this.ignoreClose = true; + this.isLoading = true; if (payload && payload.starred) { this.query.starred = true; } - if (payload && payload.tagsMode) { - return this.$timeout(() => { - this.ignoreClose = false; - this.giveSearchFocus = this.giveSearchFocus + 1; - this.getTags(); - }, 100); - } - this.$timeout(() => { - this.openCompleted = true; this.ignoreClose = false; this.giveSearchFocus = this.giveSearchFocus + 1; this.search(); @@ -75,76 +64,86 @@ export class SearchCtrl { this.moveSelection(-1); } if (evt.keyCode === 13) { - if (this.tagsMode) { - var tag = this.results[this.selectedIndex]; - if (tag) { - this.filterByTag(tag.term, null); - } - return; - } + const flattenedResult = this.getFlattenedResultForNavigation(); + const currentItem = flattenedResult[this.selectedIndex]; - var selectedDash = this.results[this.selectedIndex]; - if (selectedDash) { - this.$location.search({}); - this.$location.path(selectedDash.url); + if (currentItem) { + if (currentItem.dashboardIndex !== undefined) { + const selectedDash = this.results[currentItem.folderIndex].items[currentItem.dashboardIndex]; + + if (selectedDash) { + this.$location.search({}); + this.$location.path(selectedDash.url); + } + } else { + const selectedFolder = this.results[currentItem.folderIndex]; + + if (selectedFolder) { + selectedFolder.toggle(selectedFolder); + } + } } } } moveSelection(direction) { - var max = (this.results || []).length; - var newIndex = this.selectedIndex + direction; + if (this.results.length === 0) { + return; + } + + const flattenedResult = this.getFlattenedResultForNavigation(); + const currentItem = flattenedResult[this.selectedIndex]; + + if (currentItem) { + if (currentItem.dashboardIndex !== undefined) { + this.results[currentItem.folderIndex].items[currentItem.dashboardIndex].selected = false; + } else { + this.results[currentItem.folderIndex].selected = false; + } + } + + if (direction === 0) { + this.selectedIndex = -1; + return; + } + + const max = flattenedResult.length; + let newIndex = this.selectedIndex + direction; this.selectedIndex = ((newIndex %= max) < 0) ? newIndex + max : newIndex; + const selectedItem = flattenedResult[this.selectedIndex]; + + if (selectedItem.dashboardIndex === undefined && this.results[selectedItem.folderIndex].id === 0) { + this.moveSelection(direction); + return; + } + + if (selectedItem.dashboardIndex !== undefined) { + if (!this.results[selectedItem.folderIndex].expanded) { + this.moveSelection(direction); + return; + } + + this.results[selectedItem.folderIndex].items[selectedItem.dashboardIndex].selected = true; + return; + } + + if (this.results[selectedItem.folderIndex].hideHeader) { + this.moveSelection(direction); + return; + } + + this.results[selectedItem.folderIndex].selected = true; } searchDashboards() { - this.tagsMode = false; this.currentSearchId = this.currentSearchId + 1; var localSearchId = this.currentSearchId; - return this.backendSrv.search(this.query).then(results => { + return this.searchSrv.search(this.query).then(results => { if (localSearchId < this.currentSearchId) { return; } - - let byId = _.groupBy(results, 'id'); - let byFolderId = _.groupBy(results, 'folderId'); - let finalList = []; - - // add missing parent folders - _.each(results, (hit, index) => { - if (hit.folderId && !byId[hit.folderId]) { - const folder = { - id: hit.folderId, - uri: `db/${hit.folderSlug}`, - title: hit.folderTitle, - type: 'dash-folder' - }; - byId[hit.folderId] = folder; - results.splice(index, 0, folder); - } - }); - - // group by folder - for (let hit of results) { - if (hit.folderId) { - hit.type = "dash-child"; - } else { - finalList.push(hit); - } - - hit.url = 'dashboard/' + hit.uri; - - if (hit.type === 'dash-folder') { - if (!byFolderId[hit.id]) { - continue; - } - - for (let child of byFolderId[hit.id]) { - finalList.push(child); - } - } - } - - this.results = finalList; + this.results = results || []; + this.isLoading = false; + this.moveSelection(1); }); } @@ -153,13 +152,11 @@ export class SearchCtrl { return query.query === '' && query.starred === false && query.tag.length === 0; } - filterByTag(tag, evt) { - this.query.tag.push(tag); - this.search(); - this.giveSearchFocus = this.giveSearchFocus + 1; - if (evt) { - evt.stopPropagation(); - evt.preventDefault(); + filterByTag(tag) { + if (_.indexOf(this.query.tag, tag) === -1) { + this.query.tag.push(tag); + this.search(); + this.giveSearchFocus = this.giveSearchFocus + 1; } } @@ -172,13 +169,9 @@ export class SearchCtrl { } getTags() { - return this.backendSrv.get('/api/dashboards/tags').then((results) => { - this.tagsMode = !this.tagsMode; + return this.searchSrv.getDashboardTags().then((results) => { this.results = results; this.giveSearchFocus = this.giveSearchFocus + 1; - if ( !this.tagsMode ) { - this.search(); - } }); } @@ -190,10 +183,37 @@ export class SearchCtrl { search() { this.showImport = false; - this.selectedIndex = 0; + this.selectedIndex = -1; this.searchDashboards(); } + folderExpanding() { + this.moveSelection(0); + } + + private getFlattenedResultForNavigation() { + let folderIndex = 0; + + return _.flatMap(this.results, (s) => { + let result = []; + + result.push({ + folderIndex: folderIndex + }); + + let dashboardIndex = 0; + + result = result.concat(_.map(s.items || [], (i) => { + return { + folderIndex: folderIndex, + dashboardIndex: dashboardIndex++ + }; + })); + + folderIndex++; + return result; + }); + } } export function searchDirective() { diff --git a/public/app/core/components/search/search_results.html b/public/app/core/components/search/search_results.html new file mode 100644 index 00000000000..ad0aa3e6b71 --- /dev/null +++ b/public/app/core/components/search/search_results.html @@ -0,0 +1,47 @@ + \ No newline at end of file diff --git a/public/app/core/components/search/search_results.ts b/public/app/core/components/search/search_results.ts new file mode 100644 index 00000000000..02b6c9f355b --- /dev/null +++ b/public/app/core/components/search/search_results.ts @@ -0,0 +1,76 @@ +// import _ from 'lodash'; +import coreModule from '../../core_module'; + +export class SearchResultsCtrl { + results: any; + onSelectionChanged: any; + onTagSelected: any; + onFolderExpanding: any; + + /** @ngInject */ + constructor(private $location) { + + } + + toggleFolderExpand(section) { + if (section.toggle) { + if (!section.expanded && this.onFolderExpanding) { + this.onFolderExpanding(); + } + + section.toggle(section); + } + } + + navigateToFolder(section, evt) { + this.$location.path('/dashboards/folder/' + section.id + '/' + section.uri); + + if (evt) { + evt.stopPropagation(); + evt.preventDefault(); + } + } + + toggleSelection(item, evt) { + item.checked = !item.checked; + + if (this.onSelectionChanged) { + this.onSelectionChanged(); + } + + if (evt) { + evt.stopPropagation(); + evt.preventDefault(); + } + } + + selectTag(tag, evt) { + if (this.onTagSelected) { + this.onTagSelected({$tag: tag}); + } + + if (evt) { + evt.stopPropagation(); + evt.preventDefault(); + } + } +} + +export function searchResultsDirective() { + return { + restrict: 'E', + templateUrl: 'public/app/core/components/search/search_results.html', + controller: SearchResultsCtrl, + bindToController: true, + controllerAs: 'ctrl', + scope: { + editable: '@', + results: '=', + onSelectionChanged: '&', + onTagSelected: '&', + onFolderExpanding: '&' + }, + }; +} + +coreModule.directive('dashboardSearchResults', searchResultsDirective); diff --git a/public/app/core/components/sidemenu/sidemenu.html b/public/app/core/components/sidemenu/sidemenu.html index 2869562c5b1..e97d34739ba 100644 --- a/public/app/core/components/sidemenu/sidemenu.html +++ b/public/app/core/components/sidemenu/sidemenu.html @@ -2,6 +2,11 @@ + + +  Close + +