From 6a82098ddf9fd925c42cfef6599a8e893c0fa275 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 11:52:00 +0200 Subject: [PATCH 1/5] devenv: scripts for generating many unique dashboards --- .gitignore | 2 + .../dashboards/bulk-testing/bulkdash.jsonnet | 1140 +++++++++++++++++ devenv/dashboards/generate-bulk-dashboards.sh | 15 + 3 files changed, 1157 insertions(+) create mode 100644 devenv/dashboards/bulk-testing/bulkdash.jsonnet create mode 100755 devenv/dashboards/generate-bulk-dashboards.sh diff --git a/.gitignore b/.gitignore index cbc85835a36..45dcb52e8d8 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,5 @@ debug.test /vendor/**/.editorconfig /vendor/**/appengine* *.orig + +/devenv/dashboards/bulk-testing/*.json diff --git a/devenv/dashboards/bulk-testing/bulkdash.jsonnet b/devenv/dashboards/bulk-testing/bulkdash.jsonnet new file mode 100644 index 00000000000..17b3f8983af --- /dev/null +++ b/devenv/dashboards/bulk-testing/bulkdash.jsonnet @@ -0,0 +1,1140 @@ +{ + "annotations": { + "enable": false, + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "aliasColors": { + "cpu": "#E24D42", + "memory": "#1f78c1", + "statsd.fakesite.counters.session_start.desktop.count": "#6ED0E0" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 3, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 4, + "interactive": true, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": true, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [ + { + "alias": "cpu", + "fill": 0, + "lines": true, + "yaxis": 2, + "zindex": 2 + }, + { + "alias": "memory", + "pointradius": 2, + "points": true + } + ], + "spaceLength": 10, + "spyable": true, + "stack": false, + "steppedLine": false, + "targets": [ + { + "hide": false, + "refId": "A", + "target": "alias(movingAverage(scaleToSeconds(apps.fakesite.web_server_01.counters.request_status.code_302.count, 10), 20), 'cpu')" + }, + { + "refId": "B", + "target": "alias(statsd.fakesite.counters.session_start.desktop.count, 'memory')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "Memory / CPU", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "aliasColors": { + "logins": "#5195ce", + "logins (-1 day)": "#447EBC", + "logins (-1 hour)": "#705da0" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 1, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 3, + "interactive": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "alias(movingAverage(scaleToSeconds(apps.fakesite.web_server_01.counters.requests.count, 1), 2), 'logins')" + }, + { + "refId": "B", + "target": "alias(movingAverage(timeShift(scaleToSeconds(apps.fakesite.web_server_01.counters.requests.count, 1), '1h'), 2), 'logins (-1 hour)')" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": "1h", + "timezone": "browser", + "title": "logins", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.4)" + } + ], + "thresholds": "200,270", + "title": "Memory", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "apps.backend.backend_02.counters.requests.count" + } + ], + "thresholds": "100,270", + "title": "Sign ups", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 17, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "apps.backend.backend_04.counters.requests.count" + } + ], + "thresholds": "100,270", + "title": "Sign outs", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.7)" + } + ], + "thresholds": "100,270", + "title": "Logins", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "web_server_01": "#badff4", + "web_server_02": "#5195ce", + "web_server_03": "#1f78c1", + "web_server_04": "#0a437c" + }, + "annotate": { + "enable": false + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 6, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 11, + "w": 16, + "x": 0, + "y": 7 + }, + "id": 2, + "interactive": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "legend_counts": true, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(movingAverage(scaleToSeconds(apps.fakesite.*.counters.requests.count, 1), 2), 2)" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "server requests", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 7 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.8)" + } + ], + "thresholds": "200,270", + "title": "Logouts", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_03.counters.requests.count, 0.3)" + } + ], + "thresholds": "100,270", + "title": "Support calls", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 12 + }, + "id": 26, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.2)" + } + ], + "thresholds": "200,270", + "title": "Google hits", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#629e51", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 300, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 12 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "refId": "A", + "target": "scale(apps.backend.backend_01.counters.requests.count, 0.2)" + } + ], + "thresholds": "200,270", + "title": "Google hits", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "upper_25": "#F9E2D2", + "upper_50": "#F2C96D", + "upper_75": "#EAB839" + }, + "annotate": { + "enable": false + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": true, + "fill": 1, + "grid": { + "max": null, + "min": 0 + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 5, + "interactive": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "legend_counts": true, + "lines": false, + "linewidth": 2, + "nullPointMode": "connected", + "options": false, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "resolution": 100, + "scale": 1, + "seriesOverrides": [], + "spaceLength": 10, + "spyable": true, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(summarize(statsd.fakesite.timers.ads_timer.*, '4min', 'avg'), 4)" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "timezone": "browser", + "title": "client side full page load", + "tooltip": { + "msResolution": false, + "query_as_alias": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "zerofill": true + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "demo" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "2h", + " 6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Big Dashboard", + "uid": "000000003", + "version": 16 +} \ No newline at end of file diff --git a/devenv/dashboards/generate-bulk-dashboards.sh b/devenv/dashboards/generate-bulk-dashboards.sh new file mode 100755 index 00000000000..079a5a9c520 --- /dev/null +++ b/devenv/dashboards/generate-bulk-dashboards.sh @@ -0,0 +1,15 @@ +#/bin/bash + +if ! type "jsonnet" > /dev/null; then + echo "you need you install jsonnet to run this script" + echo "follow the instructions on https://github.com/google/jsonnet" + exit 1 +fi + +COUNTER=0 +MAX=400 +while [ $COUNTER -lt $MAX ]; do + jsonnet -o "bulk-testing/dashboard${COUNTER}.json" -e "local bulkDash = import 'bulk-testing/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" + let COUNTER=COUNTER+1 +done + From a1e6c31ec12a43f7cd7605031e9cbce7c2c667d6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 14:00:46 +0200 Subject: [PATCH 2/5] devenv: script for setting up default datasources --- .../bulk-testing/bulk-dashboards.yaml | 9 ++ devenv/dashboards/generate-bulk-dashboards.sh | 15 ---- devenv/datasources/default/default.yaml | 82 +++++++++++++++++++ devenv/setup.sh | 61 ++++++++++++++ 4 files changed, 152 insertions(+), 15 deletions(-) create mode 100644 devenv/dashboards/bulk-testing/bulk-dashboards.yaml delete mode 100755 devenv/dashboards/generate-bulk-dashboards.sh create mode 100644 devenv/datasources/default/default.yaml create mode 100755 devenv/setup.sh diff --git a/devenv/dashboards/bulk-testing/bulk-dashboards.yaml b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml new file mode 100644 index 00000000000..7838e4bc342 --- /dev/null +++ b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: 'Bulk dashboards' + folder: 'Bulk dashboards' + type: file + options: + path: /home/carl/go/src/github.com/grafana/grafana/devenv/dashboards/bulk-testing + diff --git a/devenv/dashboards/generate-bulk-dashboards.sh b/devenv/dashboards/generate-bulk-dashboards.sh deleted file mode 100755 index 079a5a9c520..00000000000 --- a/devenv/dashboards/generate-bulk-dashboards.sh +++ /dev/null @@ -1,15 +0,0 @@ -#/bin/bash - -if ! type "jsonnet" > /dev/null; then - echo "you need you install jsonnet to run this script" - echo "follow the instructions on https://github.com/google/jsonnet" - exit 1 -fi - -COUNTER=0 -MAX=400 -while [ $COUNTER -lt $MAX ]; do - jsonnet -o "bulk-testing/dashboard${COUNTER}.json" -e "local bulkDash = import 'bulk-testing/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" - let COUNTER=COUNTER+1 -done - diff --git a/devenv/datasources/default/default.yaml b/devenv/datasources/default/default.yaml new file mode 100644 index 00000000000..b721c093f3a --- /dev/null +++ b/devenv/datasources/default/default.yaml @@ -0,0 +1,82 @@ +apiVersion: 1 + +datasources: + - name: Graphite + type: graphite + access: proxy + url: http://localhost:8080 + jsonData: + graphiteVersion: "1.1" + + - name: Prometheus + type: prometheus + access: proxy + isDefault: true + url: http://localhost:9090 + + - name: InfluxDB + type: influxdb + access: proxy + database: site + user: grafana + password: grafana + url: http://localhost:8086 + jsonData: + timeInterval: "15s" + + - name: OpenTsdb + type: opentsdb + access: proxy + url: http://localhost:4242 + jsonData: + tsdbResolution: 1 + tsdbVersion: 1 + + - name: Elastic + type: elasticsearch + access: proxy + database: "[metrics-]YYYY.MM.DD" + url: http://localhost:9200 + jsonData: + interval: Daily + timeField: "@timestamp" + + - name: MySQL + type: mysql + url: localhost:3306 + database: grafana + user: grafana + password: password + + - name: MSSQL + type: mssql + url: localhost:1433 + database: grafana + user: grafana + password: "Password!" + + - name: Postgres + type: postgres + url: localhost:5432 + database: grafana + user: grafana + password: password + jsonData: + sslmode: "disable" + + - name: Cloudwatch + type: cloudwatch + editable: true + jsonData: + authType: credentials + defaultRegion: eu-west-2 + + - name: Cloudwatch keys + type: cloudwatch + editable: true + jsonData: + authType: keys + defaultRegion: eu-west-2 + secureJsonData: + accessKey: AKIAJL347VWN6MK63N2A + secretKey: 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 diff --git a/devenv/setup.sh b/devenv/setup.sh new file mode 100755 index 00000000000..d6f8f969e75 --- /dev/null +++ b/devenv/setup.sh @@ -0,0 +1,61 @@ +#/bin/bash + +bulkDashboard() { + + requiresJsonnet + + COUNTER=0 + MAX=400 + while [ $COUNTER -lt $MAX ]; do + jsonnet -o "dashboards/bulk-testing/dashboard${COUNTER}.json" -e "local bulkDash = import 'dashboards/bulk-testing/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" + let COUNTER=COUNTER+1 + done + + ln -s -f -r ./dashboards/bulk-testing/bulk-dashboards.yaml ../conf/provisioning/dashboards/custom.yaml +} + +requiresJsonnet() { + if ! type "jsonnet" > /dev/null; then + echo "you need you install jsonnet to run this script" + echo "follow the instructions on https://github.com/google/jsonnet" + exit 1 + fi +} + +defaultDashboards() { + echo "not implemented yet" +} + +defaultDatasources() { + echo "setting up all default datasources using provisioning" + + ln -s -f -r ./datasources/default/default.yaml ../conf/provisioning/datasources/custom.yaml +} + +usage() { + echo -e "install.sh\n\tThis script installs my basic setup for a debian laptop\n" + echo "Usage:" + echo " bulk-dashboards - create and provisioning 400 dashboards" + echo " default-datasources - provisiong all core datasources" +} + +main() { + local cmd=$1 + + if [[ -z "$cmd" ]]; then + usage + exit 1 + fi + + if [[ $cmd == "bulk-dashboards" ]]; then + bulkDashboard + elif [[ $cmd == "default-datasources" ]]; then + defaultDatasources + elif [[ $cmd == "default-dashboards" ]]; then + bulkDashboard + else + usage + fi +} + +main "$@" \ No newline at end of file From 1411709db1c8ce65fd45906fcbc43c7757256084 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 14:07:37 +0200 Subject: [PATCH 3/5] provisioning: place testfiles within testdata folder --- .../all-properties/not.yaml.txt => devenv/README.md | 0 devenv/datasources/default/default.yaml | 9 --------- .../provisioning/datasources/config_reader_test.go | 12 ++++++------ .../all-properties/all-properties.yaml | 0 .../all-properties/not.yaml.txt} | 0 .../all-properties/sample.yaml | 0 .../all-properties/second.yaml | 0 .../broken-yaml/broken.yaml | 0 .../broken-yaml/commented.yaml | 0 .../double-default/default-1.yaml | 0 .../double-default/default-2.yaml | 0 .../insert-two-delete-two/one-datasources.yaml | 0 .../insert-two-delete-two/two-datasources.yml | 0 .../two-datasources/two-datasources.yaml | 0 .../version-0/version-0.yaml | 0 .../testdata/zero-datasources/placeholder-for-git | 0 16 files changed, 6 insertions(+), 15 deletions(-) rename pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt => devenv/README.md (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/all-properties/all-properties.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs/zero-datasources/placeholder-for-git => testdata/all-properties/not.yaml.txt} (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/all-properties/sample.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/all-properties/second.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/broken-yaml/broken.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/broken-yaml/commented.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/double-default/default-1.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/double-default/default-2.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/insert-two-delete-two/one-datasources.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/insert-two-delete-two/two-datasources.yml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/two-datasources/two-datasources.yaml (100%) rename pkg/services/provisioning/datasources/{test-configs => testdata}/version-0/version-0.yaml (100%) create mode 100644 pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt b/devenv/README.md similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt rename to devenv/README.md diff --git a/devenv/datasources/default/default.yaml b/devenv/datasources/default/default.yaml index b721c093f3a..dc2310f15aa 100644 --- a/devenv/datasources/default/default.yaml +++ b/devenv/datasources/default/default.yaml @@ -71,12 +71,3 @@ datasources: authType: credentials defaultRegion: eu-west-2 - - name: Cloudwatch keys - type: cloudwatch - editable: true - jsonData: - authType: keys - defaultRegion: eu-west-2 - secureJsonData: - accessKey: AKIAJL347VWN6MK63N2A - secretKey: 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 diff --git a/pkg/services/provisioning/datasources/config_reader_test.go b/pkg/services/provisioning/datasources/config_reader_test.go index 89ecc5a0b68..2e407dbe4de 100644 --- a/pkg/services/provisioning/datasources/config_reader_test.go +++ b/pkg/services/provisioning/datasources/config_reader_test.go @@ -13,12 +13,12 @@ import ( var ( logger log.Logger = log.New("fake.log") - twoDatasourcesConfig = "./test-configs/two-datasources" - twoDatasourcesConfigPurgeOthers = "./test-configs/insert-two-delete-two" - doubleDatasourcesConfig = "./test-configs/double-default" - allProperties = "./test-configs/all-properties" - versionZero = "./test-configs/version-0" - brokenYaml = "./test-configs/broken-yaml" + twoDatasourcesConfig = "testdata/two-datasources" + twoDatasourcesConfigPurgeOthers = "testdata/insert-two-delete-two" + doubleDatasourcesConfig = "testdata/double-default" + allProperties = "testdata/all-properties" + versionZero = "testdata/version-0" + brokenYaml = "testdata/broken-yaml" fakeRepo *fakeRepository ) diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasources/testdata/all-properties/not.yaml.txt similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git rename to pkg/services/provisioning/datasources/testdata/all-properties/not.yaml.txt diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/sample.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/sample.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/sample.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/sample.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/second.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml rename to pkg/services/provisioning/datasources/testdata/all-properties/second.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml b/pkg/services/provisioning/datasources/testdata/broken-yaml/broken.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml rename to pkg/services/provisioning/datasources/testdata/broken-yaml/broken.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml b/pkg/services/provisioning/datasources/testdata/broken-yaml/commented.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml rename to pkg/services/provisioning/datasources/testdata/broken-yaml/commented.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml b/pkg/services/provisioning/datasources/testdata/double-default/default-1.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml rename to pkg/services/provisioning/datasources/testdata/double-default/default-1.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml b/pkg/services/provisioning/datasources/testdata/double-default/default-2.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml rename to pkg/services/provisioning/datasources/testdata/double-default/default-2.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml b/pkg/services/provisioning/datasources/testdata/insert-two-delete-two/one-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml rename to pkg/services/provisioning/datasources/testdata/insert-two-delete-two/one-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml b/pkg/services/provisioning/datasources/testdata/insert-two-delete-two/two-datasources.yml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml rename to pkg/services/provisioning/datasources/testdata/insert-two-delete-two/two-datasources.yml diff --git a/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml b/pkg/services/provisioning/datasources/testdata/two-datasources/two-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml rename to pkg/services/provisioning/datasources/testdata/two-datasources/two-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/version-0/version-0.yaml b/pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/version-0/version-0.yaml rename to pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml diff --git a/pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasources/testdata/zero-datasources/placeholder-for-git new file mode 100644 index 00000000000..e69de29bb2d From b253284accef14e4ad5fa0d89ee55c8837cb5047 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 29 May 2018 16:52:02 +0200 Subject: [PATCH 4/5] devenv: improve readme --- devenv/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/devenv/README.md b/devenv/README.md index e69de29bb2d..4ec6f672f25 100644 --- a/devenv/README.md +++ b/devenv/README.md @@ -0,0 +1,11 @@ +This folder contains useful scripts and configuration for... + +* Configuring datasources in Grafana +* Provision example dashboards in Grafana +* Run preconfiured datasources as docker containers + +want to know more? run setup! + +```bash +./setup.sh +``` From c7acbcdaf5e28092a2be9d44c348d2a767bc7e3b Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 30 May 2018 08:46:44 +0200 Subject: [PATCH 5/5] provisioning: enable relative path's this commit enable relatives path for provisioning dashboards. this enables easier dev setups --- .../bulk-testing/bulk-dashboards.yaml | 2 +- .../provisioning/dashboards/file_reader.go | 8 +- .../dashboards/file_reader_test.go | 75 ++++++++++++------- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/devenv/dashboards/bulk-testing/bulk-dashboards.yaml b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml index 7838e4bc342..e0ba8a88e68 100644 --- a/devenv/dashboards/bulk-testing/bulk-dashboards.yaml +++ b/devenv/dashboards/bulk-testing/bulk-dashboards.yaml @@ -5,5 +5,5 @@ providers: folder: 'Bulk dashboards' type: file options: - path: /home/carl/go/src/github.com/grafana/grafana/devenv/dashboards/bulk-testing + path: devenv/dashboards/bulk-testing diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index e5186e12f06..93846f5c474 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -47,9 +47,15 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade log.Error("Cannot read directory", "error", err) } + absPath, err := filepath.Abs(path) + if err != nil { + log.Error("Could not create absolute path ", "path", path) + absPath = path //if .Abs return an error we fallback to path + } + return &fileReader{ Cfg: cfg, - Path: path, + Path: absPath, log: log, dashboardService: dashboards.NewProvisioningService(), }, nil diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 084fae1310a..a04fbb23f82 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -15,14 +15,57 @@ import ( ) var ( - defaultDashboards = "./testdata/test-dashboards/folder-one" - brokenDashboards = "./testdata/test-dashboards/broken-dashboards" - oneDashboard = "./testdata/test-dashboards/one-dashboard" - containingId = "./testdata/test-dashboards/containing-id" + defaultDashboards = "testdata/test-dashboards/folder-one" + brokenDashboards = "testdata/test-dashboards/broken-dashboards" + oneDashboard = "testdata/test-dashboards/one-dashboard" + containingId = "testdata/test-dashboards/containing-id" fakeService *fakeDashboardProvisioningService ) +func TestCreatingNewDashboardFileReader(t *testing.T) { + Convey("creating new dashboard file reader", t, func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{}, + } + + Convey("using path parameter", func() { + cfg.Options["path"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + So(reader.Path, ShouldNotEqual, "") + }) + + Convey("using folder as options", func() { + cfg.Options["folder"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + So(reader.Path, ShouldNotEqual, "") + }) + + Convey("using full path", func() { + cfg.Options["folder"] = "/var/lib/grafana/dashboards" + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + + So(reader.Path, ShouldEqual, "/var/lib/grafana/dashboards") + So(filepath.IsAbs(reader.Path), ShouldBeTrue) + }) + + Convey("using relative path", func() { + cfg.Options["folder"] = defaultDashboards + reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) + So(err, ShouldBeNil) + + So(filepath.IsAbs(reader.Path), ShouldBeTrue) + }) + }) +} + func TestDashboardFileReader(t *testing.T) { Convey("Dashboard file reader", t, func() { bus.ClearBusHandlers() @@ -170,30 +213,6 @@ func TestDashboardFileReader(t *testing.T) { }) }) - Convey("Can use bpth path and folder as dashboard path", func() { - cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", - Options: map[string]interface{}{}, - } - - Convey("using path parameter", func() { - cfg.Options["path"] = defaultDashboards - reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) - So(err, ShouldBeNil) - So(reader.Path, ShouldEqual, defaultDashboards) - }) - - Convey("using folder as options", func() { - cfg.Options["folder"] = defaultDashboards - reader, err := NewDashboardFileReader(cfg, log.New("test-logger")) - So(err, ShouldBeNil) - So(reader.Path, ShouldEqual, defaultDashboards) - }) - }) - Reset(func() { dashboards.NewProvisioningService = origNewDashboardProvisioningService })