{ "__inputs": [ ], "__requires": [ ], "annotations": { "list": [ { "datasource": "$datasourcelogs", "enable": true, "expr": "{cluster=\"$cluster\", component=\"kube-diff-logger\"} | json | namespace_extracted=~\"example-service.*\" | name_extracted=\"example-api\"", "iconColor": "rgba(0, 211, 255, 1)", "instant": false, "name": "Deployments", "titleFormat": "{{cluster}}/{{namespace}}" } ] }, "editable": false, "gnetId": null, "graphTooltip": 1, "hideControls": false, "id": null, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "cache-overview" ], "targetBlank": false, "title": "Cache Dashboard", "type": "dashboards", "url": "" }, { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "cloudsql-stackdriver" ], "targetBlank": false, "title": "CloudSQL Dashboard", "type": "dashboards", "url": "" }, { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "Database", "percona" ], "targetBlank": false, "title": "Database Dashboard", "type": "dashboards", "url": "" } ], "refresh": "1m", "rows": [ { "collapse": false, "collapsed": false, "panels": [ { "datasource": "$datasource", "fieldConfig": { "defaults": { "links": [ ], "mappings": [ ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "none" } }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 0 }, "id": 2, "links": [ ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "7", "targets": [ { "expr": "sum(app_metric_counter_a{env=\"$environment\", region=\"$region\", service=\"api\"}) - 1", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Tenants", "refId": "A" }, { "expr": "sum(app_metric_counter_b{env=\"$environment\", region=\"$region\", service=\"api\"}) - 1", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Collectors", "refId": "B" }, { "expr": "sum(app_metric_counter_c{env=\"$environment\", region=\"$region\", service=\"api\"}) - 1", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Pipelines", "refId": "C" } ], "title": "Customer Stats", "transparent": false, "type": "stat" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "links": [ ], "mappings": [ ], "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": 0 }, { "color": "green", "value": 40 }, { "color": "yellow", "value": 60 }, { "color": "orange", "value": 80 }, { "color": "red", "value": 100 } ] }, "unit": "percent" } }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 0 }, "id": 3, "links": [ ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "7", "targets": [ { "expr": "avg(sum by (pod) (app_memory_usage_bytes{env=\"$environment\", region=\"$region\", instance=~\"example-api.*\"}) * 100) / min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"example-api.*\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "API Mem", "refId": "A" }, { "expr": "avg(sum by (pod) (app_memory_usage_bytes{env=\"$environment\", region=\"$region\", instance=~\"cache.*\", component=\"cache-server\"}) * 100) / min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"cache.*\", component=\"cache-server\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Cache Server Mem", "refId": "B" }, { "expr": "avg(sum by (pod) (app_memory_usage_bytes{env=\"$environment\", region=\"$region\", instance=~\"gateway.*\"}) * 100) / min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"gateway.*\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Gateway Mem ", "refId": "C" }, { "expr": "avg(sum by (pod) (rate(app_cpu_usage_ratio{env=\"$environment\", region=\"$region\", instance=~\"example-api.*\"}[1m]))) * 100 / min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"example-api.*\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "API CPU", "refId": "D" }, { "expr": "avg(sum by (pod) (rate(app_cpu_usage_ratio{env=\"$environment\", region=\"$region\", instance=~\"cache.*\", component=\"cache-server\"}[1m]))) * 100 / min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"cache.*\", component=\"cache-server\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Cache Server CPU", "refId": "E" }, { "expr": "avg(sum by (pod) (rate(app_cpu_usage_ratio{env=\"$environment\", region=\"$region\", instance=~\"gateway.*\"}[1m]))) * 100 / min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"gateway.*\"})", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Gateway CPU", "refId": "F" } ], "title": "Requested Resource Usage", "transparent": false, "type": "stat" }, { "datasource": "$datasource", "fieldConfig": { "defaults": { "links": [ ], "mappings": [ ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "reqps" } }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 9 }, "id": 4, "links": [ ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "7", "targets": [ { "expr": "sum by () (\n label_replace(\n rate(example_service_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", route!~\"|root|other|ready|metrics|debug_pprof\"}[$__rate_interval]),\n \"status\",\n \"${1}xx\",\n \"\",\n \"([0-9])..\")\n )\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{}}", "refId": "A" } ], "title": "Api Requests", "transparent": false, "type": "stat" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Overview", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "panels": [ { "aliasColors": { "Limit": "#E24D42", "Requested": "#Ffa500" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 0 }, "id": 5, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (app_memory_usage_bytes{env=\"$environment\", region=\"$region\", instance=~\"example-api.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" }, { "expr": "min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"example-api.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested", "refId": "B" }, { "expr": "min(app_resource_limit{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"example-api.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { "Limit": "#E24D42", "Requested": "#Ffa500" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 0 }, "id": 6, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (rate(app_cpu_usage_ratio{env=\"$environment\", region=\"$region\", instance=~\"example-api.*\", component=~\".*\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" }, { "expr": "min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"example-api.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested", "refId": "B" }, { "expr": "min(app_resource_limit{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"example-api.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { "1xx": "#EAB839", "2xx": "#7EB26D", "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 0 }, "id": 7, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (status_code) (\n label_replace(\n rate(example_service_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", route!~\"|root|other|ready|metrics|debug_pprof\"}[$__rate_interval]),\n \"status\",\n \"${1}xx\",\n \"status_code\",\n \"([0-9])..\")\n )\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{status_code}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Throughput (requests/sec)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "decimals": 2, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "decimals": 2, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 9 }, "id": 8, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(example_service_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\", route!~\"|root|other|ready|metrics|debug_pprof\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p99", "refId": "A" }, { "expr": "histogram_quantile(0.90, sum(rate(example_service_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\", route!~\"|root|other|ready|metrics|debug_pprof\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p90", "refId": "B" }, { "expr": "histogram_quantile(0.50, sum(rate(example_service_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\", route!~\"|root|other|ready|metrics|debug_pprof\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p50", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 9 }, "id": 9, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "(rate(example_service_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", route!~\"|root|other|ready|metrics|debug_pprof\", status_code=~\"5..\"}[$__rate_interval]) / rate(example_service_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", route!~\"|root|other|ready|metrics|debug_pprof\"}[$__rate_interval])) OR on() vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Error rate", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Error rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true } ] }, { "datasource": "$datasourcelogs", "gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 }, "id": 10, "options": { "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true }, "span": 12, "targets": [ { "expr": "{env=\"$environment\", region=\"$region\", service=\"api\"}", "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Logs", "type": "logs" }, { "datasource": "$datasourcelogs", "gridPos": { "h": 9, "w": 24, "x": 0, "y": 9 }, "id": 11, "options": { "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true }, "span": 12, "targets": [ { "expr": "{env=\"$environment\", region=\"$region\", service=\"api\"} | logfmt | level=\"error\"", "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Errors", "type": "logs" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "API", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 0 }, "id": 12, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(example_service_database_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Latency p99", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 0 }, "id": 13, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(example_service_database_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Latency p90", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 0 }, "id": 14, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(example_service_database_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Latency p50", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 9 }, "id": 15, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(example_service_database_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Total", "refId": "A" }, { "expr": "sum(rate(example_service_database_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", method=~\"GetConfig.*\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "GetConfig", "refId": "B" }, { "expr": "sum(rate(example_service_database_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", method!~\"GetConfig.*\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Other", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Throughput (requests/sec)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 9 }, "id": 16, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (go_sql_in_use_connections {env=\"$environment\", region=\"$region\", service=\"api\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}/Active", "refId": "A" }, { "expr": "sum by (pod) (go_sql_idle_connections {env=\"$environment\", region=\"$region\", service=\"api\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}/Idle", "refId": "B" }, { "expr": "sum by (pod) (go_sql_max_open_connections {env=\"$environment\", region=\"$region\", service=\"api\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}/Max", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Connections", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 9 }, "id": 17, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (rate(go_sql_wait_duration_seconds_total{env=\"$environment\", region=\"$region\", service=\"api\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Connection Wait Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Database", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "panels": [ { "aliasColors": { "Limit": "#E24D42", "Requested": "#Ffa500" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 0 }, "id": 18, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (app_memory_usage_bytes{env=\"$environment\", region=\"$region\", instance=~\"cache.*\", component=~\"cache-server\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" }, { "expr": "min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"cache.*\", component=~\"cache-server\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested", "refId": "B" }, { "expr": "min(app_resource_limit{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"cache.*\", component=~\"cache-server\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { "Limit": "#E24D42", "Requested": "#Ffa500" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 0 }, "id": 19, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (rate(app_cpu_usage_ratio{env=\"$environment\", region=\"$region\", instance=~\"cache.*\", component=~\"cache-server\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" }, { "expr": "min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"cache.*\", component=~\"cache-server\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested", "refId": "B" }, { "expr": "min(app_resource_limit{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"cache.*\", component=~\"cache-server\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 9 }, "id": 20, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (rate(example_service_cache_set_total{env=\"$environment\", region=\"$region\", service=\"api\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Sets", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 9 }, "id": 21, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (rate(example_service_cache_request_total{env=\"$environment\", region=\"$region\", service=\"api\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Requests", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 18 }, "id": 22, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (rate(example_service_cache_request_hit_total{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval]) / rate(example_service_cache_request_total{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Cache Request Hit Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 18 }, "id": 23, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(example_service_cache_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}/request", "refId": "A" }, { "expr": "histogram_quantile(0.99, sum(rate(example_service_cache_set_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}/set", "refId": "B" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Latency p99", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 27 }, "id": 24, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(example_service_cache_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}/request", "refId": "A" }, { "expr": "histogram_quantile(0.90, sum(rate(example_service_cache_set_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}/set", "refId": "B" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Latency p90", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 27 }, "id": 25, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(example_service_cache_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}/request", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(example_service_cache_set_duration_seconds_bucket{env=\"$environment\", region=\"$region\", service=\"api\"}[$__rate_interval])) by (le, method)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{method}}/set", "refId": "B" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Client Latency p50", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 36 }, "id": 26, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(example_service_cache_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Total", "refId": "A" }, { "expr": "sum(rate(example_service_cache_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", method=~\".*Config.*\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Config", "refId": "B" }, { "expr": "sum(rate(example_service_cache_request_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", method!~\".*Config.*\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Other", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Request Throughput (requests/sec)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 36 }, "id": 27, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(example_service_cache_set_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Total", "refId": "A" }, { "expr": "sum(rate(example_service_cache_set_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", method=~\".*Config.*\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Config", "refId": "B" }, { "expr": "sum(rate(example_service_cache_set_duration_seconds_count{env=\"$environment\", region=\"$region\", service=\"api\", method!~\".*Config.*\"}[1m]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Other", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Set Throughput (requests/sec)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Cache", "titleSize": "h6", "type": "row" }, { "collapse": false, "collapsed": false, "panels": [ { "aliasColors": { "Limit": "#E24D42", "Requested": "#Ffa500" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 0 }, "id": 28, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (app_memory_usage_bytes{env=\"$environment\", region=\"$region\", instance=~\"gateway.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" }, { "expr": "min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"gateway.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested", "refId": "B" }, { "expr": "min(app_resource_limit{env=\"$environment\", region=\"$region\", resource=\"memory\", instance=~\"gateway.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { "Limit": "#E24D42", "Requested": "#Ffa500" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 0, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 0 }, "id": 29, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (pod) (rate(app_cpu_usage_ratio{env=\"$environment\", region=\"$region\", instance=~\"gateway.*\", component=~\".*\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", "refId": "A" }, { "expr": "min(app_resource_quota{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"gateway.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested", "refId": "B" }, { "expr": "min(app_resource_limit{env=\"$environment\", region=\"$region\", resource=\"cpu\", instance=~\"gateway.*\", component=~\".*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 0 }, "id": 30, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(agent_management_gateway_grafanacloud_authenticator_authentication_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p99", "refId": "A" }, { "expr": "histogram_quantile(0.90, sum(rate(agent_management_gateway_grafanacloud_authenticator_authentication_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p90", "refId": "B" }, { "expr": "histogram_quantile(0.50, sum(rate(agent_management_gateway_grafanacloud_authenticator_authentication_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p50", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Authenticator latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 9 }, "id": 31, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(agent_management_gateway_grafanacloud_cachedclient_get_instance_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p99", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(agent_management_gateway_grafanacloud_cachedclient_get_instance_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p95", "refId": "B" }, { "expr": "histogram_quantile(0.9, sum(rate(agent_management_gateway_grafanacloud_cachedclient_get_instance_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p90", "refId": "C" }, { "expr": "histogram_quantile(0.5, sum(rate(agent_management_gateway_grafanacloud_cachedclient_get_instance_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p50", "refId": "D" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "GCOM cached client latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 9 }, "id": 32, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(agent_management_request_downstream_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p99", "refId": "A" }, { "expr": "histogram_quantile(0.90, sum(rate(agent_management_request_downstream_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p90", "refId": "B" }, { "expr": "histogram_quantile(0.50, sum(rate(agent_management_request_downstream_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "p50", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Handler latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { "1xx": "#EAB839", "2xx": "#7EB26D", "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 9 }, "id": 33, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (code) (\n label_replace(\n rate(grafanacom_api_request_duration_seconds_count{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval]),\n \"status\",\n \"${1}xx\",\n \"code\",\n \"([0-9])..\")\n )\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{code}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Grafana.com HTTP Client - Throughput (requests/sec)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "decimals": 2, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "decimals": 2, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 18 }, "id": 34, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(grafanacom_api_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "5m", "intervalFactor": 2, "legendFormat": "p99", "refId": "A" }, { "expr": "histogram_quantile(0.90, sum(rate(grafanacom_api_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "5m", "intervalFactor": 2, "legendFormat": "p90", "refId": "B" }, { "expr": "histogram_quantile(0.50, sum(rate(grafanacom_api_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "5m", "intervalFactor": 2, "legendFormat": "p50", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Grafana.com HTTP Client - Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 18 }, "id": 35, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "(rate(grafanacom_api_request_duration_seconds_count{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\", code=~\"5..|error\"}[$__rate_interval]) / rate(grafanacom_api_request_duration_seconds_count{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) OR on() vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Error rate", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Grafana.com HTTP Client - Error rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true } ] }, { "aliasColors": { "1xx": "#EAB839", "2xx": "#7EB26D", "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "decimals": 2, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 18 }, "id": 36, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (status_code) (\n label_replace(\n rate(auth_client_request_duration_seconds_count{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval]),\n \"status\",\n \"${1}xx\",\n \"status_code\",\n \"([0-9])..\")\n )\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{status_code}}", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Auth API HTTP Client - Throughput (requests/sec)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "decimals": 2, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "decimals": 2, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 27 }, "id": 37, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(auth_client_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "5m", "intervalFactor": 2, "legendFormat": "p99", "refId": "A" }, { "expr": "histogram_quantile(0.90, sum(rate(auth_client_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "5m", "intervalFactor": 2, "legendFormat": "p90", "refId": "B" }, { "expr": "histogram_quantile(0.50, sum(rate(auth_client_request_duration_seconds_bucket{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "5m", "intervalFactor": 2, "legendFormat": "p50", "refId": "C" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Auth API HTTP Client - Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "ms", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 27 }, "id": 38, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "(rate(auth_client_request_duration_seconds_count{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\", status_code=~\"5..|error\"}[$__rate_interval]) / rate(auth_client_request_duration_seconds_count{env=\"$environment\", region=\"$region\", job=\"$namespace/gateway\"}[$__rate_interval])) OR on() vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "Error rate", "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Auth API HTTP Client - Error rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Gateway", "titleSize": "h6", "type": "row" } ], "schemaVersion": 14, "style": "dark", "tags": [ "example-service", "as-code" ], "templating": { "list": [ { "current": { "value": "prometheus-datasource" }, "hide": 0, "label": null, "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "current": { "value": "logs-datasource" }, "hide": 0, "label": null, "name": "datasourcelogs", "options": [ ], "query": "loki", "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": null, "current": { "text": "cluster-1", "value": "cluster-1" }, "datasource": "$datasource", "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "cluster", "options": [ ], "query": "label_values(example_service_inflight_requests{}, cluster)", "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "text": "cluster-1", "value": "cluster-1" }, "datasource": "$datasource", "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "namespace", "options": [ ], "query": "label_values(example_service_inflight_requests{cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "example-service.*", "sort": 0, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-2h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Service Overview Dashboard", "uid": "ba9b93e998e09d254266b11192894a8d", "version": 0 }