Dashboard migrations: Fix span: 0 bug and panel ordering in v16 dashboard migration (#112051)
Fix span: 0 bug and panel ordering in v16 dashboard migration Fix span: 0 handling to match frontend behavior by defaulting to DEFAULT_PANEL_SPAN. Fix panel ordering issue by using stable sort instead of unstable sort. Fix collapsed property handling to only set when input row has collapse property. Add comprehensive test cases for span: 0 bug and collapsed property behavior. Add sanitized test input file for span: 0 demo dashboard with generic values instead of internal Grafana infrastructure references. All backend migration tests and frontend comparison tests pass.
This commit is contained in:
Dominik Prokop
@@ -198,7 +198,7 @@ func sortPanelsByGridPos(dashboard map[string]interface{}) {
|
||||
return
|
||||
}
|
||||
|
||||
sort.Slice(panels, func(i, j int) bool {
|
||||
sort.SliceStable(panels, func(i, j int) bool {
|
||||
panelA := panels[i]
|
||||
panelB := panels[j]
|
||||
|
||||
@@ -831,7 +831,7 @@ func cleanupPanelList(panels []interface{}) {
|
||||
|
||||
// sortPanelsByGridPosition sorts panels by grid position (matches frontend sortPanelsByGridPos behavior)
|
||||
func sortPanelsByGridPosition(panels []interface{}) {
|
||||
sort.Slice(panels, func(i, j int) bool {
|
||||
sort.SliceStable(panels, func(i, j int) bool {
|
||||
panelA, okA := panels[i].(map[string]interface{})
|
||||
panelB, okB := panels[j].(map[string]interface{})
|
||||
if !okA || !okB {
|
||||
|
||||
@@ -49,10 +49,15 @@ func upgradeToGridLayout(dashboard map[string]interface{}) {
|
||||
maxPanelID := getMaxPanelID(rows)
|
||||
nextRowID := maxPanelID + 1
|
||||
|
||||
// Get existing panels
|
||||
var finalPanels []interface{}
|
||||
if existingPanels, ok := dashboard["panels"].([]interface{}); ok {
|
||||
finalPanels = existingPanels
|
||||
// Match frontend: dashboard.panels already exists with top-level panels
|
||||
// The frontend's this.dashboard.panels is initialized in the constructor with existing panels
|
||||
// Then upgradeToGridLayout adds more panels to it
|
||||
|
||||
// Initialize panels array - make a copy to avoid modifying the original
|
||||
panels := []interface{}{}
|
||||
if existingPanels, ok := dashboard["panels"].([]interface{}); ok && len(existingPanels) > 0 {
|
||||
// Copy existing panels to preserve order
|
||||
panels = append(panels, existingPanels...)
|
||||
}
|
||||
|
||||
// Add special "row" panels if even one row is collapsed, repeated or has visible title (line 1028 in TS)
|
||||
@@ -72,7 +77,14 @@ func upgradeToGridLayout(dashboard map[string]interface{}) {
|
||||
|
||||
height := getRowHeight(row)
|
||||
rowGridHeight := getGridHeight(height)
|
||||
isCollapsed := GetBoolValue(row, "collapse")
|
||||
// Check if collapse property exists and get its value
|
||||
collapseValue, hasCollapseProperty := row["collapse"]
|
||||
isCollapsed := false
|
||||
if hasCollapseProperty {
|
||||
if b, ok := collapseValue.(bool); ok {
|
||||
isCollapsed = b
|
||||
}
|
||||
}
|
||||
|
||||
var rowPanel map[string]interface{}
|
||||
|
||||
@@ -110,9 +122,9 @@ func upgradeToGridLayout(dashboard map[string]interface{}) {
|
||||
},
|
||||
}
|
||||
|
||||
// Set collapsed property only if the original row had a collapse property
|
||||
// This matches the frontend behavior: rowPanel.collapsed = row.collapse
|
||||
if _, hasCollapse := row["collapse"]; hasCollapse {
|
||||
// Match frontend behavior: rowPanel.collapsed = row.collapse (line 1065 in TS)
|
||||
// Only set collapsed property if the original row had a collapse property
|
||||
if hasCollapseProperty {
|
||||
rowPanel["collapsed"] = isCollapsed
|
||||
}
|
||||
nextRowID++
|
||||
@@ -128,20 +140,14 @@ func upgradeToGridLayout(dashboard map[string]interface{}) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if panel already has gridPos but no valid span
|
||||
// If span is missing or zero, and gridPos exists, preserve gridPos dimensions
|
||||
var panelWidth, panelHeight int
|
||||
// Match frontend logic: panel.span = panel.span || DEFAULT_PANEL_SPAN (line 1082 in TS)
|
||||
span := GetFloatValue(panel, "span", 0)
|
||||
existingGridPos, hasGridPos := panel["gridPos"].(map[string]interface{})
|
||||
|
||||
if hasGridPos && span == 0 {
|
||||
// Panel already has gridPos but no valid span - preserve its dimensions
|
||||
panelWidth = GetIntValue(existingGridPos, "w", int(defaultPanelSpan*widthFactor))
|
||||
panelHeight = GetIntValue(existingGridPos, "h", rowGridHeight)
|
||||
} else {
|
||||
panelWidth, panelHeight = calculatePanelDimensionsFromSpan(span, panel, widthFactor, rowGridHeight)
|
||||
if span == 0 {
|
||||
span = defaultPanelSpan
|
||||
}
|
||||
|
||||
panelWidth, panelHeight := calculatePanelDimensionsFromSpan(span, panel, widthFactor, rowGridHeight)
|
||||
|
||||
panelPos := rowArea.getPanelPosition(panelHeight, panelWidth)
|
||||
yPos = rowArea.yPos
|
||||
|
||||
@@ -157,21 +163,21 @@ func upgradeToGridLayout(dashboard map[string]interface{}) {
|
||||
// Remove span (line 1080 in TS)
|
||||
delete(panel, "span")
|
||||
|
||||
// Exact logic from lines 1082-1086 in TS
|
||||
// Match frontend logic: lines 1101-1105 in TS
|
||||
if rowPanel != nil && isCollapsed {
|
||||
// Add to collapsed row's nested panels
|
||||
// Add to collapsed row's nested panels (line 1102)
|
||||
if rowPanelPanels, ok := rowPanel["panels"].([]interface{}); ok {
|
||||
rowPanel["panels"] = append(rowPanelPanels, panel)
|
||||
}
|
||||
} else {
|
||||
// Add directly to dashboard panels
|
||||
finalPanels = append(finalPanels, panel)
|
||||
// Add directly to panels array like frontend (line 1104)
|
||||
panels = append(panels, panel)
|
||||
}
|
||||
}
|
||||
|
||||
// Add row panel after processing all panels (lines 1089-1091 in TS)
|
||||
// Add row panel after regular panels from this row (lines 1108-1110 in TS)
|
||||
if rowPanel != nil {
|
||||
finalPanels = append(finalPanels, rowPanel)
|
||||
panels = append(panels, rowPanel)
|
||||
}
|
||||
|
||||
// Update yPos (lines 1093-1095 in TS)
|
||||
@@ -181,7 +187,7 @@ func upgradeToGridLayout(dashboard map[string]interface{}) {
|
||||
}
|
||||
|
||||
// Update the dashboard
|
||||
dashboard["panels"] = finalPanels
|
||||
dashboard["panels"] = panels
|
||||
delete(dashboard, "rows")
|
||||
}
|
||||
|
||||
@@ -313,10 +319,7 @@ func getGridHeight(height float64) int {
|
||||
}
|
||||
|
||||
func calculatePanelDimensionsFromSpan(span float64, panel map[string]interface{}, widthFactor float64, defaultHeight int) (int, int) {
|
||||
// Set default span if still 0
|
||||
if span == 0 {
|
||||
span = defaultPanelSpan
|
||||
}
|
||||
// span should already be normalized by caller (line 1082 in DashboardMigrator.ts)
|
||||
|
||||
if minSpan, hasMinSpan := panel["minSpan"]; hasMinSpan {
|
||||
if minSpanFloat, ok := ConvertToFloat(minSpan); ok && minSpanFloat > 0 {
|
||||
|
||||
@@ -1532,6 +1532,123 @@ func TestV16(t *testing.T) {
|
||||
// rows field should be removed
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should handle span zero by defaulting to DEFAULT_PANEL_SPAN",
|
||||
input: map[string]interface{}{
|
||||
"schemaVersion": 15,
|
||||
"rows": []interface{}{
|
||||
map[string]interface{}{
|
||||
"collapse": false,
|
||||
"showTitle": true, // Need this to create row panel
|
||||
"title": "Test Row",
|
||||
"height": 250,
|
||||
"panels": []interface{}{
|
||||
map[string]interface{}{
|
||||
"id": 1,
|
||||
"type": "graph",
|
||||
"span": 0, // This should be defaulted to 4 (DEFAULT_PANEL_SPAN)
|
||||
},
|
||||
map[string]interface{}{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"span": 6, // Normal span value
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: map[string]interface{}{
|
||||
"schemaVersion": 16,
|
||||
"panels": []interface{}{
|
||||
map[string]interface{}{
|
||||
"id": 1,
|
||||
"type": "graph",
|
||||
"gridPos": map[string]interface{}{
|
||||
"x": 0,
|
||||
"y": 1,
|
||||
"w": 8, // span 0 -> DEFAULT_PANEL_SPAN (4) -> 4 * 2 = 8 width
|
||||
"h": 7, // default height
|
||||
},
|
||||
},
|
||||
map[string]interface{}{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"gridPos": map[string]interface{}{
|
||||
"x": 8, // After first panel
|
||||
"y": 1,
|
||||
"w": 12, // span 6 -> 6 * 2 = 12 width
|
||||
"h": 7, // default height
|
||||
},
|
||||
},
|
||||
// Row panel should be created because showTitle is true
|
||||
map[string]interface{}{
|
||||
"id": 3,
|
||||
"type": "row",
|
||||
"title": "Test Row",
|
||||
"collapsed": false, // Set because input has "collapse": false
|
||||
"repeat": "",
|
||||
"panels": []interface{}{},
|
||||
"gridPos": map[string]interface{}{
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 24,
|
||||
"h": 7,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should not set collapsed property when input row has no collapse property",
|
||||
input: map[string]interface{}{
|
||||
"schemaVersion": 15,
|
||||
"rows": []interface{}{
|
||||
map[string]interface{}{
|
||||
// No "collapse" property in input
|
||||
"showTitle": true,
|
||||
"title": "Test Row",
|
||||
"height": 250,
|
||||
"panels": []interface{}{
|
||||
map[string]interface{}{
|
||||
"id": 1,
|
||||
"type": "graph",
|
||||
"span": 12,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: map[string]interface{}{
|
||||
"schemaVersion": 16,
|
||||
"panels": []interface{}{
|
||||
map[string]interface{}{
|
||||
"id": 1,
|
||||
"type": "graph",
|
||||
"gridPos": map[string]interface{}{
|
||||
"x": 0,
|
||||
"y": 1,
|
||||
"w": 24, // span 12 -> 12 * 2 = 24 width
|
||||
"h": 7, // default height
|
||||
},
|
||||
},
|
||||
// Row panel should be created because showTitle is true
|
||||
map[string]interface{}{
|
||||
"id": 2,
|
||||
"type": "row",
|
||||
"title": "Test Row",
|
||||
// No "collapsed" property because input had no "collapse" property
|
||||
"repeat": "",
|
||||
"panels": []interface{}{},
|
||||
"gridPos": map[string]interface{}{
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 24,
|
||||
"h": 7,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runMigrationTests(t, tests, schemaversion.V16)
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
{
|
||||
"__requires": [
|
||||
{
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"type": "grafana",
|
||||
"version": "8.0.0"
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": false,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"hideControls": false,
|
||||
"links": [
|
||||
{
|
||||
"icon": "external link",
|
||||
"targetBlank": true,
|
||||
"title": "External Documentation",
|
||||
"type": "link",
|
||||
"url": "https://example.com/docs"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"options": {
|
||||
"content": "This dashboard demonstrates various monitoring components for application observability and performance metrics.\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Application Monitoring",
|
||||
"type": "text"
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"rows": [
|
||||
{
|
||||
"collapse": false,
|
||||
"collapsed": false,
|
||||
"height": "250px",
|
||||
"panels": [
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"content": "This service handles background processing tasks for the application system. It manages various types of operations including data synchronization, resource management, and batch processing.\n\nSupported operation types:\n1. Sync: Synchronizes data between different systems\n2. Process: Handles batch data processing tasks\n3. Cleanup: Removes outdated or temporary resources\n4. Update: Applies configuration changes across services\n\nService dependencies:\n- Data API: For reading and writing application data\n- Configuration Service: For managing system settings\n- Queue Service: For handling task scheduling\n- Storage Service: For persistent data management\n- Auth Service: For authentication and authorization\n- Metrics Service: For collecting operational statistics\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"span": 0,
|
||||
"title": "Service Overview",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"content": "Error monitoring helps identify issues in the system. This section displays error logs and success rates for operations.",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"span": 0,
|
||||
"title": "Error Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.95
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 3,
|
||||
"x": 0,
|
||||
"y": 19
|
||||
},
|
||||
"id": 8,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (action) (app_jobs_processed_total{outcome=\"success\", cluster=\"$cluster\", namespace=\"default\"})\n/\nsum by (action) (app_jobs_processed_total{cluster=\"$cluster\", namespace=\"default\"})\n",
|
||||
"legendFormat": "{{action}}"
|
||||
}
|
||||
],
|
||||
"title": "Job Success Rate",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 10,
|
||||
"x": 3,
|
||||
"y": 19
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"enableLogDetails": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{namespace=\"default\", cluster=\"$cluster\", job=\"app-service\"} | logfmt | level=\"error\""
|
||||
}
|
||||
],
|
||||
"title": "Errors",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 11,
|
||||
"x": 13,
|
||||
"y": 19
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"enableLogDetails": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{namespace=\"default\", cluster=\"$cluster\", job=\"app-service\"} | logfmt"
|
||||
}
|
||||
],
|
||||
"title": "All",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"content": "Performance monitoring examines factors that affect system response times, including operation duration, queue lengths, and processing delays. This section provides metrics and traces for performance analysis.\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"span": 0,
|
||||
"title": "Performance Analysis",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Number of concurrent processing threads available for handling operations",
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 5,
|
||||
"x": 0,
|
||||
"y": 31
|
||||
},
|
||||
"id": 12,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(app_worker_threads_active{cluster=\"$cluster\", namespace=\"default\"})",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"title": "Concurrent Job Drivers",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "${tempo}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 19,
|
||||
"x": 5,
|
||||
"y": 31
|
||||
},
|
||||
"id": 13,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"filters": [
|
||||
{
|
||||
"id": "span-name",
|
||||
"operator": "=",
|
||||
"scope": "span",
|
||||
"tag": "name",
|
||||
"value": [
|
||||
"provisioning.sync.process"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "k8s-cluster-name",
|
||||
"operator": "=",
|
||||
"scope": "resource",
|
||||
"tag": "k8s.cluster.name",
|
||||
"value": [
|
||||
"$cluster"
|
||||
]
|
||||
}
|
||||
],
|
||||
"query": "{name=\"app.operation.process\"}",
|
||||
"queryType": "traceqlSearch"
|
||||
}
|
||||
],
|
||||
"title": "Recent Operation Traces",
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Histogram showing p99, p95, p50, and p10 percentiles for job processing duration based on number of resources changed",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 2
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 55
|
||||
},
|
||||
"id": 14,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) > 0",
|
||||
"legendFormat": "{{action}} q0.99 - size {{resources_changed_bucket}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.9, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) > 0",
|
||||
"legendFormat": "{{action}} q0.95 - size {{resources_changed_bucket}}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) > 0",
|
||||
"legendFormat": "{{action}} q0.5 - size {{resources_changed_bucket}}",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) > 0",
|
||||
"legendFormat": "{{action}} q0.1 - size {{resources_changed_bucket}}",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"timeFrom": "7d",
|
||||
"title": "7d avg of job durations",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "reduce",
|
||||
"options": {
|
||||
"mode": "seriesToRows",
|
||||
"reducers": [
|
||||
"mean"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "seriesToRows"
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"renameByName": {
|
||||
"Field": "Type",
|
||||
"Mean": "Avg Duration",
|
||||
"Metric": "Legend",
|
||||
"Value": "Duration"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Histogram showing p99, p95, p50, and p10 percentiles for job processing duration based on number of resources changed",
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 16,
|
||||
"x": 8,
|
||||
"y": 55
|
||||
},
|
||||
"id": 15,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.99 - size {{resources_changed_bucket}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.95 - size {{resources_changed_bucket}}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.5 - size {{resources_changed_bucket}}",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.1 - size {{resources_changed_bucket}}",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"title": "Job Duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Total number of jobs waiting to be processed",
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 4,
|
||||
"x": 0,
|
||||
"y": 65
|
||||
},
|
||||
"id": 16,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "clamp_min(sum(app_operation_queue_size{cluster=\"$cluster\", namespace=\"default\"}), 0)",
|
||||
"legendFormat": "Queue size"
|
||||
}
|
||||
],
|
||||
"title": "Queue Size",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 4,
|
||||
"x": 4,
|
||||
"y": 65
|
||||
},
|
||||
"id": 17,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg(histogram_quantile(0.5, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le)))",
|
||||
"legendFormat": "Queue size"
|
||||
}
|
||||
],
|
||||
"timeFrom": "7d",
|
||||
"title": "7d avg Queue Wait Time",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "How long a job is in the queue before being picked up",
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 16,
|
||||
"x": 8,
|
||||
"y": 65
|
||||
},
|
||||
"id": 18,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.99",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.95",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.5",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.1",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"title": "Queue Wait Time",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 52
|
||||
},
|
||||
"id": 19,
|
||||
"options": {
|
||||
"content": "Resource utilization monitoring for application containers",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"span": 0,
|
||||
"title": "Resource Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 7,
|
||||
"x": 0,
|
||||
"y": 55
|
||||
},
|
||||
"id": 20,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count by (cluster, channel)(label_replace(label_replace(kube_pod_container_info{namespace=\"default\", container=\"app-worker\", pod=~\"app-worker.*\", cluster=~\"$cluster\"}, \"version\", \"$1\", \"image\", \".+:(.+)\"), \"channel\", \"$1\", \"container\", \".+-(.+)\"))",
|
||||
"legendFormat": "{{cluster}}"
|
||||
}
|
||||
],
|
||||
"title": "Running Pod(s)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 8,
|
||||
"x": 7,
|
||||
"y": 55
|
||||
},
|
||||
"id": 21,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_requests{namespace=\"default\", resource=\"memory\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"})",
|
||||
"legendFormat": "Memory Request"
|
||||
},
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_limits{namespace=\"default\", resource=\"memory\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"})",
|
||||
"legendFormat": "Memory Limit"
|
||||
},
|
||||
{
|
||||
"expr": "max(container_memory_usage_bytes{namespace=\"default\",cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"}) by (pod)",
|
||||
"legendFormat": "Container usage {{pod}}"
|
||||
}
|
||||
],
|
||||
"title": "Memory Utilization",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 9,
|
||||
"x": 15,
|
||||
"y": 55
|
||||
},
|
||||
"id": 22,
|
||||
"span": 0,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(irate(container_cpu_usage_seconds_total{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\"}[$__rate_interval])) by (pod, container, cpu)",
|
||||
"legendFormat": "Usage {{pod}}"
|
||||
},
|
||||
{
|
||||
"expr": "sum(irate(container_cpu_cfs_throttled_seconds_total{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\"}[$__rate_interval])) by (pod, container)",
|
||||
"legendFormat": "Throttling {{pod}}"
|
||||
},
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_limits{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\", resource=\"cpu\"})",
|
||||
"legendFormat": "CPU limit"
|
||||
},
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_requests{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\", resource=\"cpu\"})",
|
||||
"legendFormat": "CPU request"
|
||||
}
|
||||
],
|
||||
"title": "CPU Utilization",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"repeat": null,
|
||||
"repeatIteration": null,
|
||||
"repeatRowId": null,
|
||||
"showTitle": true,
|
||||
"title": "Application Service",
|
||||
"titleSize": "h6"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 15,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"as-code"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"value": "prometheus-datasource"
|
||||
},
|
||||
"hide": 0,
|
||||
"label": "Data source",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"value": "prometheus-datasource"
|
||||
},
|
||||
"name": "prom",
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"value": "loki-datasource"
|
||||
},
|
||||
"name": "loki",
|
||||
"query": "loki",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"text": "tempo-datasource",
|
||||
"value": "tempo-datasource"
|
||||
},
|
||||
"name": "tempo",
|
||||
"query": "tempo",
|
||||
"refresh": 1,
|
||||
"regex": ".*tempo.*",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"text": "demo-cluster",
|
||||
"value": "demo-cluster"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"name": "cluster",
|
||||
"query": "label_values(app_worker_threads_active,cluster)",
|
||||
"refresh": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"time_options": [
|
||||
"5m",
|
||||
"15m",
|
||||
"1h",
|
||||
"6h",
|
||||
"12h",
|
||||
"24h",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
]
|
||||
},
|
||||
"timezone": "utc",
|
||||
"title": "Span Zero Demo Dashboard",
|
||||
"uid": "span-zero-demo-dashboard",
|
||||
"version": 0
|
||||
}
|
||||
+881
@@ -0,0 +1,881 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations \u0026 Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"links": [
|
||||
{
|
||||
"icon": "external link",
|
||||
"targetBlank": true,
|
||||
"title": "External Documentation",
|
||||
"type": "link",
|
||||
"url": "https://example.com/docs"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"content": "This dashboard demonstrates various monitoring components for application observability and performance metrics.\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Application Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 23,
|
||||
"panels": [],
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Application Service",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"content": "This service handles background processing tasks for the application system. It manages various types of operations including data synchronization, resource management, and batch processing.\n\nSupported operation types:\n1. Sync: Synchronizes data between different systems\n2. Process: Handles batch data processing tasks\n3. Cleanup: Removes outdated or temporary resources\n4. Update: Applies configuration changes across services\n\nService dependencies:\n- Data API: For reading and writing application data\n- Configuration Service: For managing system settings\n- Queue Service: For handling task scheduling\n- Storage Service: For persistent data management\n- Auth Service: For authentication and authorization\n- Metrics Service: For collecting operational statistics\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Service Overview",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 1
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"content": "Error monitoring helps identify issues in the system. This section displays error logs and success rates for operations.",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Error Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.95
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 1
|
||||
},
|
||||
"id": 8,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "sum by (action) (app_jobs_processed_total{outcome=\"success\", cluster=\"$cluster\", namespace=\"default\"})\n/\nsum by (action) (app_jobs_processed_total{cluster=\"$cluster\", namespace=\"default\"})\n",
|
||||
"legendFormat": "{{action}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Job Success Rate",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"enableLogDetails": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"expr": "{namespace=\"default\", cluster=\"$cluster\", job=\"app-service\"} | logfmt | level=\"error\"",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Errors",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 8
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"enableLogDetails": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"expr": "{namespace=\"default\", cluster=\"$cluster\", job=\"app-service\"} | logfmt",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "All",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 8
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"content": "Performance monitoring examines factors that affect system response times, including operation duration, queue lengths, and processing delays. This section provides metrics and traces for performance analysis.\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Performance Analysis",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Number of concurrent processing threads available for handling operations",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"id": 12,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "max(app_worker_threads_active{cluster=\"$cluster\", namespace=\"default\"})",
|
||||
"instant": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Concurrent Job Drivers",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "${tempo}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 15
|
||||
},
|
||||
"id": 13,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "${tempo}"
|
||||
},
|
||||
"filters": [
|
||||
{
|
||||
"id": "span-name",
|
||||
"operator": "=",
|
||||
"scope": "span",
|
||||
"tag": "name",
|
||||
"value": [
|
||||
"provisioning.sync.process"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "k8s-cluster-name",
|
||||
"operator": "=",
|
||||
"scope": "resource",
|
||||
"tag": "k8s.cluster.name",
|
||||
"value": [
|
||||
"$cluster"
|
||||
]
|
||||
}
|
||||
],
|
||||
"query": "{name=\"app.operation.process\"}",
|
||||
"queryType": "traceqlSearch",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Recent Operation Traces",
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Histogram showing p99, p95, p50, and p10 percentiles for job processing duration based on number of resources changed",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 2
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 15
|
||||
},
|
||||
"id": 14,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.99 - size {{resources_changed_bucket}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.9, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.95 - size {{resources_changed_bucket}}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.5 - size {{resources_changed_bucket}}",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.1 - size {{resources_changed_bucket}}",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"timeFrom": "7d",
|
||||
"title": "7d avg of job durations",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "reduce",
|
||||
"options": {
|
||||
"mode": "seriesToRows",
|
||||
"reducers": [
|
||||
"mean"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "seriesToRows"
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"renameByName": {
|
||||
"Field": "Type",
|
||||
"Mean": "Avg Duration",
|
||||
"Metric": "Legend",
|
||||
"Value": "Duration"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Histogram showing p99, p95, p50, and p10 percentiles for job processing duration based on number of resources changed",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 22
|
||||
},
|
||||
"id": 15,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.99 - size {{resources_changed_bucket}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.95 - size {{resources_changed_bucket}}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.5 - size {{resources_changed_bucket}}",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.1 - size {{resources_changed_bucket}}",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"title": "Job Duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Total number of jobs waiting to be processed",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 22
|
||||
},
|
||||
"id": 16,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "clamp_min(sum(app_operation_queue_size{cluster=\"$cluster\", namespace=\"default\"}), 0)",
|
||||
"legendFormat": "Queue size",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Queue Size",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 22
|
||||
},
|
||||
"id": 17,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "avg(histogram_quantile(0.5, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le)))",
|
||||
"legendFormat": "Queue size",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": "7d",
|
||||
"title": "7d avg Queue Wait Time",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "How long a job is in the queue before being picked up",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 29
|
||||
},
|
||||
"id": 18,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.99",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.95",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.5",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.1",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"title": "Queue Wait Time",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 29
|
||||
},
|
||||
"id": 19,
|
||||
"options": {
|
||||
"content": "Resource utilization monitoring for application containers",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"apiVersion": "v1",
|
||||
"type": "prometheus",
|
||||
"uid": "default-ds-uid"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Resource Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 29
|
||||
},
|
||||
"id": 20,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "count by (cluster, channel)(label_replace(label_replace(kube_pod_container_info{namespace=\"default\", container=\"app-worker\", pod=~\"app-worker.*\", cluster=~\"$cluster\"}, \"version\", \"$1\", \"image\", \".+:(.+)\"), \"channel\", \"$1\", \"container\", \".+-(.+)\"))",
|
||||
"legendFormat": "{{cluster}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Running Pod(s)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
},
|
||||
"id": 21,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "max(kube_pod_container_resource_requests{namespace=\"default\", resource=\"memory\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"})",
|
||||
"legendFormat": "Memory Request",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "max(kube_pod_container_resource_limits{namespace=\"default\", resource=\"memory\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"})",
|
||||
"legendFormat": "Memory Limit",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "max(container_memory_usage_bytes{namespace=\"default\",cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"}) by (pod)",
|
||||
"legendFormat": "Container usage {{pod}}",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Memory Utilization",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 36
|
||||
},
|
||||
"id": 22,
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "sum(irate(container_cpu_usage_seconds_total{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\"}[$__rate_interval])) by (pod, container, cpu)",
|
||||
"legendFormat": "Usage {{pod}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "sum(irate(container_cpu_cfs_throttled_seconds_total{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\"}[$__rate_interval])) by (pod, container)",
|
||||
"legendFormat": "Throttling {{pod}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "max(kube_pod_container_resource_limits{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\", resource=\"cpu\"})",
|
||||
"legendFormat": "CPU limit",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"expr": "max(kube_pod_container_resource_requests{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\", resource=\"cpu\"})",
|
||||
"legendFormat": "CPU request",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "CPU Utilization",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 42,
|
||||
"tags": [
|
||||
"as-code"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"value": "prometheus-datasource"
|
||||
},
|
||||
"hide": 0,
|
||||
"label": "Data source",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"value": "prometheus-datasource"
|
||||
},
|
||||
"name": "prom",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"value": "loki-datasource"
|
||||
},
|
||||
"name": "loki",
|
||||
"options": [],
|
||||
"query": "loki",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"text": "tempo-datasource",
|
||||
"value": "tempo-datasource"
|
||||
},
|
||||
"name": "tempo",
|
||||
"options": [],
|
||||
"query": "tempo",
|
||||
"refresh": 1,
|
||||
"regex": ".*tempo.*",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"text": "demo-cluster",
|
||||
"value": "demo-cluster"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"name": "cluster",
|
||||
"options": [],
|
||||
"query": "label_values(app_worker_threads_active,cluster)",
|
||||
"refresh": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
]
|
||||
},
|
||||
"timezone": "utc",
|
||||
"title": "Span Zero Demo Dashboard",
|
||||
"uid": "span-zero-demo-dashboard",
|
||||
"weekStart": ""
|
||||
}
|
||||
+694
@@ -0,0 +1,694 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations \u0026 Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"links": [
|
||||
{
|
||||
"icon": "external link",
|
||||
"targetBlank": true,
|
||||
"title": "External Documentation",
|
||||
"type": "link",
|
||||
"url": "https://example.com/docs"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"content": "This dashboard demonstrates various monitoring components for application observability and performance metrics.\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Application Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 23,
|
||||
"panels": [],
|
||||
"title": "Application Service",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"content": "This service handles background processing tasks for the application system. It manages various types of operations including data synchronization, resource management, and batch processing.\n\nSupported operation types:\n1. Sync: Synchronizes data between different systems\n2. Process: Handles batch data processing tasks\n3. Cleanup: Removes outdated or temporary resources\n4. Update: Applies configuration changes across services\n\nService dependencies:\n- Data API: For reading and writing application data\n- Configuration Service: For managing system settings\n- Queue Service: For handling task scheduling\n- Storage Service: For persistent data management\n- Auth Service: For authentication and authorization\n- Metrics Service: For collecting operational statistics\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Service Overview",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 1
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"content": "Error monitoring helps identify issues in the system. This section displays error logs and success rates for operations.",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Error Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.95
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 1
|
||||
},
|
||||
"id": 8,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (action) (app_jobs_processed_total{outcome=\"success\", cluster=\"$cluster\", namespace=\"default\"})\n/\nsum by (action) (app_jobs_processed_total{cluster=\"$cluster\", namespace=\"default\"})\n",
|
||||
"legendFormat": "{{action}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Job Success Rate",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"enableLogDetails": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{namespace=\"default\", cluster=\"$cluster\", job=\"app-service\"} | logfmt | level=\"error\"",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Errors",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "${loki}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 8
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"enableLogDetails": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{namespace=\"default\", cluster=\"$cluster\", job=\"app-service\"} | logfmt",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "All",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 8
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"content": "Performance monitoring examines factors that affect system response times, including operation duration, queue lengths, and processing delays. This section provides metrics and traces for performance analysis.\n",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Performance Analysis",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Number of concurrent processing threads available for handling operations",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"id": 12,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(app_worker_threads_active{cluster=\"$cluster\", namespace=\"default\"})",
|
||||
"instant": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Concurrent Job Drivers",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "${tempo}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 15
|
||||
},
|
||||
"id": 13,
|
||||
"targets": [
|
||||
{
|
||||
"filters": [
|
||||
{
|
||||
"id": "span-name",
|
||||
"operator": "=",
|
||||
"scope": "span",
|
||||
"tag": "name",
|
||||
"value": [
|
||||
"provisioning.sync.process"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "k8s-cluster-name",
|
||||
"operator": "=",
|
||||
"scope": "resource",
|
||||
"tag": "k8s.cluster.name",
|
||||
"value": [
|
||||
"$cluster"
|
||||
]
|
||||
}
|
||||
],
|
||||
"query": "{name=\"app.operation.process\"}",
|
||||
"queryType": "traceqlSearch",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Recent Operation Traces",
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Histogram showing p99, p95, p50, and p10 percentiles for job processing duration based on number of resources changed",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 2
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 15
|
||||
},
|
||||
"id": 14,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.99 - size {{resources_changed_bucket}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.9, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.95 - size {{resources_changed_bucket}}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.5 - size {{resources_changed_bucket}}",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le, resources_changed_bucket, action)) and on(resources_changed_bucket, action) sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (resources_changed_bucket, action) \u003e 0",
|
||||
"legendFormat": "{{action}} q0.1 - size {{resources_changed_bucket}}",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"timeFrom": "7d",
|
||||
"title": "7d avg of job durations",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "reduce",
|
||||
"options": {
|
||||
"mode": "seriesToRows",
|
||||
"reducers": [
|
||||
"mean"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "seriesToRows"
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"renameByName": {
|
||||
"Field": "Type",
|
||||
"Mean": "Avg Duration",
|
||||
"Metric": "Legend",
|
||||
"Value": "Duration"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Histogram showing p99, p95, p50, and p10 percentiles for job processing duration based on number of resources changed",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 22
|
||||
},
|
||||
"id": 15,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.99 - size {{resources_changed_bucket}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.95 - size {{resources_changed_bucket}}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.5 - size {{resources_changed_bucket}}",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[5m])) by (le, resources_changed_bucket, action))",
|
||||
"legendFormat": "{{action}} q0.1 - size {{resources_changed_bucket}}",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"title": "Job Duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "Total number of jobs waiting to be processed",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 22
|
||||
},
|
||||
"id": 16,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "clamp_min(sum(app_operation_queue_size{cluster=\"$cluster\", namespace=\"default\"}), 0)",
|
||||
"legendFormat": "Queue size",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Queue Size",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 22
|
||||
},
|
||||
"id": 17,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg(histogram_quantile(0.5, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[7d])) by (le)))",
|
||||
"legendFormat": "Queue size",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": "7d",
|
||||
"title": "7d avg Queue Wait Time",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"description": "How long a job is in the queue before being picked up",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 29
|
||||
},
|
||||
"id": 18,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.99",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.95",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.5",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.1, sum(rate(app_operation_queue_wait_seconds_bucket{cluster=\"$cluster\", namespace=\"default\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "q0.1",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"title": "Queue Wait Time",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 29
|
||||
},
|
||||
"id": 19,
|
||||
"options": {
|
||||
"content": "Resource utilization monitoring for application containers",
|
||||
"mode": "markdown"
|
||||
},
|
||||
"title": "Resource Monitoring",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 29
|
||||
},
|
||||
"id": 20,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count by (cluster, channel)(label_replace(label_replace(kube_pod_container_info{namespace=\"default\", container=\"app-worker\", pod=~\"app-worker.*\", cluster=~\"$cluster\"}, \"version\", \"$1\", \"image\", \".+:(.+)\"), \"channel\", \"$1\", \"container\", \".+-(.+)\"))",
|
||||
"legendFormat": "{{cluster}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Running Pod(s)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
},
|
||||
"id": 21,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_requests{namespace=\"default\", resource=\"memory\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"})",
|
||||
"legendFormat": "Memory Request",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_limits{namespace=\"default\", resource=\"memory\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"})",
|
||||
"legendFormat": "Memory Limit",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "max(container_memory_usage_bytes{namespace=\"default\",cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker.*\"}) by (pod)",
|
||||
"legendFormat": "Container usage {{pod}}",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Memory Utilization",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 36
|
||||
},
|
||||
"id": 22,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(irate(container_cpu_usage_seconds_total{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\"}[$__rate_interval])) by (pod, container, cpu)",
|
||||
"legendFormat": "Usage {{pod}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(irate(container_cpu_cfs_throttled_seconds_total{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\"}[$__rate_interval])) by (pod, container)",
|
||||
"legendFormat": "Throttling {{pod}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_limits{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\", resource=\"cpu\"})",
|
||||
"legendFormat": "CPU limit",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "max(kube_pod_container_resource_requests{namespace=\"default\", cluster=~\"$cluster\", container=\"app-worker\", pod=~\"app-worker-.*\", resource=\"cpu\"})",
|
||||
"legendFormat": "CPU request",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"title": "CPU Utilization",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 16,
|
||||
"tags": [
|
||||
"as-code"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"value": "prometheus-datasource"
|
||||
},
|
||||
"hide": 0,
|
||||
"label": "Data source",
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"value": "prometheus-datasource"
|
||||
},
|
||||
"name": "prom",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"value": "loki-datasource"
|
||||
},
|
||||
"name": "loki",
|
||||
"options": [],
|
||||
"query": "loki",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"text": "tempo-datasource",
|
||||
"value": "tempo-datasource"
|
||||
},
|
||||
"name": "tempo",
|
||||
"options": [],
|
||||
"query": "tempo",
|
||||
"refresh": 1,
|
||||
"regex": ".*tempo.*",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"text": "demo-cluster",
|
||||
"value": "demo-cluster"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${prom}"
|
||||
},
|
||||
"name": "cluster",
|
||||
"options": [],
|
||||
"query": "label_values(app_worker_threads_active,cluster)",
|
||||
"refresh": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"time_options": [
|
||||
"5m",
|
||||
"15m",
|
||||
"1h",
|
||||
"6h",
|
||||
"12h",
|
||||
"24h",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
]
|
||||
},
|
||||
"timezone": "utc",
|
||||
"title": "Span Zero Demo Dashboard",
|
||||
"uid": "span-zero-demo-dashboard",
|
||||
"weekStart": ""
|
||||
}
|
||||
Reference in New Issue
Block a user