Dashboard Migrations: V6 pulldowns to annotations; variables (#112221)

* migrate to v19

* migrate to v18

* Migration to be verified: v17 Convert minSpan to maxPerRow in panels

* Migration to be verified: 16 Grid layout migration

* Refactor v17 and v19 migrations to use shared helper functions

* Migration to be verified: 15 No-op migration for schema consistency

* Migration to be verified: 14 Shared crosshair to graph tooltip migration

* cleanup

* wip

* complete migration

* fix lint issues

* refactor and test with minimal graph config

* update tests

* migrate to v12

* extract defaults outside the func

* lint

* lint

* add missing showValues prop

* migrate to v11

* migrate to v10

* add test files

* update

* migrate to v9

* migrate to v8

* add context and fix latest version

* add context

* add context

* generate snapshots

* v13 should be no-op

* clean up

* fix tests

* add context

* snapshots

* generate snapshots

* update

* snapshots

* wip

* fix test

* remove nav when cleaning up defaults

* migrate to v6

* update comments

* remove v28

* remove singlestat migraiton from frontend migrator because this is an automigration

* remove unused function

* Remove v24 table plugin logic

* cleanup

* remove plugin version for automigrate as it was used only in v24 and v28 that have been removed

* cleanup

* update snapshot

* update snapshot

* update snapshot

* update snapshot

* remove test

---------

Co-authored-by: Dominik Prokop <dominik.prokop@grafana.com>
This commit is contained in:
Haris Rozajac
2025-10-14 07:16:34 -06:00
committed by GitHub
parent 8a827b5b05
commit 394922c4c8
9 changed files with 1033 additions and 21 deletions
@@ -305,23 +305,6 @@ func TestConversionMetrics(t *testing.T) {
expectedSourceSchema: "v2alpha1",
expectedTargetSchema: "v2beta1",
},
{
name: "v0 to v1 conversion with minimum version error (succeeds but marks failed)",
source: &dashv0.Dashboard{
ObjectMeta: metav1.ObjectMeta{UID: "test-uid-4"},
Spec: common.Unstructured{Object: map[string]any{
"title": "old dashboard",
"schemaVersion": 5, // Below minimum version (13)
}},
},
target: &dashv1.Dashboard{},
expectAPISuccess: true, // Conversion succeeds but status indicates failure
expectMetricsSuccess: false,
expectedSourceAPI: dashv0.APIVERSION,
expectedTargetAPI: dashv1.APIVERSION,
expectedSourceSchema: "5",
expectedTargetSchema: fmt.Sprintf("%d", 41), // LATEST_VERSION
},
}
for _, tt := range tests {
@@ -66,7 +66,6 @@ func applyFrontendDefaults(dashboard map[string]interface{}) {
sortPanelsByGridPos(dashboard)
// Built-in components
addBuiltInAnnotationQuery(dashboard)
initMeta(dashboard)
// Variable cleanup
@@ -1053,7 +1052,8 @@ func cleanupDashboardDefaults(dashboard map[string]interface{}) {
// These properties are lost during frontend's property copying loop in getSaveModelCloneOld()
delete(dashboard, "preload") // Transient dashboard loading state
delete(dashboard, "iteration") // Template variable iteration timestamp
delete(dashboard, "nav")
delete(dashboard, "nav") // Removed after V7 migration
delete(dashboard, "pulldowns") // Removed after V6 migration - frontend doesn't have this property
}
// cleanupFieldConfigDefaults removes properties that frontend considers as defaults and omits
+5 -1
View File
@@ -126,7 +126,11 @@ func (m *migrator) migrate(ctx context.Context, dash map[string]interface{}, tar
}
}
// 6. Clean up the dashboard to match frontend getSaveModel behavior
// 6. Add built-in annotation query after all migrations are complete
// This matches the frontend DashboardModel constructor behavior
addBuiltInAnnotationQuery(dash)
// 7. Clean up the dashboard to match frontend getSaveModel behavior
// This removes properties that shouldn't be persisted and filters out default values
cleanupDashboardForSave(dash)
@@ -7,7 +7,7 @@ import (
)
const (
MIN_VERSION = 6
MIN_VERSION = 5
LATEST_VERSION = 42
)
@@ -35,6 +35,7 @@ type PanelPluginInfo struct {
func GetMigrations(dsInfoProvider DataSourceInfoProvider) map[int]SchemaVersionMigrationFunc {
return map[int]SchemaVersionMigrationFunc{
6: V6,
7: V7,
8: V8,
9: V9,
@@ -0,0 +1,100 @@
package schemaversion
import "context"
// V6 migration handles the pulldowns to annotations conversion and template variable updates.
// This migration moves annotations from the legacy pulldowns array to the new annotations structure
// and updates template variables to use the new schema format.
//
// Background:
// In earlier versions, dashboards used a "pulldowns" property array to store various UI elements
// including annotations. This migration extracts annotations from pulldowns and creates the new
// annotations structure. It also updates template variables to ensure proper datasource handling
// and type normalization.
//
// Example before migration:
// {
// "schemaVersion": 5,
// "pulldowns": [
// { "type": "filtering", "enable": true },
// { "type": "annotations", "enable": true, "annotations": [{"name": "old"}] }
// ],
// "templating": {
// "list": [
// { "name": "server", "type": "filter" },
// { "name": "metric", "datasource": undefined, "allFormat": undefined }
// ]
// }
// }
//
// Example after migration:
// {
// "schemaVersion": 6,
// "annotations": {
// "list": [{"name": "old"}]
// },
// "templating": {
// "list": [
// { "name": "server", "type": "query", "datasource": null },
// { "name": "metric", "type": "query", "datasource": null }
// ]
// }
// }
func V6(_ context.Context, dashboard map[string]interface{}) error {
dashboard["schemaVersion"] = 6
// Move drop-downs to new schema (matches frontend DashboardMigrator logic)
// Find annotations in pulldowns array: equivalent to find(old.pulldowns, { type: 'annotations' })
if pulldowns, ok := dashboard["pulldowns"].([]interface{}); ok {
for _, pulldownInterface := range pulldowns {
if pulldown, ok := pulldownInterface.(map[string]interface{}); ok {
if pulldownType, exists := pulldown["type"]; exists && pulldownType == "annotations" {
// Found annotations pulldown, extract annotations
if annotations, hasAnnotations := pulldown["annotations"]; hasAnnotations {
dashboard["annotations"] = map[string]interface{}{
"list": annotations,
}
} else {
// If no annotations property, create empty list
dashboard["annotations"] = map[string]interface{}{
"list": []interface{}{},
}
}
break // Found what we're looking for, no need to continue
}
}
}
}
// Update template variables
if templating, ok := dashboard["templating"].(map[string]interface{}); ok {
if list, ok := templating["list"].([]interface{}); ok {
for _, variableInterface := range list {
if variable, ok := variableInterface.(map[string]interface{}); ok {
// If datasource is undefined/missing, set to null
if _, exists := variable["datasource"]; !exists {
variable["datasource"] = nil
}
// Convert 'filter' type to 'query'
if varType, exists := variable["type"]; exists && varType == "filter" {
variable["type"] = "query"
}
// If type is undefined/missing, set to 'query'
if _, exists := variable["type"]; !exists {
variable["type"] = "query"
}
// Remove allFormat if it's undefined
if allFormat, exists := variable["allFormat"]; exists && allFormat == nil {
delete(variable, "allFormat")
}
}
}
}
}
return nil
}
@@ -0,0 +1,541 @@
package schemaversion_test
import (
"testing"
"github.com/grafana/grafana/apps/dashboard/pkg/migration/schemaversion"
)
func TestV6Migration(t *testing.T) {
testCases := []migrationTestCase{
{
name: "pulldowns to annotations conversion with existing annotations",
input: map[string]interface{}{
"schemaVersion": 5,
"pulldowns": []interface{}{
map[string]interface{}{
"type": "filtering",
"enable": true,
},
map[string]interface{}{
"type": "annotations",
"enable": true,
"annotations": []interface{}{
map[string]interface{}{
"name": "old annotation",
"datasource": "prometheus",
"enable": true,
},
},
},
},
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "filter",
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"pulldowns": []interface{}{
map[string]interface{}{
"type": "filtering",
"enable": true,
},
map[string]interface{}{
"type": "annotations",
"enable": true,
"annotations": []interface{}{
map[string]interface{}{
"name": "old annotation",
"datasource": "prometheus",
"enable": true,
},
},
},
},
"annotations": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "old annotation",
"datasource": "prometheus",
"enable": true,
},
},
},
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"datasource": nil,
},
},
},
},
},
{
name: "pulldowns to annotations conversion with empty annotations",
input: map[string]interface{}{
"schemaVersion": 5,
"pulldowns": []interface{}{
map[string]interface{}{
"type": "annotations",
"enable": true,
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"pulldowns": []interface{}{
map[string]interface{}{
"type": "annotations",
"enable": true,
},
},
"annotations": map[string]interface{}{
"list": []interface{}{},
},
},
},
{
name: "no annotations pulldown found",
input: map[string]interface{}{
"schemaVersion": 5,
"pulldowns": []interface{}{
map[string]interface{}{
"type": "filtering",
"enable": true,
},
map[string]interface{}{
"type": "other",
"enable": false,
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"pulldowns": []interface{}{
map[string]interface{}{
"type": "filtering",
"enable": true,
},
map[string]interface{}{
"type": "other",
"enable": false,
},
},
},
},
{
name: "no pulldowns property",
input: map[string]interface{}{
"schemaVersion": 5,
"title": "Test Dashboard",
},
expected: map[string]interface{}{
"schemaVersion": 6,
"title": "Test Dashboard",
},
},
{
name: "empty pulldowns array",
input: map[string]interface{}{
"schemaVersion": 5,
"pulldowns": []interface{}{},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"pulldowns": []interface{}{},
},
},
{
name: "template variables migration - filter to query type",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "filter",
},
map[string]interface{}{
"name": "metric",
"type": "query",
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"datasource": nil,
},
map[string]interface{}{
"name": "metric",
"type": "query",
"datasource": nil,
},
},
},
},
},
{
name: "template variables migration - missing type becomes query",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"datasource": nil,
},
},
},
},
},
{
name: "template variables migration - existing datasource preserved",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"datasource": "prometheus",
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"datasource": "prometheus",
},
},
},
},
},
{
name: "template variables migration - allFormat removal",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"allFormat": nil,
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"datasource": nil,
},
},
},
},
},
{
name: "template variables migration - allFormat with value preserved",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"allFormat": "glob",
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "server",
"type": "query",
"datasource": nil,
"allFormat": "glob",
},
},
},
},
},
{
name: "no templating property",
input: map[string]interface{}{
"schemaVersion": 5,
"title": "Test Dashboard",
},
expected: map[string]interface{}{
"schemaVersion": 6,
"title": "Test Dashboard",
},
},
{
name: "empty templating list",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": []interface{}{},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": []interface{}{},
},
},
},
{
name: "complex dashboard with both pulldowns and templating",
input: map[string]interface{}{
"schemaVersion": 5,
"title": "Complex Dashboard",
"pulldowns": []interface{}{
map[string]interface{}{
"type": "filtering",
"enable": true,
},
map[string]interface{}{
"type": "annotations",
"enable": true,
"annotations": []interface{}{
map[string]interface{}{
"name": "deployment",
"datasource": "prometheus",
"enable": true,
"iconColor": "red",
},
map[string]interface{}{
"name": "alerts",
"datasource": "loki",
"enable": false,
},
},
},
},
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "environment",
"type": "filter",
"allFormat": nil,
},
map[string]interface{}{
"name": "service",
"datasource": "prometheus",
"allFormat": "glob",
},
map[string]interface{}{
"name": "region",
"type": "custom",
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"title": "Complex Dashboard",
"pulldowns": []interface{}{
map[string]interface{}{
"type": "filtering",
"enable": true,
},
map[string]interface{}{
"type": "annotations",
"enable": true,
"annotations": []interface{}{
map[string]interface{}{
"name": "deployment",
"datasource": "prometheus",
"enable": true,
"iconColor": "red",
},
map[string]interface{}{
"name": "alerts",
"datasource": "loki",
"enable": false,
},
},
},
},
"annotations": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "deployment",
"datasource": "prometheus",
"enable": true,
"iconColor": "red",
},
map[string]interface{}{
"name": "alerts",
"datasource": "loki",
"enable": false,
},
},
},
"templating": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "environment",
"type": "query",
"datasource": nil,
},
map[string]interface{}{
"name": "service",
"type": "query",
"datasource": "prometheus",
"allFormat": "glob",
},
map[string]interface{}{
"name": "region",
"type": "custom",
"datasource": nil,
},
},
},
},
},
{
name: "invalid pulldowns structure",
input: map[string]interface{}{
"schemaVersion": 5,
"pulldowns": "invalid_structure",
},
expected: map[string]interface{}{
"schemaVersion": 6,
"pulldowns": "invalid_structure",
},
},
{
name: "invalid templating structure",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": "invalid_structure",
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": "invalid_structure",
},
},
{
name: "invalid templating list structure",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": "invalid_list",
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": "invalid_list",
},
},
},
{
name: "pulldown item with invalid structure",
input: map[string]interface{}{
"schemaVersion": 5,
"pulldowns": []interface{}{
"invalid_pulldown",
map[string]interface{}{
"type": "annotations",
"enable": true,
"annotations": []interface{}{
map[string]interface{}{
"name": "valid annotation",
},
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"pulldowns": []interface{}{
"invalid_pulldown",
map[string]interface{}{
"type": "annotations",
"enable": true,
"annotations": []interface{}{
map[string]interface{}{
"name": "valid annotation",
},
},
},
},
"annotations": map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "valid annotation",
},
},
},
},
},
{
name: "template variable with invalid structure",
input: map[string]interface{}{
"schemaVersion": 5,
"templating": map[string]interface{}{
"list": []interface{}{
"invalid_variable",
map[string]interface{}{
"name": "valid_variable",
"type": "filter",
},
},
},
},
expected: map[string]interface{}{
"schemaVersion": 6,
"templating": map[string]interface{}{
"list": []interface{}{
"invalid_variable",
map[string]interface{}{
"name": "valid_variable",
"type": "query",
"datasource": nil,
},
},
},
},
},
}
runMigrationTests(t, testCases, schemaversion.V6)
}
@@ -0,0 +1,106 @@
{
"title": "V6 Pulldowns and Template Variables Migration Test",
"schemaVersion": 5,
"pulldowns": [
{
"type": "filtering",
"enable": true
},
{
"type": "annotations",
"enable": true,
"annotations": [
{
"name": "deployment",
"datasource": "prometheus",
"enable": true,
"iconColor": "red",
"query": "ALERTS{alertname=\"DeploymentStarted\"}"
},
{
"name": "alerts",
"datasource": "loki",
"enable": false,
"iconColor": "yellow",
"query": "{job=\"alertmanager\"}"
}
]
}
],
"templating": {
"list": [
{
"name": "environment",
"type": "filter",
"allFormat": null
},
{
"name": "service",
"datasource": "prometheus",
"allFormat": "glob"
},
{
"name": "region",
"type": "custom",
"options": [
{
"text": "us-east-1",
"value": "us-east-1"
},
{
"text": "us-west-2",
"value": "us-west-2"
}
]
},
{
"name": "instance",
"type": "query",
"datasource": "prometheus",
"query": "label_values(up, instance)",
"refresh": 1
}
]
},
"panels": [
{
"id": 1,
"title": "CPU Usage",
"type": "graph",
"targets": [
{
"expr": "cpu_usage{environment=\"$environment\", service=\"$service\"}",
"refId": "A"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
}
},
{
"id": 2,
"title": "Memory Usage",
"type": "singlestat",
"targets": [
{
"expr": "memory_usage{region=\"$region\"}",
"refId": "B"
}
],
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
}
}
],
"time": {
"from": "now-1h",
"to": "now"
},
"refresh": "30s"
}
@@ -0,0 +1,154 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
},
{
"datasource": {
"uid": "prometheus"
},
"enable": true,
"iconColor": "red",
"name": "deployment",
"query": "ALERTS{alertname=\"DeploymentStarted\"}"
},
{
"datasource": {
"uid": "loki"
},
"enable": false,
"iconColor": "yellow",
"name": "alerts",
"query": "{job=\"alertmanager\"}"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"panels": [
{
"autoMigrateFrom": "graph",
"datasource": {
"apiVersion": "v1",
"type": "prometheus",
"uid": "default-ds-uid"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"targets": [
{
"datasource": {
"apiVersion": "v1",
"type": "prometheus",
"uid": "default-ds-uid"
},
"expr": "cpu_usage{environment=\"$environment\", service=\"$service\"}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"autoMigrateFrom": "singlestat",
"datasource": {
"apiVersion": "v1",
"type": "prometheus",
"uid": "default-ds-uid"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"targets": [
{
"datasource": {
"apiVersion": "v1",
"type": "prometheus",
"uid": "default-ds-uid"
},
"expr": "memory_usage{region=\"$region\"}",
"refId": "B"
}
],
"title": "Memory Usage",
"type": "stat"
}
],
"refresh": "30s",
"schemaVersion": 42,
"tags": [],
"templating": {
"list": [
{
"datasource": {
"apiVersion": "v1",
"type": "prometheus",
"uid": "default-ds-uid"
},
"name": "environment",
"options": [],
"refresh": 1,
"type": "query"
},
{
"allFormat": "glob",
"datasource": "prometheus",
"name": "service",
"options": [],
"refresh": 1,
"type": "query"
},
{
"name": "region",
"options": [
{
"text": "us-east-1",
"value": "us-east-1"
},
{
"text": "us-west-2",
"value": "us-west-2"
}
],
"type": "custom"
},
{
"datasource": "prometheus",
"name": "instance",
"options": [],
"query": "label_values(up, instance)",
"refresh": 1,
"type": "query"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "V6 Pulldowns and Template Variables Migration Test",
"weekStart": ""
}
@@ -0,0 +1,123 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
},
{
"datasource": "prometheus",
"enable": true,
"iconColor": "red",
"name": "deployment",
"query": "ALERTS{alertname=\"DeploymentStarted\"}"
},
{
"datasource": "loki",
"enable": false,
"iconColor": "yellow",
"name": "alerts",
"query": "{job=\"alertmanager\"}"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"panels": [
{
"autoMigrateFrom": "graph",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"targets": [
{
"expr": "cpu_usage{environment=\"$environment\", service=\"$service\"}",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "timeseries"
},
{
"autoMigrateFrom": "singlestat",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"targets": [
{
"expr": "memory_usage{region=\"$region\"}",
"refId": "B"
}
],
"title": "Memory Usage",
"type": "stat"
}
],
"refresh": "30s",
"schemaVersion": 6,
"tags": [],
"templating": {
"list": [
{
"name": "environment",
"options": [],
"type": "query"
},
{
"allFormat": "glob",
"datasource": "prometheus",
"name": "service",
"options": [],
"type": "query"
},
{
"name": "region",
"options": [
{
"text": "us-east-1",
"value": "us-east-1"
},
{
"text": "us-west-2",
"value": "us-west-2"
}
],
"type": "custom"
},
{
"datasource": "prometheus",
"name": "instance",
"options": [],
"query": "label_values(up, instance)",
"refresh": 1,
"type": "query"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "V6 Pulldowns and Template Variables Migration Test",
"weekStart": ""
}