Dashboards: Backend schema version migration (#99392)
This commit is contained in:
Todd Treece
@@ -0,0 +1,30 @@
|
||||
package migration
|
||||
|
||||
import "github.com/grafana/grafana/pkg/apis/dashboard/migration/schemaversion"
|
||||
|
||||
func Migrate(dash map[string]interface{}, targetVersion int) error {
|
||||
if dash == nil {
|
||||
dash = map[string]interface{}{}
|
||||
}
|
||||
inputVersion := schemaversion.GetSchemaVersion(dash)
|
||||
dash["schemaVersion"] = inputVersion
|
||||
|
||||
if inputVersion < schemaversion.MINIUM_VERSION {
|
||||
return schemaversion.NewMinimumVersionError(inputVersion)
|
||||
}
|
||||
|
||||
for nextVersion := inputVersion + 1; nextVersion <= targetVersion; nextVersion++ {
|
||||
if migration, ok := schemaversion.Migrations[nextVersion]; ok {
|
||||
if err := migration(dash); err != nil {
|
||||
return schemaversion.NewMigrationError("migration failed", inputVersion, nextVersion)
|
||||
}
|
||||
dash["schemaVersion"] = nextVersion
|
||||
}
|
||||
}
|
||||
|
||||
if schemaversion.GetSchemaVersion(dash) != targetVersion {
|
||||
return schemaversion.NewMigrationError("schema version not migrated to target version", inputVersion, targetVersion)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package migration_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apis/dashboard/migration"
|
||||
"github.com/grafana/grafana/pkg/apis/dashboard/migration/schemaversion"
|
||||
)
|
||||
|
||||
const INPUT_DIR = "testdata/input"
|
||||
const OUTPUT_DIR = "testdata/output"
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
files, err := os.ReadDir(INPUT_DIR)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("minimum version check", func(t *testing.T) {
|
||||
err := migration.Migrate(map[string]interface{}{
|
||||
"schemaVersion": schemaversion.MINIUM_VERSION - 1,
|
||||
}, schemaversion.MINIUM_VERSION)
|
||||
|
||||
var minVersionErr = schemaversion.NewMinimumVersionError(schemaversion.MINIUM_VERSION - 1)
|
||||
require.ErrorAs(t, err, &minVersionErr)
|
||||
})
|
||||
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
inputDash, inputVersion, name := load(t, filepath.Join(INPUT_DIR, f.Name()))
|
||||
|
||||
t.Run("input check "+f.Name(), func(t *testing.T) {
|
||||
// use input version as the target version to ensure there are no changes
|
||||
require.NoError(t, migration.Migrate(inputDash, inputVersion), "input check migration failed")
|
||||
outBytes, err := json.MarshalIndent(inputDash, "", " ")
|
||||
require.NoError(t, err, "failed to marshal migrated dashboard")
|
||||
// We can ignore gosec G304 here since it's a test
|
||||
// nolint:gosec
|
||||
expectedDash, err := os.ReadFile(filepath.Join(INPUT_DIR, f.Name()))
|
||||
require.NoError(t, err, "failed to read expected output file")
|
||||
require.JSONEq(t, string(expectedDash), string(outBytes), "%s input check did not match", f.Name())
|
||||
})
|
||||
|
||||
for targetVersion := range schemaversion.Migrations {
|
||||
testName := fmt.Sprintf("%s v%d to v%d", name, inputVersion, targetVersion)
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
testMigration(t, f, targetVersion)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testMigration(t *testing.T, file fs.DirEntry, targetVersion int) {
|
||||
t.Helper()
|
||||
dash, inputVersion, name := load(t, filepath.Join(INPUT_DIR, file.Name()))
|
||||
require.NoError(t, migration.Migrate(dash, targetVersion), "%d migration failed", targetVersion)
|
||||
|
||||
outPath := filepath.Join(OUTPUT_DIR, fmt.Sprintf("%d.%s.%d.json", inputVersion, name, targetVersion))
|
||||
outBytes, err := json.MarshalIndent(dash, "", " ")
|
||||
require.NoError(t, err, "failed to marshal migrated dashboard")
|
||||
|
||||
if _, err := os.Stat(outPath); os.IsNotExist(err) {
|
||||
err = os.WriteFile(outPath, outBytes, 0644)
|
||||
require.NoError(t, err, "failed to write new output file", outPath)
|
||||
return
|
||||
}
|
||||
|
||||
// We can ignore gosec G304 here since it's a test
|
||||
// nolint:gosec
|
||||
existingBytes, err := os.ReadFile(outPath)
|
||||
require.NoError(t, err, "failed to read existing output file")
|
||||
require.JSONEq(t, string(existingBytes), string(outBytes), "%s did not match", outPath)
|
||||
}
|
||||
|
||||
func parseInputName(t *testing.T, name string) (int, string) {
|
||||
t.Helper()
|
||||
parts := strings.SplitN(filepath.Base(name), ".", 3)
|
||||
if len(parts) < 3 {
|
||||
t.Fatalf("invalid input filename: %s", name)
|
||||
}
|
||||
iv, err := strconv.Atoi(parts[0])
|
||||
require.NoError(t, err, "failed to parse input version")
|
||||
return iv, parts[1]
|
||||
}
|
||||
|
||||
func load(t *testing.T, path string) (dash map[string]interface{}, inputVersion int, name string) {
|
||||
// We can ignore gosec G304 here since it's a test
|
||||
// nolint:gosec
|
||||
inputBytes, err := os.ReadFile(path)
|
||||
require.NoError(t, err, "failed to read embedded input file")
|
||||
require.NoError(t, json.Unmarshal(inputBytes, &dash), "failed to unmarshal dashboard JSON")
|
||||
inputVersion, name = parseInputName(t, path)
|
||||
return dash, inputVersion, name
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package schemaversion
|
||||
|
||||
import "fmt"
|
||||
|
||||
var _ error = &MinimumVersionError{}
|
||||
var _ error = &MigrationError{}
|
||||
|
||||
// MinimumVersionError is an error that is returned when the schema version is below the minimum version.
|
||||
func NewMinimumVersionError(inputVersion int) *MinimumVersionError {
|
||||
return &MinimumVersionError{inputVersion: inputVersion}
|
||||
}
|
||||
|
||||
// MinimumVersionError is an error type for minimum version errors.
|
||||
type MinimumVersionError struct {
|
||||
inputVersion int
|
||||
}
|
||||
|
||||
func (e *MinimumVersionError) Error() string {
|
||||
return fmt.Errorf("input schema version is below minimum version. input: %d minimum: %d", e.inputVersion, MINIUM_VERSION).Error()
|
||||
}
|
||||
|
||||
// ErrMigrationFailed is an error that is returned when a migration fails.
|
||||
func NewMigrationError(msg string, currentVersion, targetVersion int) *MigrationError {
|
||||
return &MigrationError{
|
||||
msg: msg,
|
||||
targetVersion: targetVersion,
|
||||
currentVersion: currentVersion,
|
||||
}
|
||||
}
|
||||
|
||||
// MigrationError is an error type for migration errors.
|
||||
type MigrationError struct {
|
||||
msg string
|
||||
targetVersion int
|
||||
currentVersion int
|
||||
}
|
||||
|
||||
func (e *MigrationError) Error() string {
|
||||
return fmt.Errorf("schema migration from version %d to %d failed: %v", e.currentVersion, e.targetVersion, e.msg).Error()
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package schemaversion
|
||||
|
||||
import "strconv"
|
||||
|
||||
type SchemaVersionMigrationFunc func(map[string]interface{}) error
|
||||
|
||||
const (
|
||||
MINIUM_VERSION = 39
|
||||
LATEST_VERSION = 40
|
||||
)
|
||||
|
||||
var Migrations = map[int]SchemaVersionMigrationFunc{
|
||||
40: V40,
|
||||
}
|
||||
|
||||
func GetSchemaVersion(dash map[string]interface{}) int {
|
||||
if v, ok := dash["schemaVersion"]; ok {
|
||||
switch v := v.(type) {
|
||||
case int:
|
||||
return v
|
||||
case float64:
|
||||
return int(v)
|
||||
case string:
|
||||
if version, err := strconv.Atoi(v); err == nil {
|
||||
return version
|
||||
}
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package schemaversion_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apis/dashboard/migration/schemaversion"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGetSchemaVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
dash map[string]interface{}
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
name: "schemaVersion as int",
|
||||
dash: map[string]interface{}{
|
||||
"schemaVersion": 16,
|
||||
},
|
||||
expected: 16,
|
||||
},
|
||||
{
|
||||
name: "schemaVersion as float64",
|
||||
dash: map[string]interface{}{
|
||||
"schemaVersion": 40.2345,
|
||||
},
|
||||
expected: 40,
|
||||
},
|
||||
{
|
||||
name: "schemaVersion is not set",
|
||||
dash: map[string]interface{}{},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "schemaVersion as string int",
|
||||
dash: map[string]interface{}{
|
||||
"schemaVersion": "5",
|
||||
},
|
||||
expected: 5,
|
||||
},
|
||||
{
|
||||
name: "schemaVersion as invalid string",
|
||||
dash: map[string]interface{}{
|
||||
"schemaVersion": "foo",
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := schemaversion.GetSchemaVersion(tt.dash)
|
||||
require.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package schemaversion
|
||||
|
||||
func V40(dash map[string]interface{}) error {
|
||||
dash["schemaVersion"] = int(40)
|
||||
if _, ok := dash["refresh"].(string); !ok {
|
||||
dash["refresh"] = ""
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package schemaversion_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apis/dashboard/migration/schemaversion"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestV40(t *testing.T) {
|
||||
tests := []migrationTestCase{
|
||||
{
|
||||
name: "refresh not set",
|
||||
input: map[string]interface{}{
|
||||
"title": "Test Dashboard",
|
||||
},
|
||||
expected: map[string]interface{}{
|
||||
"title": "Test Dashboard",
|
||||
"schemaVersion": 40,
|
||||
"refresh": "",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "boolean refresh value is converted to an empty string",
|
||||
input: map[string]interface{}{
|
||||
"title": "Test Dashboard",
|
||||
"schemaVersion": 39,
|
||||
"refresh": true,
|
||||
},
|
||||
expected: map[string]interface{}{
|
||||
"title": "Test Dashboard",
|
||||
"schemaVersion": 40,
|
||||
"refresh": "",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "string refresh value is not converted",
|
||||
input: map[string]interface{}{
|
||||
"title": "Test Dashboard",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "1m",
|
||||
},
|
||||
expected: map[string]interface{}{
|
||||
"title": "Test Dashboard",
|
||||
"schemaVersion": 40,
|
||||
"refresh": "1m",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runMigrationTests(t, tests, schemaversion.V40)
|
||||
}
|
||||
|
||||
type migrationTestCase struct {
|
||||
name string
|
||||
input map[string]interface{}
|
||||
expected map[string]interface{}
|
||||
}
|
||||
|
||||
func runMigrationTests(t *testing.T, testCases []migrationTestCase, migrationFunc schemaversion.SchemaVersionMigrationFunc) {
|
||||
t.Helper()
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := migrationFunc(tt.input)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.expected, tt.input)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "datasource",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "11.5.0-81438",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "datasource",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"queryType": "randomWalk",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Panel Title",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "utc",
|
||||
"title": "New dashboard",
|
||||
"version": 0,
|
||||
"weekStart": "",
|
||||
"refresh": true,
|
||||
"schemaVersion": 39
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations \u0026 Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "datasource",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "11.5.0-81438",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "datasource",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"queryType": "randomWalk",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Panel Title",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
"refresh": "",
|
||||
"schemaVersion": 40,
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "utc",
|
||||
"title": "New dashboard",
|
||||
"version": 0,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -1,26 +1,35 @@
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
conversion "k8s.io/apimachinery/pkg/conversion"
|
||||
klog "k8s.io/klog/v2"
|
||||
|
||||
common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/apis/dashboard/migration"
|
||||
"github.com/grafana/grafana/pkg/apis/dashboard/migration/schemaversion"
|
||||
)
|
||||
|
||||
func Convert_v0alpha1_Unstructured_To_v1alpha1_DashboardSpec(in *common.Unstructured, out *DashboardSpec, s conversion.Scope) error {
|
||||
out.Unstructured = *in
|
||||
|
||||
t, ok := in.Object["title"]
|
||||
if !ok {
|
||||
return nil // skip setting the title if it's not in the unstructured object
|
||||
err := migration.Migrate(in.Object, schemaversion.LATEST_VERSION)
|
||||
if err != nil {
|
||||
minErr := &schemaversion.MinimumVersionError{}
|
||||
if errors.As(err, &minErr) {
|
||||
in.Object["__migrationError"] = err.Error()
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
title, ok := t.(string)
|
||||
out.Unstructured = *in
|
||||
|
||||
t, ok := in.Object["title"].(string)
|
||||
if !ok {
|
||||
klog.V(5).Infof("unstructured dashboard title field is not a string %v", t)
|
||||
return nil // skip setting the title if it's not a string in the unstructured object
|
||||
}
|
||||
out.Title = title
|
||||
out.Title = t
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user