Storage: Add command line tool to migrate legacy dashboards (and folders) to unified storage (#99199)

2025-02-11 19:57:46 +02:00
parent b6ea06f259
commit a5355fd66c
36 changed files with 3569 additions and 614 deletions
@@ -144,6 +144,11 @@ var adminCommands = []*cli.Command{
 				Usage:  "Migrates passwords from unsecured fields to secure_json_data field. Return ok unless there is an error. Safe to execute multiple times.",
 				Action: runDbCommand(datamigrations.EncryptDatasourcePasswords),
 			},
+			{
+				Name:   "to-unified-storage",
+				Usage:  "Migrates classic SQL data into unified storage",
+				Action: runDbCommand(datamigrations.ToUnifiedStorage),
+			},
 		},
 	},
 	{
@@ -0,0 +1,70 @@
+package datamigrations
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/services/provisioning"
+	"github.com/grafana/grafana/pkg/services/provisioning/dashboards"
+)
+
+var (
+	_ provisioning.ProvisioningService = (*stubProvisioning)(nil)
+)
+
+func newStubProvisioning(path string) (provisioning.ProvisioningService, error) {
+	cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards"))
+	if err != nil {
+		return nil, err
+	}
+	stub := &stubProvisioning{
+		path: make(map[string]string),
+	}
+	for _, cfg := range cfgs {
+		stub.path[cfg.Name] = cfg.Options["path"].(string)
+	}
+	return &stubProvisioning{}, nil
+}
+
+type stubProvisioning struct {
+	path map[string]string // name > options.path
+}
+
+// GetAllowUIUpdatesFromConfig implements provisioning.ProvisioningService.
+func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool {
+	return false
+}
+
+func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string {
+	return s.path[name]
+}
+
+// ProvisionAlerting implements provisioning.ProvisioningService.
+func (s *stubProvisioning) ProvisionAlerting(ctx context.Context) error {
+	panic("unimplemented")
+}
+
+// ProvisionDashboards implements provisioning.ProvisioningService.
+func (s *stubProvisioning) ProvisionDashboards(ctx context.Context) error {
+	panic("unimplemented")
+}
+
+// ProvisionDatasources implements provisioning.ProvisioningService.
+func (s *stubProvisioning) ProvisionDatasources(ctx context.Context) error {
+	panic("unimplemented")
+}
+
+// ProvisionPlugins implements provisioning.ProvisioningService.
+func (s *stubProvisioning) ProvisionPlugins(ctx context.Context) error {
+	panic("unimplemented")
+}
+
+// Run implements provisioning.ProvisioningService.
+func (s *stubProvisioning) Run(ctx context.Context) error {
+	panic("unimplemented")
+}
+
+// RunInitProvisioners implements provisioning.ProvisioningService.
+func (s *stubProvisioning) RunInitProvisioners(ctx context.Context) error {
+	panic("unimplemented")
+}
@@ -0,0 +1,207 @@
+package datamigrations
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	dashboard "github.com/grafana/grafana/pkg/apis/dashboard"
+	folders "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
+
+	authlib "github.com/grafana/authlib/types"
+	"github.com/grafana/grafana/pkg/apimachinery/identity"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/storage/legacysql"
+	"github.com/grafana/grafana/pkg/storage/unified"
+	"github.com/grafana/grafana/pkg/storage/unified/parquet"
+	"github.com/grafana/grafana/pkg/storage/unified/resource"
+)
+
+// ToUnifiedStorage converts dashboards+folders into unified storage
+func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) error {
+	namespace := "default" // TODO... from command line
+	ns, err := authlib.ParseNamespace(namespace)
+	if err != nil {
+		return err
+	}
+	ctx := identity.WithServiceIdentityContext(context.Background(), ns.OrgID)
+	start := time.Now()
+	last := time.Now()
+
+	opts := legacy.MigrateOptions{
+		Namespace: namespace,
+		Resources: []schema.GroupResource{
+			{Group: folders.GROUP, Resource: folders.RESOURCE},
+			{Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE},
+			{Group: dashboard.GROUP, Resource: dashboard.LIBRARY_PANEL_RESOURCE},
+		},
+		LargeObjects: nil, // TODO... from config
+		Progress: func(count int, msg string) {
+			if count < 1 || time.Since(last) > time.Second {
+				fmt.Printf("[%4d] %s\n", count, msg)
+				last = time.Now()
+			}
+		},
+	}
+
+	provisioning, err := newStubProvisioning(cfg.ProvisioningPath)
+	if err != nil {
+		return err
+	}
+
+	migrator := legacy.NewDashboardAccess(
+		legacysql.NewDatabaseProvider(sqlStore),
+		authlib.OrgNamespaceFormatter,
+		nil, provisioning, false,
+	)
+
+	yes, err := promptYesNo(fmt.Sprintf("Count legacy resources for namespace: %s?", opts.Namespace))
+	if err != nil {
+		return err
+	}
+	if yes {
+		opts.OnlyCount = true
+		rsp, err := migrator.Migrate(ctx, opts)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Counting DONE: %s\n", time.Since(start))
+		if rsp != nil {
+			jj, _ := json.MarshalIndent(rsp, "", "  ")
+			fmt.Printf("%s\n", string(jj))
+		}
+	}
+
+	opts.OnlyCount = false
+	opts.WithHistory, err = promptYesNo("Include history in exports?")
+	if err != nil {
+		return err
+	}
+
+	yes, err = promptYesNo("Export legacy resources to parquet file?")
+	if err != nil {
+		return err
+	}
+	if yes {
+		file, err := os.CreateTemp(cfg.DataPath, "grafana-export-*.parquet")
+		if err != nil {
+			return err
+		}
+		start = time.Now()
+		last = time.Now()
+		opts.Store, err = newParquetClient(file)
+		if err != nil {
+			return err
+		}
+		rsp, err := migrator.Migrate(ctx, opts)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("Parquet export DONE: %s\n", time.Since(start))
+		if rsp != nil {
+			jj, _ := json.MarshalIndent(rsp, "", "  ")
+			fmt.Printf("%s\n", string(jj))
+		}
+		fmt.Printf("File: %s\n", file.Name())
+	}
+
+	yes, err = promptYesNo("Export legacy resources to unified storage?")
+	if err != nil {
+		return err
+	}
+	if yes {
+		client, err := newUnifiedClient(cfg, sqlStore)
+		if err != nil {
+			return err
+		}
+
+		// Check the stats (eventually compare)
+		req := &resource.ResourceStatsRequest{
+			Namespace: opts.Namespace,
+		}
+		for _, r := range opts.Resources {
+			req.Kinds = append(req.Kinds, fmt.Sprintf("%s/%s", r.Group, r.Resource))
+		}
+
+		stats, err := client.GetStats(ctx, req)
+		if err != nil {
+			return err
+		}
+
+		if stats != nil {
+			fmt.Printf("Existing resources in unified storage:\n")
+			jj, _ := json.MarshalIndent(stats, "", "  ")
+			fmt.Printf("%s\n", string(jj))
+		}
+
+		yes, err = promptYesNo("Would you like to continue? (existing resources will be replaced)")
+		if err != nil {
+			return err
+		}
+		if yes {
+			start = time.Now()
+			last = time.Now()
+			opts.Store = client
+			opts.BlobStore = client
+			rsp, err := migrator.Migrate(ctx, opts)
+			if err != nil {
+				return err
+			}
+			fmt.Printf("Unified storage export: %s\n", time.Since(start))
+			if rsp != nil {
+				jj, _ := json.MarshalIndent(rsp, "", "  ")
+				fmt.Printf("%s\n", string(jj))
+			}
+		}
+	}
+	return nil
+}
+
+func promptYesNo(prompt string) (bool, error) {
+	line := ""
+	for {
+		fmt.Printf("%s (Y/N) >", prompt)
+		_, err := fmt.Scanln(&line)
+		if err != nil && err.Error() != "unexpected newline" {
+			return false, err
+		}
+		switch strings.ToLower(line) {
+		case "y", "yes":
+			return true, nil
+		case "n", "no":
+			return false, nil
+		}
+	}
+}
+
+func newUnifiedClient(cfg *setting.Cfg, sqlStore db.DB) (resource.ResourceClient, error) {
+	return unified.ProvideUnifiedStorageClient(cfg,
+		featuremgmt.WithFeatures(), // none??
+		sqlStore,
+		tracing.NewNoopTracerService(),
+		prometheus.NewPedanticRegistry(),
+		authlib.FixedAccessClient(true), // always true!
+		nil,                             // document supplier
+	)
+}
+
+func newParquetClient(file *os.File) (resource.BatchStoreClient, error) {
+	writer, err := parquet.NewParquetWriter(file)
+	if err != nil {
+		return nil, err
+	}
+	client := parquet.NewBatchResourceWriterClient(writer)
+	return client, nil
+}
@@ -0,0 +1,29 @@
+package datamigrations
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/infra/db"
+)
+
+func TestUnifiedStorageCommand(t *testing.T) {
+	// setup datasources with password, basic_auth and none
+	store := db.InitTestDB(t)
+	err := store.WithDbSession(context.Background(), func(sess *db.Session) error {
+		unistoreMigrationTest(t, sess, store)
+		return nil
+	})
+	require.NoError(t, err)
+}
+
+func unistoreMigrationTest(t *testing.T, session *db.Session, sqlstore db.DB) {
+	// empty stats
+
+	t.Run("get stats", func(t *testing.T) {
+		fmt.Printf("TODO... add folders and check that they migrate\n")
+	})
+}