feat: inject unified data migrations in dual writer (#114138)
* feat: draft changes for on-prem unified migration * feat: further draft changes for on-prem unified migration * fix: remove some tbis * refactor: rename * fix: another approach * fix: background service related issues * fix: address comments * fix: make gen-go * fix: background service related issues * feat: refactor dual writer and legacy migrator * fix: minor issues * feat: working version in oss * fix: wire * fix: revert test data override * fix: enterprise related issues * chore: add todo * fix: revert dual writer method * fix: lint * chore: logger format * fix: reduce log level * fix: log change * fix: disable * fix: address comments * fix: return error on dual writer service * fix: merge conflict --------- Co-authored-by: Rafael Paulovic <rafael.paulovic@grafana.com>
This commit is contained in:
Mustafa Sencer Özcan
co-authored by
Rafael Paulovic
Rafael Paulovic
parent
cb06bba243
commit
30c04ab3fc
@@ -1,70 +0,0 @@
|
||||
package datamigrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning/dashboards"
|
||||
)
|
||||
|
||||
var (
|
||||
_ provisioning.ProvisioningService = (*stubProvisioning)(nil)
|
||||
)
|
||||
|
||||
func newStubProvisioning(path string) (provisioning.ProvisioningService, error) {
|
||||
cfgs, err := dashboards.ReadDashboardConfig(filepath.Join(path, "dashboards"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stub := &stubProvisioning{
|
||||
path: make(map[string]string),
|
||||
}
|
||||
for _, cfg := range cfgs {
|
||||
stub.path[cfg.Name] = cfg.Options["path"].(string)
|
||||
}
|
||||
return &stubProvisioning{}, nil
|
||||
}
|
||||
|
||||
type stubProvisioning struct {
|
||||
path map[string]string // name > options.path
|
||||
}
|
||||
|
||||
// GetAllowUIUpdatesFromConfig implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) GetAllowUIUpdatesFromConfig(name string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *stubProvisioning) GetDashboardProvisionerResolvedPath(name string) string {
|
||||
return s.path[name]
|
||||
}
|
||||
|
||||
// ProvisionAlerting implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionAlerting(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// ProvisionDashboards implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionDashboards(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// ProvisionDatasources implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionDatasources(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// ProvisionPlugins implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) ProvisionPlugins(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// Run implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) Run(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
|
||||
// RunInitProvisioners implements provisioning.ProvisioningService.
|
||||
func (s *stubProvisioning) RunInitProvisioners(ctx context.Context) error {
|
||||
panic("unimplemented")
|
||||
}
|
||||
@@ -24,10 +24,11 @@ import (
|
||||
"github.com/grafana/grafana/pkg/registry/apis/dashboard/legacy"
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol/acimpl"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/search/sort"
|
||||
"github.com/grafana/grafana/pkg/services/provisioning"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/migrations"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/parquet"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
@@ -52,7 +53,6 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
{Group: folders.GROUP, Resource: folders.RESOURCE},
|
||||
{Group: dashboard.GROUP, Resource: dashboard.DASHBOARD_RESOURCE},
|
||||
},
|
||||
LargeObjects: nil, // TODO... from config
|
||||
Progress: func(count int, msg string) {
|
||||
const minInterval = time.Second
|
||||
shouldPrint := count < 1 || time.Since(last) > minInterval
|
||||
@@ -69,31 +69,26 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
}
|
||||
featureToggles := featuremgmt.ProvideToggles(featureManager)
|
||||
|
||||
provisioning, err := newStubProvisioning(cfg.ProvisioningPath)
|
||||
provisioning, err := provisioning.ProvideStubProvisioningService(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
migrator := legacy.NewDashboardAccess(
|
||||
grpcClient, err := newUnifiedClient(cfg, sqlStore, featureToggles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dashboardAccess := legacy.ProvideMigratorDashboardAccessor(
|
||||
legacysql.NewDatabaseProvider(sqlStore),
|
||||
authlib.OrgNamespaceFormatter,
|
||||
nil, // no dashboards.Store
|
||||
provisioning,
|
||||
nil, // no librarypanels.Service
|
||||
sort.ProvideService(),
|
||||
nil, // we don't delete during migration, and this is only need to delete permission.
|
||||
acimpl.ProvideAccessControl(featuremgmt.WithFeatures()),
|
||||
featureToggles,
|
||||
)
|
||||
|
||||
client, err := newUnifiedClient(cfg, sqlStore, featureToggles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if c.Bool("non-interactive") {
|
||||
opts.Store = client
|
||||
opts.BlobStore = client
|
||||
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient)
|
||||
|
||||
opts.WithHistory = true // always include history in non-interactive mode
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if exitErr := handleMigrationError(err, rsp); exitErr != nil {
|
||||
@@ -113,6 +108,8 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
return err
|
||||
}
|
||||
if yes {
|
||||
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, nil) // no need for grpc client for counting
|
||||
|
||||
opts.OnlyCount = true
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
@@ -141,12 +138,13 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start = time.Now()
|
||||
last = time.Now()
|
||||
opts.Store, err = newParquetClient(file)
|
||||
parquetClient, err := newParquetClient(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
migrator := migrations.ProvideUnifiedMigratorParquet(dashboardAccess, parquetClient)
|
||||
start = time.Now()
|
||||
last = time.Now()
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -172,7 +170,7 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
req.Kinds = append(req.Kinds, fmt.Sprintf("%s/%s", r.Group, r.Resource))
|
||||
}
|
||||
|
||||
stats, err := client.GetStats(ctx, req)
|
||||
stats, err := grpcClient.GetStats(ctx, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -188,10 +186,9 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
return err
|
||||
}
|
||||
if yes {
|
||||
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient)
|
||||
start = time.Now()
|
||||
last = time.Now()
|
||||
opts.Store = client
|
||||
opts.BlobStore = client
|
||||
rsp, err := migrator.Migrate(ctx, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user