CloudMigrations: Add support for migration of plugin resources (#95612)

* start plugins migration

* more plugin work

* add warning

* fakepluginsettings test

* tests get plugins

* lint

* load logos

* go lint

* get all plugins once

* locales

* josh suggestion to inject query in rtk

* more plugin filters

* remove datasource warning

* access control for plugins

* remove unused method

* lint

* use gcom list
This commit is contained in:
Dana Axinte
2025-01-06 10:37:15 +00:00
committed by GitHub
parent 6ca6ad4df7
commit 1699dfa307
23 changed files with 431 additions and 264 deletions
@@ -19,6 +19,7 @@ import (
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/authapi"
"github.com/grafana/grafana/pkg/services/authapi/fake"
"github.com/grafana/grafana/pkg/services/cloudmigration"
@@ -32,6 +33,7 @@ import (
"github.com/grafana/grafana/pkg/services/gcom"
"github.com/grafana/grafana/pkg/services/libraryelements"
"github.com/grafana/grafana/pkg/services/ngalert"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginsettings"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
"github.com/grafana/grafana/pkg/services/secrets"
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
@@ -68,6 +70,8 @@ type Service struct {
dashboardService dashboards.DashboardService
folderService folder.Service
pluginStore pluginstore.Store
accessControl accesscontrol.AccessControl
pluginSettingsService pluginsettings.Service
secretsService secrets.Service
kvStore *kvstore.NamespacedKVStore
libraryElementsService libraryelements.Service
@@ -105,6 +109,8 @@ func ProvideService(
dashboardService dashboards.DashboardService,
folderService folder.Service,
pluginStore pluginstore.Store,
pluginSettingsService pluginsettings.Service,
accessControl accesscontrol.AccessControl,
kvStore kvstore.KVStore,
libraryElementsService libraryelements.Service,
ngAlert *ngalert.AlertNG,
@@ -125,6 +131,8 @@ func ProvideService(
dashboardService: dashboardService,
folderService: folderService,
pluginStore: pluginStore,
pluginSettingsService: pluginSettingsService,
accessControl: accessControl,
kvStore: kvstore.WithNamespace(kvStore, 0, "cloudmigration"),
libraryElementsService: libraryElementsService,
ngAlert: ngAlert,
@@ -588,13 +596,7 @@ func (s *Service) GetSnapshot(ctx context.Context, query cloudmigration.GetSnaps
s.log.Error("unexpected GMS snapshot state: %s", snapshotMeta.State)
return snapshot, nil
}
// For 11.2 we only support core data sources. Apply a warning for any non-core ones before storing.
resources, err := s.getResourcesWithPluginWarnings(ctx, snapshotMeta.Results)
if err != nil {
// treat this as non-fatal since the migration still succeeded
s.log.Error("error applying plugin warnings, please open a bug report: %w", err)
}
resources := snapshotMeta.Results
// Log the errors for resources with errors at migration
for _, resource := range resources {
@@ -901,40 +903,3 @@ func (s *Service) deleteLocalFiles(snapshots []cloudmigration.CloudMigrationSnap
}
return err
}
// getResourcesWithPluginWarnings iterates through each resource and, if a non-core datasource, applies a warning that we only support core
func (s *Service) getResourcesWithPluginWarnings(ctx context.Context, results []cloudmigration.CloudMigrationResource) ([]cloudmigration.CloudMigrationResource, error) {
dsList, err := s.dsService.GetAllDataSources(ctx, &datasources.GetAllDataSourcesQuery{})
if err != nil {
return nil, fmt.Errorf("getting all data sources: %w", err)
}
dsMap := make(map[string]*datasources.DataSource, len(dsList))
for i := 0; i < len(dsList); i++ {
dsMap[dsList[i].UID] = dsList[i]
}
for i := 0; i < len(results); i++ {
r := results[i]
if r.Type == cloudmigration.DatasourceDataType &&
r.Error == "" { // any error returned by GMS takes priority
ds, ok := dsMap[r.RefID]
if !ok {
s.log.Error("data source with id %s was not found in data sources list", r.RefID)
continue
}
p, found := s.pluginStore.Plugin(ctx, ds.Type)
// if the plugin is not found, it means it was uninstalled, meaning it wasn't core
if !p.IsCorePlugin() || !found {
r.Status = cloudmigration.ItemStatusWarning
r.ErrorCode = cloudmigration.ErrOnlyCoreDataSources
r.Error = "Only core data sources are supported. Please ensure the plugin is installed on the cloud stack."
}
results[i] = r
}
}
return results, nil
}
@@ -24,8 +24,6 @@ import (
"github.com/grafana/grafana/pkg/services/annotations/annotationstest"
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/services/cloudmigration/gmsclient"
"github.com/grafana/grafana/pkg/services/contexthandler/ctxkey"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/datasources"
datafakes "github.com/grafana/grafana/pkg/services/datasources/fakes"
@@ -39,6 +37,7 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/models"
ngalertstore "github.com/grafana/grafana/pkg/services/ngalert/store"
ngalertfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginsettings"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
"github.com/grafana/grafana/pkg/services/quota/quotatest"
secretsfakes "github.com/grafana/grafana/pkg/services/secrets/fakes"
@@ -447,146 +446,6 @@ func Test_SortFolders(t *testing.T) {
require.Equal(t, expected, sortedFolders)
}
func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
// Insert a processing snapshot into the database before we start so we query GMS
createTokenResp, err := s.CreateToken(context.Background())
assert.NoError(t, err)
assert.NotEmpty(t, createTokenResp.Token)
sess, err := s.store.CreateMigrationSession(context.Background(), cloudmigration.CloudMigrationSession{
AuthToken: createTokenResp.Token,
})
require.NoError(t, err)
snapshotUid, err := s.store.CreateSnapshot(context.Background(), cloudmigration.CloudMigrationSnapshot{
UID: uuid.NewString(),
SessionUID: sess.UID,
Status: cloudmigration.SnapshotStatusProcessing,
GMSSnapshotUID: "gms uid",
})
require.NoError(t, err)
// GMS should return: a core ds, a non-core ds, a non-core ds with an error, and a ds that has been uninstalled
gmsClientMock := &gmsClientMock{
getSnapshotResponse: &cloudmigration.GetSnapshotStatusResponse{
State: cloudmigration.SnapshotStateFinished,
Results: []cloudmigration.CloudMigrationResource{
{
Name: "1 name",
ParentName: "1 parent name",
Type: cloudmigration.DatasourceDataType,
RefID: "1", // this will be core
Status: cloudmigration.ItemStatusOK,
SnapshotUID: snapshotUid,
},
{
Name: "2 name",
ParentName: "",
Type: cloudmigration.DatasourceDataType,
RefID: "2", // this will be non-core
Status: cloudmigration.ItemStatusOK,
SnapshotUID: snapshotUid,
},
{
Name: "3 name",
ParentName: "3 parent name",
Type: cloudmigration.DatasourceDataType,
RefID: "3", // this will be non-core with an error
Status: cloudmigration.ItemStatusError,
Error: "please don't overwrite me",
SnapshotUID: snapshotUid,
},
{
Name: "4 name",
ParentName: "4 folder name",
Type: cloudmigration.DatasourceDataType,
RefID: "4", // this will be deleted
Status: cloudmigration.ItemStatusOK,
SnapshotUID: snapshotUid,
},
},
},
}
s.gmsClient = gmsClientMock
// Update the internal plugin store and ds store with seed data matching the descriptions above
s.pluginStore = pluginstore.NewFakePluginStore([]pluginstore.Plugin{
{
JSONData: plugins.JSONData{
ID: "1",
},
Class: plugins.ClassCore,
},
{
JSONData: plugins.JSONData{
ID: "2",
},
Class: plugins.ClassExternal,
},
{
JSONData: plugins.JSONData{
ID: "3",
},
Class: plugins.ClassExternal,
},
}...)
s.dsService = &datafakes.FakeDataSourceService{
DataSources: []*datasources.DataSource{
{UID: "1", Type: "1"},
{UID: "2", Type: "2"},
{UID: "3", Type: "3"},
{UID: "4", Type: "4"},
},
}
var snapshot *cloudmigration.CloudMigrationSnapshot
hasFourResources := func() bool {
// Retrieve the snapshot with results
var err error
snapshot, err = s.GetSnapshot(ctxWithSignedInUser(), cloudmigration.GetSnapshotsQuery{
SnapshotUID: snapshotUid,
SessionUID: sess.UID,
ResultPage: 1,
ResultLimit: 10,
})
if !assert.NoError(t, err) {
return false
}
return len(snapshot.Resources) == 4
}
require.Eventually(t, hasFourResources, time.Second, 10*time.Millisecond)
findRef := func(id string) *cloudmigration.CloudMigrationResource {
for _, r := range snapshot.Resources {
if r.RefID == id {
return &r
}
}
return nil
}
shouldBeUnaltered := findRef("1")
assert.Equal(t, cloudmigration.ItemStatusOK, shouldBeUnaltered.Status)
assert.Empty(t, shouldBeUnaltered.Error)
shouldBeAltered := findRef("2")
assert.Equal(t, cloudmigration.ItemStatusWarning, shouldBeAltered.Status)
assert.Equal(t, shouldBeAltered.Error, "Only core data sources are supported. Please ensure the plugin is installed on the cloud stack.")
shouldHaveOriginalError := findRef("3")
assert.Equal(t, cloudmigration.ItemStatusError, shouldHaveOriginalError.Status)
assert.Equal(t, shouldHaveOriginalError.Error, "please don't overwrite me")
uninstalledAltered := findRef("4")
assert.Equal(t, cloudmigration.ItemStatusWarning, uninstalledAltered.Status)
assert.Equal(t, uninstalledAltered.Error, "Only core data sources are supported. Please ensure the plugin is installed on the cloud stack.")
}
func TestDeleteSession(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
user := &user.SignedInUser{UserUID: "user123"}
@@ -817,13 +676,135 @@ func TestGetLibraryElementsCommands(t *testing.T) {
require.Equal(t, createLibraryElementCmd.UID, cmds[0].UID)
}
func ctxWithSignedInUser() context.Context {
c := &contextmodel.ReqContext{
SignedInUser: &user.SignedInUser{OrgID: 1},
// NOTE: this should be on the plugin object
func TestIsPublicSignatureType(t *testing.T) {
testcases := []struct {
signature plugins.SignatureType
expectedPublic bool
}{
{
signature: plugins.SignatureTypeCommunity,
expectedPublic: true,
},
{
signature: plugins.SignatureTypeCommercial,
expectedPublic: true,
},
{
signature: plugins.SignatureTypeGrafana,
expectedPublic: true,
},
{
signature: plugins.SignatureTypePrivate,
expectedPublic: false,
},
{
signature: plugins.SignatureTypePrivateGlob,
expectedPublic: false,
},
}
k := ctxkey.Key{}
ctx := context.WithValue(context.Background(), k, c)
return ctx
for _, testcase := range testcases {
resPublic := IsPublicSignatureType(testcase.signature)
require.Equal(t, resPublic, testcase.expectedPublic)
}
}
func TestGetPlugins(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
user := &user.SignedInUser{OrgID: 1}
s.pluginStore = pluginstore.NewFakePluginStore([]pluginstore.Plugin{
{
JSONData: plugins.JSONData{
ID: "plugin-core",
Type: plugins.TypeDataSource,
},
Class: plugins.ClassCore,
Signature: plugins.SignatureStatusValid,
SignatureType: plugins.SignatureTypeGrafana,
},
{
JSONData: plugins.JSONData{
ID: "plugin-external-valid-grafana",
Type: plugins.TypeDataSource,
AutoEnabled: false,
},
Class: plugins.ClassExternal,
Signature: plugins.SignatureStatusValid,
SignatureType: plugins.SignatureTypeGrafana,
},
{
JSONData: plugins.JSONData{
ID: "plugin-external-valid-commercial",
Type: plugins.TypePanel,
},
Class: plugins.ClassExternal,
Signature: plugins.SignatureStatusValid,
SignatureType: plugins.SignatureTypeCommercial,
},
{
JSONData: plugins.JSONData{
ID: "plugin-external-valid-community",
Type: plugins.TypePanel,
},
Class: plugins.ClassExternal,
Signature: plugins.SignatureStatusValid,
SignatureType: plugins.SignatureTypeCommunity,
},
{
JSONData: plugins.JSONData{
ID: "plugin-external-invalid",
Type: plugins.TypePanel,
},
Class: plugins.ClassExternal,
Signature: plugins.SignatureStatusInvalid,
SignatureType: plugins.SignatureTypeGrafana,
},
{
JSONData: plugins.JSONData{
ID: "plugin-external-unsigned",
Type: plugins.TypePanel,
},
Class: plugins.ClassExternal,
Signature: plugins.SignatureStatusUnsigned,
SignatureType: plugins.SignatureTypeGrafana,
},
{
JSONData: plugins.JSONData{
ID: "plugin-external-valid-private",
Type: plugins.TypeApp,
},
Class: plugins.ClassExternal,
Signature: plugins.SignatureStatusUnsigned,
SignatureType: plugins.SignatureTypePrivate,
},
}...)
s.pluginSettingsService = &pluginsettings.FakePluginSettings{Plugins: map[string]*pluginsettings.DTO{
"plugin-external-valid-grafana": {ID: 0, OrgID: user.OrgID, PluginID: "plugin-external-valid-grafana", PluginVersion: "1.0.0", Enabled: true},
}}
plugins, err := s.getPlugins(ctx, user)
require.NoError(t, err)
require.NotNil(t, plugins)
require.Len(t, plugins, 3)
expectedPluginIDs := []string{"plugin-external-valid-grafana", "plugin-external-valid-commercial", "plugin-external-valid-community"}
pluginsIDs := make([]string, 0)
for _, plugin := range plugins {
// Special case of using the settings from the settings store
if plugin.ID == "plugin-external-valid-grafana" {
require.True(t, plugin.SettingCmd.Enabled)
}
pluginsIDs = append(pluginsIDs, plugin.ID)
}
require.ElementsMatch(t, pluginsIDs, expectedPluginIDs)
}
type configOverrides func(c *setting.Cfg)
@@ -949,6 +930,8 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool, cfgOverrides ...conf
dashboardService,
mockFolder,
&pluginstore.FakePluginStore{},
&pluginsettings.FakePluginSettings{},
actest.FakeAccessControl{ExpectedEvaluate: true},
kvstore.ProvideService(sqlStore),
&libraryelementsfake.LibraryElementService{},
ng,
@@ -4,6 +4,7 @@ import (
"context"
cryptoRand "crypto/rand"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@@ -14,12 +15,18 @@ import (
snapshot "github.com/grafana/grafana-cloud-migration-snapshot/src"
"github.com/grafana/grafana-cloud-migration-snapshot/src/contracts"
"github.com/grafana/grafana-cloud-migration-snapshot/src/infra/crypto"
"github.com/grafana/grafana/pkg/infra/tracing"
plugins "github.com/grafana/grafana/pkg/plugins"
ac "github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/cloudmigration"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
libraryelements "github.com/grafana/grafana/pkg/services/libraryelements/model"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginaccesscontrol"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginsettings"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/util/retryer"
"golang.org/x/crypto/nacl/box"
@@ -37,12 +44,20 @@ var currentMigrationTypes = []cloudmigration.MigrateDataType{
cloudmigration.ContactPointType,
cloudmigration.NotificationPolicyType,
cloudmigration.AlertRuleType,
cloudmigration.PluginDataType,
}
func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.SignedInUser) (*cloudmigration.MigrateDataRequest, error) {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.getMigrationDataJSON")
defer span.End()
// Plugins
plugins, err := s.getPlugins(ctx, signedInUser)
if err != nil {
s.log.Error("Failed to get plugins", "err", err)
return nil, err
}
// Data sources
dataSources, err := s.getDataSourceCommands(ctx, signedInUser)
if err != nil {
@@ -100,10 +115,19 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
migrationDataSlice := make(
[]cloudmigration.MigrateDataRequestItem, 0,
len(dataSources)+len(dashs)+len(folders)+len(libraryElements)+
len(plugins)+len(dataSources)+len(dashs)+len(folders)+len(libraryElements)+
len(muteTimings)+len(notificationTemplates)+len(contactPoints)+len(alertRules),
)
for _, plugin := range plugins {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.PluginDataType,
RefID: plugin.ID,
Name: plugin.Name,
Data: plugin.SettingCmd,
})
}
for _, ds := range dataSources {
migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
Type: cloudmigration.DatasourceDataType,
@@ -356,6 +380,105 @@ func (s *Service) getLibraryElementsCommands(ctx context.Context, signedInUser *
return cmds, nil
}
type PluginCmd struct {
ID string `json:"id"`
Name string `json:"name"`
SettingCmd pluginsettings.UpdatePluginSettingCmd `json:"settingCmd"`
}
// IsPublicSignatureType returns true if plugin signature type is public
func IsPublicSignatureType(signatureType plugins.SignatureType) bool {
switch signatureType {
case plugins.SignatureTypeGrafana, plugins.SignatureTypeCommercial, plugins.SignatureTypeCommunity:
return true
case plugins.SignatureTypePrivate, plugins.SignatureTypePrivateGlob:
return false
}
return false
}
// getPlugins returns the json payloads required by the plugin creation API
func (s *Service) getPlugins(ctx context.Context, signedInUser *user.SignedInUser) ([]PluginCmd, error) {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.getPlugins")
defer span.End()
results := make([]PluginCmd, 0)
plugins := s.pluginStore.Plugins(ctx)
// Obtain plugins from gcom
requestID := tracing.TraceIDFromContext(ctx, false)
gcomPlugins, err := s.gcomService.GetPlugins(ctx, requestID)
if err != nil {
return results, fmt.Errorf("fetching gcom plugins: %w", err)
}
// Permissions for listing plugins, taken from plugins api
userIsOrgAdmin := signedInUser.HasRole(org.RoleAdmin)
hasAccess, _ := s.accessControl.Evaluate(ctx, signedInUser, ac.EvalAny(
ac.EvalPermission(datasources.ActionCreate),
ac.EvalPermission(pluginaccesscontrol.ActionInstall),
))
if !(userIsOrgAdmin || hasAccess) {
s.log.Info("user is not allowed to list non-core plugins", "UID", signedInUser.UserUID)
return results, nil
}
for _, plugin := range plugins {
// filter plugins to keep only the ones allowed by gcom
if _, exists := gcomPlugins[plugin.ID]; !exists {
continue
}
// filter plugins to keep only non core, signed, with public signature type plugins
if plugin.IsCorePlugin() || !plugin.Signature.IsValid() || !IsPublicSignatureType(plugin.SignatureType) {
continue
}
// filter out dependent app plugins
if plugin.IncludedInAppID != "" {
continue
}
// Permissions filtering, taken from plugins api
hasAccess, _ = s.accessControl.Evaluate(ctx, signedInUser, ac.EvalPermission(pluginaccesscontrol.ActionWrite, pluginaccesscontrol.ScopeProvider.GetResourceScope(plugin.ID)))
if !hasAccess {
continue
}
pluginSettingCmd := pluginsettings.UpdatePluginSettingCmd{
Enabled: plugin.JSONData.AutoEnabled,
Pinned: plugin.Pinned,
PluginVersion: plugin.Info.Version,
PluginId: plugin.ID,
}
// get plugin settings from db if they exist
ps, err := s.pluginSettingsService.GetPluginSettingByPluginID(ctx, &pluginsettings.GetByPluginIDArgs{
PluginID: plugin.ID,
OrgID: signedInUser.OrgID,
})
if err != nil && !errors.Is(err, pluginsettings.ErrPluginSettingNotFound) {
return nil, fmt.Errorf("failed to get plugin settings: %w", err)
} else if ps != nil {
pluginSettingCmd.Enabled = ps.Enabled
pluginSettingCmd.Pinned = ps.Pinned
pluginSettingCmd.JsonData = ps.JSONData
decryptedData, err := s.secretsService.DecryptJsonData(ctx, ps.SecureJSONData)
if err != nil {
return nil, fmt.Errorf("failed to decrypt secure json data: %w", err)
}
pluginSettingCmd.SecureJsonData = decryptedData
}
results = append(results, PluginCmd{
ID: plugin.ID,
Name: plugin.Name,
SettingCmd: pluginSettingCmd,
})
}
return results, nil
}
// asynchronous process for writing the snapshot to the filesystem and updating the snapshot status
func (s *Service) buildSnapshot(ctx context.Context, signedInUser *user.SignedInUser, maxItemsPerPartition uint32, metadata []byte, snapshotMeta cloudmigration.CloudMigrationSnapshot) error {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.buildSnapshot")