From 45a7f649fe7153500e0e606c98e058083af8e76a Mon Sep 17 00:00:00 2001 From: Michael Mandrus <41969079+mmandrus@users.noreply.github.com> Date: Sat, 20 Apr 2024 23:51:58 -0400 Subject: [PATCH] CMS: Create local implementation of cloud migration for dev use (#86637) * add developer mode property to config * create cms stub * cleanup * implement and wire up gcom stub * fix errors * don't document the flag --- conf/defaults.ini | 2 +- .../cloudmigrationimpl/cloudmigration.go | 19 ++++-- .../cloudmigrationimpl/gcomstub.go | 60 +++++++++++++++++++ .../cloudmigration/cloudmigrationtest/fake.go | 15 ----- .../cloudmigration/cmsclient/client.go | 17 ++++++ .../cmsclient/{cmsclient.go => cms_client.go} | 16 ++--- .../cmsclient/inmemory_client.go | 48 +++++++++++++++ pkg/services/gcom/gcom.go | 4 +- pkg/setting/setting_cloud_migration.go | 3 + 9 files changed, 149 insertions(+), 35 deletions(-) create mode 100644 pkg/services/cloudmigration/cloudmigrationimpl/gcomstub.go delete mode 100644 pkg/services/cloudmigration/cloudmigrationtest/fake.go create mode 100644 pkg/services/cloudmigration/cmsclient/client.go rename pkg/services/cloudmigration/cmsclient/{cmsclient.go => cms_client.go} (83%) create mode 100644 pkg/services/cloudmigration/cmsclient/inmemory_client.go diff --git a/conf/defaults.ini b/conf/defaults.ini index 43ebf88b59f..e1acf689bb5 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1839,4 +1839,4 @@ fetch_access_policy_timeout = 5s # How long to wait for a request to create to delete an access policy to complete delete_access_policy_timeout = 5s # The domain name used to access cms -domain = grafana-dev.net +domain = grafana-dev.net \ No newline at end of file diff --git a/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go b/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go index 3aaa12e6b11..d6bbad01a72 100644 --- a/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go +++ b/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go @@ -84,7 +84,6 @@ func ProvideService( cfg: cfg, features: features, dsService: dsService, - gcomService: gcom.New(gcom.Config{ApiURL: cfg.GrafanaComAPIURL, Token: cfg.CloudMigration.GcomAPIToken}), tracer: tracer, metrics: newMetrics(), secretsService: secretsService, @@ -93,12 +92,20 @@ func ProvideService( } s.api = api.RegisterApi(routeRegister, s, tracer) - // get CMS path from the config - domain, err := s.parseCloudMigrationConfig() - if err != nil { - return nil, fmt.Errorf("config parse error: %w", err) + if !cfg.CloudMigration.IsDeveloperMode { + // get CMS path from the config + domain, err := s.parseCloudMigrationConfig() + if err != nil { + return nil, fmt.Errorf("config parse error: %w", err) + } + s.cmsClient = cmsclient.NewCMSClient(domain) + + s.gcomService = gcom.New(gcom.Config{ApiURL: cfg.GrafanaComAPIURL, Token: cfg.CloudMigration.GcomAPIToken}) + } else { + s.cmsClient = cmsclient.NewInMemoryClient() + s.gcomService = &gcomStub{map[string]gcom.AccessPolicy{}} + s.cfg.StackID = "12345" } - s.cmsClient = cmsclient.NewCMSClient(domain) if err := s.registerMetrics(prom, s.metrics); err != nil { s.log.Warn("error registering prom metrics", "error", err.Error()) diff --git a/pkg/services/cloudmigration/cloudmigrationimpl/gcomstub.go b/pkg/services/cloudmigration/cloudmigrationimpl/gcomstub.go new file mode 100644 index 00000000000..79d4001b5b0 --- /dev/null +++ b/pkg/services/cloudmigration/cloudmigrationimpl/gcomstub.go @@ -0,0 +1,60 @@ +package cloudmigrationimpl + +import ( + "context" + "fmt" + "strconv" + + "github.com/grafana/grafana/pkg/services/gcom" + "github.com/grafana/grafana/pkg/util" +) + +type gcomStub struct { + policies map[string]gcom.AccessPolicy +} + +func (client *gcomStub) GetInstanceByID(ctx context.Context, requestID string, instanceID string) (gcom.Instance, error) { + id, err := strconv.Atoi(instanceID) + if err != nil { + return gcom.Instance{}, fmt.Errorf("parsing instanceID: %w", err) + } + return gcom.Instance{ + ID: id, + Slug: "stubinstance", + RegionSlug: "fake-region", + ClusterSlug: "fake-cluser", + }, nil +} + +func (client *gcomStub) CreateAccessPolicy(ctx context.Context, params gcom.CreateAccessPolicyParams, payload gcom.CreateAccessPolicyPayload) (gcom.AccessPolicy, error) { + randStr := fmt.Sprintf("random-policy-%s", util.GenerateShortUID()) + policy := gcom.AccessPolicy{ + ID: randStr, + Name: randStr, + } + client.policies[policy.ID] = policy + return policy, nil +} + +func (client *gcomStub) DeleteAccessPolicy(ctx context.Context, params gcom.DeleteAccessPolicyParams) (bool, error) { + delete(client.policies, params.AccessPolicyID) + return true, nil +} + +func (client *gcomStub) ListAccessPolicies(ctx context.Context, params gcom.ListAccessPoliciesParams) ([]gcom.AccessPolicy, error) { + items := make([]gcom.AccessPolicy, 0) + for _, v := range client.policies { + items = append(items, v) + } + return items, nil +} + +func (client *gcomStub) CreateToken(ctx context.Context, params gcom.CreateTokenParams, payload gcom.CreateTokenPayload) (gcom.Token, error) { + token := gcom.Token{ + ID: fmt.Sprintf("random-token-%s", util.GenerateShortUID()), + Name: payload.Name, + AccessPolicyID: payload.AccessPolicyID, + Token: fmt.Sprintf("completely_fake_token_%s", util.GenerateShortUID()), + } + return token, nil +} diff --git a/pkg/services/cloudmigration/cloudmigrationtest/fake.go b/pkg/services/cloudmigration/cloudmigrationtest/fake.go deleted file mode 100644 index 7f410d3a27d..00000000000 --- a/pkg/services/cloudmigration/cloudmigrationtest/fake.go +++ /dev/null @@ -1,15 +0,0 @@ -package cloudmigrationtest - -import ( - "context" - - "github.com/grafana/grafana/pkg/services/cloudmigration" -) - -type Service struct { - ExpectedError error -} - -func (s *Service) MigrateDatasources(ctx context.Context, request *cloudmigration.MigrateDatasourcesRequest) (*cloudmigration.MigrateDatasourcesResponse, error) { - return nil, cloudmigration.ErrInternalNotImplementedError -} diff --git a/pkg/services/cloudmigration/cmsclient/client.go b/pkg/services/cloudmigration/cmsclient/client.go new file mode 100644 index 00000000000..1694a3112f7 --- /dev/null +++ b/pkg/services/cloudmigration/cmsclient/client.go @@ -0,0 +1,17 @@ +package cmsclient + +import ( + "context" + + "github.com/grafana/grafana/pkg/services/cloudmigration" + "github.com/grafana/grafana/pkg/util/errutil" +) + +type Client interface { + ValidateKey(context.Context, cloudmigration.CloudMigration) error + MigrateData(context.Context, cloudmigration.CloudMigration, cloudmigration.MigrateDataRequestDTO) (*cloudmigration.MigrateDataResponseDTO, error) +} + +const logPrefix = "cloudmigration.cmsclient" + +var ErrMigrationNotDeleted = errutil.Internal("cloudmigrations.developerModeEnabled", errutil.WithPublicMessage("Developer mode enabled")) diff --git a/pkg/services/cloudmigration/cmsclient/cmsclient.go b/pkg/services/cloudmigration/cmsclient/cms_client.go similarity index 83% rename from pkg/services/cloudmigration/cmsclient/cmsclient.go rename to pkg/services/cloudmigration/cmsclient/cms_client.go index e614d36a4a8..576e1385d71 100644 --- a/pkg/services/cloudmigration/cmsclient/cmsclient.go +++ b/pkg/services/cloudmigration/cmsclient/cms_client.go @@ -11,26 +11,20 @@ import ( "github.com/grafana/grafana/pkg/services/cloudmigration" ) -type Client interface { - ValidateKey(context.Context, cloudmigration.CloudMigration) error - MigrateData(context.Context, cloudmigration.CloudMigration, cloudmigration.MigrateDataRequestDTO) (*cloudmigration.MigrateDataResponseDTO, error) -} - -const logPrefix = "cloudmigration.cmsclient" - +// NewCMSClient returns an implementation of Client that queries CloudMigrationService func NewCMSClient(domain string) Client { - return &clientImpl{ + return &cmsClientImpl{ domain: domain, log: log.New(logPrefix), } } -type clientImpl struct { +type cmsClientImpl struct { domain string log *log.ConcreteLogger } -func (c *clientImpl) ValidateKey(ctx context.Context, cm cloudmigration.CloudMigration) error { +func (c *cmsClientImpl) ValidateKey(ctx context.Context, cm cloudmigration.CloudMigration) error { logger := c.log.FromContext(ctx) path := fmt.Sprintf("https://cms-%s.%s/cloud-migrations/api/v1/validate-key", cm.ClusterSlug, c.domain) @@ -69,7 +63,7 @@ func (c *clientImpl) ValidateKey(ctx context.Context, cm cloudmigration.CloudMig return nil } -func (c *clientImpl) MigrateData(ctx context.Context, cm cloudmigration.CloudMigration, request cloudmigration.MigrateDataRequestDTO) (*cloudmigration.MigrateDataResponseDTO, error) { +func (c *cmsClientImpl) MigrateData(ctx context.Context, cm cloudmigration.CloudMigration, request cloudmigration.MigrateDataRequestDTO) (*cloudmigration.MigrateDataResponseDTO, error) { logger := c.log.FromContext(ctx) path := fmt.Sprintf("https://cms-%s.%s/cloud-migrations/api/v1/migrate-data", cm.ClusterSlug, c.domain) diff --git a/pkg/services/cloudmigration/cmsclient/inmemory_client.go b/pkg/services/cloudmigration/cmsclient/inmemory_client.go new file mode 100644 index 00000000000..3c44abe67ae --- /dev/null +++ b/pkg/services/cloudmigration/cmsclient/inmemory_client.go @@ -0,0 +1,48 @@ +package cmsclient + +import ( + "context" + "math/rand" + + "github.com/grafana/grafana/pkg/services/cloudmigration" +) + +// NewInMemoryClient returns an implementation of Client that returns canned responses +func NewInMemoryClient() Client { + return &memoryClientImpl{} +} + +type memoryClientImpl struct{} + +func (c *memoryClientImpl) ValidateKey(ctx context.Context, cm cloudmigration.CloudMigration) error { + // return ErrMigrationNotDeleted + return nil +} + +func (c *memoryClientImpl) MigrateData( + ctx context.Context, + cm cloudmigration.CloudMigration, + request cloudmigration.MigrateDataRequestDTO, +) (*cloudmigration.MigrateDataResponseDTO, error) { + //return nil, ErrMigrationNotDeleted + + result := cloudmigration.MigrateDataResponseDTO{ + Items: make([]cloudmigration.MigrateDataResponseItemDTO, len(request.Items)), + } + + for i, v := range request.Items { + result.Items[i] = cloudmigration.MigrateDataResponseItemDTO{ + Type: v.Type, + RefID: v.RefID, + Status: cloudmigration.ItemStatusOK, + } + } + + // simulate flakiness on one random item + i := rand.Intn(len(result.Items)) + failedItem := result.Items[i] + failedItem.Status, failedItem.Error = cloudmigration.ItemStatusError, "simulated random error" + result.Items[i] = failedItem + + return &result, nil +} diff --git a/pkg/services/gcom/gcom.go b/pkg/services/gcom/gcom.go index dbcbeb1bf65..54be4386742 100644 --- a/pkg/services/gcom/gcom.go +++ b/pkg/services/gcom/gcom.go @@ -63,7 +63,7 @@ type ListAccessPoliciesParams struct { Name string } -type listAccessPoliciesResponse struct { +type ListAccessPoliciesResponse struct { Items []AccessPolicy `json:"items"` } @@ -273,7 +273,7 @@ func (client *GcomClient) ListAccessPolicies(ctx context.Context, params ListAcc return nil, fmt.Errorf("unexpected response when listing access policies: code=%d body=%s", response.StatusCode, body) } - var responseBody listAccessPoliciesResponse + var responseBody ListAccessPoliciesResponse if err := json.NewDecoder(response.Body).Decode(&responseBody); err != nil { return responseBody.Items, fmt.Errorf("unmarshaling response body: %w", err) } diff --git a/pkg/setting/setting_cloud_migration.go b/pkg/setting/setting_cloud_migration.go index 7519ae535b7..a289123edee 100644 --- a/pkg/setting/setting_cloud_migration.go +++ b/pkg/setting/setting_cloud_migration.go @@ -13,6 +13,8 @@ type CloudMigrationSettings struct { DeleteAccessPolicyTimeout time.Duration CreateTokenTimeout time.Duration TokenExpiresAfter time.Duration + + IsDeveloperMode bool } func (cfg *Cfg) readCloudMigrationSettings() { @@ -25,4 +27,5 @@ func (cfg *Cfg) readCloudMigrationSettings() { cfg.CloudMigration.DeleteAccessPolicyTimeout = cloudMigration.Key("delete_access_policy_timeout").MustDuration(5 * time.Second) cfg.CloudMigration.CreateTokenTimeout = cloudMigration.Key("create_token_timeout").MustDuration(5 * time.Second) cfg.CloudMigration.TokenExpiresAfter = cloudMigration.Key("token_expires_after").MustDuration(7 * 24 * time.Hour) + cfg.CloudMigration.IsDeveloperMode = cloudMigration.Key("developer_mode").MustBool(false) }