From 5ad31ebe398e7dd65f31b076a205fc062d75d3c6 Mon Sep 17 00:00:00 2001 From: Mihai Doarna Date: Wed, 5 Feb 2025 10:55:47 +0200 Subject: [PATCH] API keys: Migrate API keys to service accounts at startup (#96924) * migrate API keys to SA at startup * send metrics with api key migration stats * address feedback * run API keys migration in a server lock * update logging --- .../serviceaccounts/manager/service.go | 62 +++++++++++++++++++ pkg/services/serviceaccounts/manager/stats.go | 36 +++++++++++ 2 files changed, 98 insertions(+) diff --git a/pkg/services/serviceaccounts/manager/service.go b/pkg/services/serviceaccounts/manager/service.go index bfb3938b65c..2bb64c3646a 100644 --- a/pkg/services/serviceaccounts/manager/service.go +++ b/pkg/services/serviceaccounts/manager/service.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/kvstore" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/serverlock" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/apikey" @@ -39,6 +40,8 @@ type ServiceAccountsService struct { log log.Logger backgroundLog log.Logger secretScanService secretscan.Checker + orgService org.Service + serverLock *serverlock.ServerLockService secretScanEnabled bool secretScanInterval time.Duration @@ -54,6 +57,7 @@ func ProvideServiceAccountsService( orgService org.Service, acService accesscontrol.Service, permissions accesscontrol.ServiceAccountPermissionsService, + serverLockService *serverlock.ServerLockService, ) (*ServiceAccountsService, error) { serviceAccountsStore := database.ProvideServiceAccountsStore( cfg, @@ -71,6 +75,8 @@ func ProvideServiceAccountsService( store: serviceAccountsStore, log: log.New("serviceaccounts"), backgroundLog: log.New("serviceaccounts.background"), + orgService: orgService, + serverLock: serverLockService, } if err := RegisterRoles(acService); err != nil { @@ -102,6 +108,17 @@ func (sa *ServiceAccountsService) Run(ctx context.Context) error { sa.log.Warn("Failed to get usage metrics", "error", err.Error()) } + err := sa.serverLock.LockAndExecute(ctx, "migrate API keys to service accounts", time.Minute*30, func(context.Context) { + err := sa.migrateAPIKeysForAllOrgs(ctx) + if err != nil { + sa.log.Warn("Failed to migrate API keys", "error", err.Error()) + } + }) + + if err != nil { + sa.log.Error("Failed to lock and execute the migration of API keys to service accounts", "error", err) + } + updateStatsTicker := time.NewTicker(metricsCollectionInterval) defer updateStatsTicker.Stop() @@ -299,6 +316,51 @@ func (sa *ServiceAccountsService) MigrateApiKeysToServiceAccounts(ctx context.Co return sa.store.MigrateApiKeysToServiceAccounts(ctx, orgID) } +func (sa *ServiceAccountsService) migrateAPIKeysForAllOrgs(ctx context.Context) error { + sa.log.Debug("Starting to migrate API keys to service accounts") + + total := 0 + migrated := 0 + failed := 0 + errorsTotal := 0 + + defer func() { + if total > 0 || errorsTotal > 0 { + sa.log.Info("API key migration finished", "total_keys", total, "successful_keys", migrated, "failed_keys", failed, "errors", errorsTotal) + } + setAPIKeyMigrationStats(total, migrated, failed) + }() + + orgs, err := sa.orgService.Search(ctx, &org.SearchOrgsQuery{}) + if err != nil { + return err + } + + for _, o := range orgs { + sa.log.Debug("Migrating API keys for an org", "orgId", o.ID) + + result, err := sa.store.MigrateApiKeysToServiceAccounts(ctx, o.ID) + if err != nil { + sa.log.Warn("Failed to migrate API keys", "error", err.Error(), "orgId", o.ID) + errorsTotal += 1 + continue + } + if result.Failed > 0 { + sa.log.Warn("Some API keys failed to be migrated", "total_keys", result.Total, "failed_keys", result.Failed, "orgId", o.ID) + } else if result.Total > 0 { + sa.log.Info("API key migration was successful", "orgId", o.ID, "total_keys", result.Total) + } else { + sa.log.Debug("No API keys found to migrate", "orgId", o.ID) + } + + total += result.Total + migrated += result.Migrated + failed += result.Failed + } + + return nil +} + func validOrgID(orgID int64) error { if orgID == 0 { return serviceaccounts.ErrServiceAccountInvalidOrgID.Errorf("invalid org ID 0 has been specified") diff --git a/pkg/services/serviceaccounts/manager/stats.go b/pkg/services/serviceaccounts/manager/stats.go index 521688ca66f..a882ad2db41 100644 --- a/pkg/services/serviceaccounts/manager/stats.go +++ b/pkg/services/serviceaccounts/manager/stats.go @@ -20,6 +20,15 @@ var ( // MStatTotalServiceAccountTokens is a metric gauge for total number of service account tokens MStatTotalServiceAccountTokens prometheus.Gauge + // MStatTotalMigratedAPIKeysToSATokens is a metric gauge for total number of API keys to be migrated to service account tokens + MStatTotalMigratedAPIKeysToSATokens prometheus.Gauge + + // MStatSuccessfullyMigratedAPIKeysToSATokens is a metric gauge for total number of successful migrations of API keys to service account tokens + MStatSuccessfullyMigratedAPIKeysToSATokens prometheus.Gauge + + // MStatFailedMigratedAPIKeysToSATokens is a metric gauge for total number of failed migrations of API keys to service account tokens + MStatFailedMigratedAPIKeysToSATokens prometheus.Gauge + Initialised bool = false ) @@ -42,10 +51,31 @@ func init() { Namespace: ExporterName, }) + MStatTotalMigratedAPIKeysToSATokens = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "stat_total_migrated_api_keys_to_sa_tokens", + Help: "total number of API keys to be migrated to service account tokens", + Namespace: ExporterName, + }) + + MStatSuccessfullyMigratedAPIKeysToSATokens = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "stat_successfully_migrated_api_keys_to_sa_tokens", + Help: "total number of successful migrations of API keys to service account tokens", + Namespace: ExporterName, + }) + + MStatFailedMigratedAPIKeysToSATokens = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "stat_failed_migrated_api_keys_to_sa_tokens", + Help: "total number of failed migrations of API keys to service account tokens", + Namespace: ExporterName, + }) + prometheus.MustRegister( MStatTotalServiceAccounts, MStatTotalServiceAccountTokens, MStatTotalServiceAccountsNoRole, + MStatTotalMigratedAPIKeysToSATokens, + MStatSuccessfullyMigratedAPIKeysToSATokens, + MStatFailedMigratedAPIKeysToSATokens, ) } @@ -81,3 +111,9 @@ func (sa *ServiceAccountsService) getUsageMetrics(ctx context.Context) (map[stri return stats, nil } + +func setAPIKeyMigrationStats(total, migrated, failed int) { + MStatTotalMigratedAPIKeysToSATokens.Set(float64(total)) + MStatSuccessfullyMigratedAPIKeysToSATokens.Set(float64(migrated)) + MStatFailedMigratedAPIKeysToSATokens.Set(float64(failed)) +}