API keys: Migrate API keys to service accounts at startup (#96924)

* migrate API keys to SA at startup

* send metrics with api key migration stats

* address feedback

* run API keys migration in a server lock

* update logging

pkg/services/serviceaccounts/manager/service.go

@@ -13,6 +13,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/serverlock"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/apikey"
@@ -39,6 +40,8 @@ type ServiceAccountsService struct {
 	log               log.Logger
 	backgroundLog     log.Logger
 	secretScanService secretscan.Checker
+	orgService        org.Service
+	serverLock        *serverlock.ServerLockService
 
 	secretScanEnabled  bool
 	secretScanInterval time.Duration
@@ -54,6 +57,7 @@ func ProvideServiceAccountsService(
 	orgService org.Service,
 	acService accesscontrol.Service,
 	permissions accesscontrol.ServiceAccountPermissionsService,
+	serverLockService *serverlock.ServerLockService,
 ) (*ServiceAccountsService, error) {
 	serviceAccountsStore := database.ProvideServiceAccountsStore(
 		cfg,
@@ -71,6 +75,8 @@ func ProvideServiceAccountsService(
 		store:         serviceAccountsStore,
 		log:           log.New("serviceaccounts"),
 		backgroundLog: log.New("serviceaccounts.background"),
+		orgService:    orgService,
+		serverLock:    serverLockService,
 	}
 
 	if err := RegisterRoles(acService); err != nil {
@@ -102,6 +108,17 @@ func (sa *ServiceAccountsService) Run(ctx context.Context) error {
 		sa.log.Warn("Failed to get usage metrics", "error", err.Error())
 	}
 
+	err := sa.serverLock.LockAndExecute(ctx, "migrate API keys to service accounts", time.Minute*30, func(context.Context) {
+		err := sa.migrateAPIKeysForAllOrgs(ctx)
+		if err != nil {
+			sa.log.Warn("Failed to migrate API keys", "error", err.Error())
+		}
+	})
+
+	if err != nil {
+		sa.log.Error("Failed to lock and execute the migration of API keys to service accounts", "error", err)
+	}
+
 	updateStatsTicker := time.NewTicker(metricsCollectionInterval)
 	defer updateStatsTicker.Stop()
 
@@ -299,6 +316,51 @@ func (sa *ServiceAccountsService) MigrateApiKeysToServiceAccounts(ctx context.Co
 	return sa.store.MigrateApiKeysToServiceAccounts(ctx, orgID)
 }
 
+func (sa *ServiceAccountsService) migrateAPIKeysForAllOrgs(ctx context.Context) error {
+	sa.log.Debug("Starting to migrate API keys to service accounts")
+
+	total := 0
+	migrated := 0
+	failed := 0
+	errorsTotal := 0
+
+	defer func() {
+		if total > 0 || errorsTotal > 0 {
+			sa.log.Info("API key migration finished", "total_keys", total, "successful_keys", migrated, "failed_keys", failed, "errors", errorsTotal)
+		}
+		setAPIKeyMigrationStats(total, migrated, failed)
+	}()
+
+	orgs, err := sa.orgService.Search(ctx, &org.SearchOrgsQuery{})
+	if err != nil {
+		return err
+	}
+
+	for _, o := range orgs {
+		sa.log.Debug("Migrating API keys for an org", "orgId", o.ID)
+
+		result, err := sa.store.MigrateApiKeysToServiceAccounts(ctx, o.ID)
+		if err != nil {
+			sa.log.Warn("Failed to migrate API keys", "error", err.Error(), "orgId", o.ID)
+			errorsTotal += 1
+			continue
+		}
+		if result.Failed > 0 {
+			sa.log.Warn("Some API keys failed to be migrated", "total_keys", result.Total, "failed_keys", result.Failed, "orgId", o.ID)
+		} else if result.Total > 0 {
+			sa.log.Info("API key migration was successful", "orgId", o.ID, "total_keys", result.Total)
+		} else {
+			sa.log.Debug("No API keys found to migrate", "orgId", o.ID)
+		}
+
+		total += result.Total
+		migrated += result.Migrated
+		failed += result.Failed
+	}
+
+	return nil
+}
+
 func validOrgID(orgID int64) error {
 	if orgID == 0 {
 		return serviceaccounts.ErrServiceAccountInvalidOrgID.Errorf("invalid org ID 0 has been specified")

pkg/services/serviceaccounts/manager/stats.go

@@ -20,6 +20,15 @@ var (
 	// MStatTotalServiceAccountTokens is a metric gauge for total number of service account tokens
 	MStatTotalServiceAccountTokens prometheus.Gauge
 
+	// MStatTotalMigratedAPIKeysToSATokens is a metric gauge for total number of API keys to be migrated to service account tokens
+	MStatTotalMigratedAPIKeysToSATokens prometheus.Gauge
+
+	// MStatSuccessfullyMigratedAPIKeysToSATokens is a metric gauge for total number of successful migrations of API keys to service account tokens
+	MStatSuccessfullyMigratedAPIKeysToSATokens prometheus.Gauge
+
+	// MStatFailedMigratedAPIKeysToSATokens is a metric gauge for total number of failed migrations of API keys to service account tokens
+	MStatFailedMigratedAPIKeysToSATokens prometheus.Gauge
+
 	Initialised bool = false
 )
 
@@ -42,10 +51,31 @@ func init() {
 		Namespace: ExporterName,
 	})
 
+	MStatTotalMigratedAPIKeysToSATokens = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_total_migrated_api_keys_to_sa_tokens",
+		Help:      "total number of API keys to be migrated to service account tokens",
+		Namespace: ExporterName,
+	})
+
+	MStatSuccessfullyMigratedAPIKeysToSATokens = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_successfully_migrated_api_keys_to_sa_tokens",
+		Help:      "total number of successful migrations of API keys to service account tokens",
+		Namespace: ExporterName,
+	})
+
+	MStatFailedMigratedAPIKeysToSATokens = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_failed_migrated_api_keys_to_sa_tokens",
+		Help:      "total number of failed migrations of API keys to service account tokens",
+		Namespace: ExporterName,
+	})
+
 	prometheus.MustRegister(
 		MStatTotalServiceAccounts,
 		MStatTotalServiceAccountTokens,
 		MStatTotalServiceAccountsNoRole,
+		MStatTotalMigratedAPIKeysToSATokens,
+		MStatSuccessfullyMigratedAPIKeysToSATokens,
+		MStatFailedMigratedAPIKeysToSATokens,
 	)
 }
 
@@ -81,3 +111,9 @@ func (sa *ServiceAccountsService) getUsageMetrics(ctx context.Context) (map[stri
 
 	return stats, nil
 }
+
+func setAPIKeyMigrationStats(total, migrated, failed int) {
+	MStatTotalMigratedAPIKeysToSATokens.Set(float64(total))
+	MStatSuccessfullyMigratedAPIKeysToSATokens.Set(float64(migrated))
+	MStatFailedMigratedAPIKeysToSATokens.Set(float64(failed))
+}