Secrets: Implement basic unified secret store service (#45804)

* wip: Implement kvstore for secrets

* wip: Refactor kvstore for secrets

* wip: Add format key function to secrets kvstore sql

* wip: Add migration for secrets kvstore

* Remove unused Key field from secrets kvstore

* Remove secret values from debug logs

* Integrate unified secrets with datasources

* Fix minor issues and tests for kvstore

* Create test service helper for secret store

* Remove encryption tests from datasources

* Move secret operations after datasources

* Fix datasource proxy tests

* Fix legacy data tests

* Add Name to all delete data source commands

* Implement decryption cache on sql secret store

* Fix minor issue with cache and tests

* Use secret type on secret store datasource operations

* Add comments to make create and update clear

* Rename itemFound variable to isFound

* Improve secret deletion and cache management

* Add base64 encoding to sql secret store

* Move secret retrieval to decrypted values function

* Refactor decrypt secure json data functions

* Fix expr tests

* Fix datasource tests

* Fix plugin proxy tests

* Fix query tests

* Fix metrics api tests

* Remove unused fake secrets service from query tests

* Add rename function to secret store

* Add check for error renaming secret

* Remove bus from tests to fix merge conflicts

* Add background secrets migration to datasources

* Get datasource secure json fields from secrets

* Move migration to secret store

* Revert "Move migration to secret store"

This reverts commit 7c3f872072.

* Add secret service to datasource service on tests

* Fix datasource tests

* Remove merge conflict on wire

* Add ctx to data source http transport on prometheus stats collector

* Add ctx to data source http transport on stats collector test

pkg/tsdb/legacydata/service/service.go

@@ -40,6 +40,11 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer
 		return legacydata.DataResponse{}, err
 	}
 
+	decryptedValues, err := h.dataSourcesService.DecryptedValues(ctx, ds)
+	if err != nil {
+		return legacydata.DataResponse{}, err
+	}
+
 	instanceSettings := &backend.DataSourceInstanceSettings{
 		ID:                      ds.Id,
 		Name:                    ds.Name,
@@ -49,7 +54,7 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer
 		BasicAuthEnabled:        ds.BasicAuth,
 		BasicAuthUser:           ds.BasicAuthUser,
 		JSONData:                jsonDataBytes,
-		DecryptedSecureJSONData: h.dataSourcesService.DecryptedValues(ds),
+		DecryptedSecureJSONData: decryptedValues,
 		Updated:                 ds.Updated,
 		UID:                     ds.Uid,
 	}

pkg/tsdb/legacydata/service/service_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/oauthtoken"
 	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	"github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/legacydata"
@@ -38,8 +39,9 @@ func TestHandleRequest(t *testing.T) {
 			actualReq = req
 			return backend.NewQueryDataResponse(), nil
 		}
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+		dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 		s := ProvideService(client, nil, dsService)
 
 		ds := &models.DataSource{Id: 12, Type: "unregisteredType", JsonData: simplejson.New()}