Secrets: Implement secrets manager plugin fallback store (#54496)

* Refactor fallback to be isolated to plugin secret store

* Check for error value on replace fallback test helper

* Move ResetPlugin from test_helpers.go to plugin.go

* Add check to GetUnwrappedStoreFromCache

* Add fallback GetAll query to WithFallbackEnabled

* Add mutex lock to WithFallbackEnabled

* Add cache to fallback store

* Fix linter issues

* Fix linter issues

* Fix linter issues

pkg/services/secrets/kvstore/cache.go

@@ -2,6 +2,7 @@ package kvstore
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -9,13 +10,15 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 )
 
+var errSecretStoreIsNotCached = errors.New("SecretsKVStore is not a CachedKVStore")
+
 type CachedKVStore struct {
 	log   log.Logger
 	cache *localcache.CacheService
 	store SecretsKVStore
 }
 
-func NewCachedKVStore(store SecretsKVStore, defaultExpiration time.Duration, cleanupInterval time.Duration) *CachedKVStore {
+func WithCache(store SecretsKVStore, defaultExpiration time.Duration, cleanupInterval time.Duration) *CachedKVStore {
 	return &CachedKVStore{
 		log:   log.New("secrets.kvstore"),
 		cache: localcache.New(defaultExpiration, cleanupInterval),
@@ -81,14 +84,9 @@ func (kv *CachedKVStore) GetAll(ctx context.Context) ([]Item, error) {
 	return kv.store.GetAll(ctx)
 }
 
-func (kv *CachedKVStore) Fallback() SecretsKVStore {
-	return kv.store.Fallback()
-}
-
-func (kv *CachedKVStore) SetFallback(store SecretsKVStore) error {
-	return kv.store.SetFallback(store)
-}
-
-func (kv *CachedKVStore) GetUnwrappedStore() SecretsKVStore {
-	return kv.store
+func GetUnwrappedStoreFromCache(kv SecretsKVStore) (SecretsKVStore, error) {
+	if cache, ok := kv.(*CachedKVStore); ok {
+		return cache.store, nil
+	}
+	return nil, errSecretStoreIsNotCached
 }