Storage: Add blob storage interfaces (#90932)

Co-authored-by: Jean-Philippe Quémémer <jeanphilippe.quemener@grafana.com>

pkg/services/apiserver/aggregator/aggregator_test.go

@@ -41,7 +41,7 @@ func TestAggregatorPostStartHooks(t *testing.T) {
 	cfg.GenericConfig.SharedInformerFactory = informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 10*time.Minute)
 
 	// override the RESTOptionsGetter to use the in memory storage options
-	restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(*storagebackend.NewDefaultConfig("memory", nil))
+	restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(*storagebackend.NewDefaultConfig("memory", nil), make(map[string]any))
 	require.NoError(t, err)
 	cfg.GenericConfig.RESTOptionsGetter = restOptionsGetter
 

pkg/services/apiserver/builder/helper.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"time"
 
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -165,6 +166,7 @@ func InstallAPIs(
 	namespaceMapper request.NamespaceMapper,
 	kvStore grafanarest.NamespacedKVStore,
 	serverLock ServerLockService,
+	features featuremgmt.FeatureToggles,
 ) error {
 	// dual writing is only enabled when the storage type is not legacy.
 	// this is needed to support setting a default RESTOptionsGetter for new APIs that don't

pkg/services/apiserver/config.go

@@ -53,6 +53,7 @@ func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o
 	o.StorageOptions.StorageType = options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeLegacy)))
 	o.StorageOptions.DataPath = apiserverCfg.Key("storage_path").MustString(filepath.Join(cfg.DataPath, "grafana-apiserver"))
 	o.StorageOptions.Address = apiserverCfg.Key("address").MustString(o.StorageOptions.Address)
+	o.StorageOptions.BlobStoreURL = apiserverCfg.Key("blob_url").MustString(o.StorageOptions.BlobStoreURL)
 
 	// unified storage configs look like
 	// [unified_storage.<group>.<resource>]

pkg/services/apiserver/options/grafana-aggregator.go

@@ -74,7 +74,7 @@ func (o *GrafanaAggregatorOptions) ApplyTo(aggregatorConfig *aggregatorapiserver
 		return err
 	}
 	// override the RESTOptionsGetter to use the in memory storage options
-	restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig)
+	restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig, make(map[string]any))
 	if err != nil {
 		return err
 	}

pkg/services/apiserver/options/kube-aggregator.go

@@ -81,7 +81,7 @@ func (o *KubeAggregatorOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Co
 		return err
 	}
 	// override the RESTOptionsGetter to use the in memory storage options
-	restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig)
+	restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig, make(map[string]any))
 	if err != nil {
 		return err
 	}

pkg/services/apiserver/options/storage.go

@@ -30,6 +30,13 @@ type StorageOptions struct { // The desired storage type
 	// For file storage, this is the requested path
 	DataPath string
 
+	// Optional blob storage connection string
+	// file:///path/to/dir
+	// gs://my-bucket (using default credentials)
+	// s3://my-bucket?region=us-west-1 (using default credentials)
+	// azblob://my-container
+	BlobStoreURL string
+
 	// {resource}.{group} = 1|2|3|4
 	UnifiedStorageConfig map[string]setting.UnifiedStorageConfig
 }
@@ -59,6 +66,11 @@ func (o *StorageOptions) Validate() []error {
 	if _, _, err := net.SplitHostPort(o.Address); err != nil {
 		errs = append(errs, fmt.Errorf("--grafana-apiserver-storage-address must be a valid network address: %v", err))
 	}
+
+	// Only works for single tenant grafana right now
+	if o.BlobStoreURL != "" && o.StorageType != StorageTypeUnified {
+		errs = append(errs, fmt.Errorf("blob storage is only valid with unified storage"))
+	}
 	return errs
 }
 

pkg/services/apiserver/service.go

@@ -160,7 +160,6 @@ func ProvideService(
 		serverLockService: serverLockService,
 		unified:           unified,
 	}
-
 	// This will be used when running as a dskit service
 	s.BasicService = services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer)
 
@@ -290,9 +289,14 @@ func (s *service) start(ctx context.Context) error {
 			return err
 		}
 	} else {
+		// This is needed as the apistore doesn't allow any core grafana dependencies.
+		features := make(map[string]any)
+		if s.features.IsEnabled(context.Background(), featuremgmt.FlagUnifiedStorageBigObjectsSupport) {
+			features[featuremgmt.FlagUnifiedStorageBigObjectsSupport] = struct{}{}
+		}
 		// Use unified storage client
 		serverConfig.Config.RESTOptionsGetter = apistore.NewRESTOptionsGetterForClient(
-			s.unified, o.RecommendedOptions.Etcd.StorageConfig)
+			s.unified, o.RecommendedOptions.Etcd.StorageConfig, features)
 	}
 
 	// Add OpenAPI specs for each group+version
@@ -319,7 +323,7 @@ func (s *service) start(ctx context.Context) error {
 	// Install the API group+version
 	err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions,
 		// Required for the dual writer initialization
-		s.metrics, request.GetNamespaceMapper(s.cfg), kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"), s.serverLockService,
+		s.metrics, request.GetNamespaceMapper(s.cfg), kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"), s.serverLockService, s.features,
 	)
 	if err != nil {
 		return err

pkg/services/featuremgmt/registry.go

@@ -1516,6 +1516,12 @@ var (
 			Stage:       FeatureStageExperimental,
 			Owner:       grafanaPluginsPlatformSquad,
 		},
+		{
+			Name:        "unifiedStorageBigObjectsSupport",
+			Description: "Enables to save big objects in blob storage",
+			Stage:       FeatureStageExperimental,
+			Owner:       grafanaSearchAndStorageSquad,
+		},
 	}
 )
 

pkg/services/featuremgmt/toggles_gen.csv

@@ -201,3 +201,4 @@ useSessionStorageForRedirection,preview,@grafana/identity-access-team,false,fals
 rolePickerDrawer,experimental,@grafana/identity-access-team,false,false,false
 unifiedStorageSearch,experimental,@grafana/search-and-storage,false,false,false
 pluginsSriChecks,experimental,@grafana/plugins-platform-backend,false,false,false
+unifiedStorageBigObjectsSupport,experimental,@grafana/search-and-storage,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -814,4 +814,8 @@ const (
 	// FlagPluginsSriChecks
 	// Enables SRI checks for plugin assets
 	FlagPluginsSriChecks = "pluginsSriChecks"
+
+	// FlagUnifiedStorageBigObjectsSupport
+	// Enables to save big objects in blob storage
+	FlagUnifiedStorageBigObjectsSupport = "unifiedStorageBigObjectsSupport"
 )

pkg/services/featuremgmt/toggles_gen.json

@@ -3080,6 +3080,31 @@
         "requiresRestart": true
       }
     },
+    {
+      "metadata": {
+        "name": "unifiedStorageBigObjectSupport",
+        "resourceVersion": "1728561321640",
+        "creationTimestamp": "2024-10-10T11:55:21Z",
+        "deletionTimestamp": "2024-10-15T12:09:18Z"
+      },
+      "spec": {
+        "description": "Enables to save big objects in blob storage",
+        "stage": "experimental",
+        "codeowner": "@grafana/search-and-storage"
+      }
+    },
+    {
+      "metadata": {
+        "name": "unifiedStorageBigObjectsSupport",
+        "resourceVersion": "1728994158474",
+        "creationTimestamp": "2024-10-15T12:09:18Z"
+      },
+      "spec": {
+        "description": "Enables to save big objects in blob storage",
+        "stage": "experimental",
+        "codeowner": "@grafana/search-and-storage"
+      }
+    },
     {
       "metadata": {
         "name": "unifiedStorageSearch",