UniStore/Large Objects: Make threshold configurable (#101774)

* Make blob threshold configurable
* Test condition for deconstructing large objects
* Refactor blob threshold naming

pkg/services/apiserver/builder/common.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/kube-openapi/pkg/spec3"
 
 	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
+	"github.com/grafana/grafana/pkg/services/apiserver/options"
 	"github.com/grafana/grafana/pkg/storage/unified/apistore"
 )
 
@@ -73,11 +74,12 @@ type APIGroupPostStartHookProvider interface {
 }
 
 type APIGroupOptions struct {
-	Scheme           *runtime.Scheme
-	OptsGetter       generic.RESTOptionsGetter
-	DualWriteBuilder grafanarest.DualWriteBuilder
-	MetricsRegister  prometheus.Registerer
-	StorageOptions   apistore.StorageOptionsRegister
+	Scheme              *runtime.Scheme
+	OptsGetter          generic.RESTOptionsGetter
+	DualWriteBuilder    grafanarest.DualWriteBuilder
+	MetricsRegister     prometheus.Registerer
+	StorageOptsRegister apistore.StorageOptionsRegister
+	StorageOpts         *options.StorageOptions
 }
 
 // Builders that implement OpenAPIPostProcessor are given a chance to modify the schema directly

pkg/services/apiserver/builder/helper.go

@@ -396,11 +396,12 @@ func InstallAPIs(
 		g := genericapiserver.NewDefaultAPIGroupInfo(group, scheme, metav1.ParameterCodec, codecs)
 		for _, b := range buildersForGroup {
 			if err := b.UpdateAPIGroupInfo(&g, APIGroupOptions{
-				Scheme:           scheme,
-				OptsGetter:       optsGetter,
-				DualWriteBuilder: dualWrite,
-				MetricsRegister:  reg,
-				StorageOptions:   optsregister,
+				Scheme:              scheme,
+				OptsGetter:          optsGetter,
+				DualWriteBuilder:    dualWrite,
+				MetricsRegister:     reg,
+				StorageOptsRegister: optsregister,
+				StorageOpts:         storageOpts,
 			}); err != nil {
 				return err
 			}

pkg/services/apiserver/config.go

@@ -56,6 +56,7 @@ func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o
 	o.StorageOptions.DataPath = apiserverCfg.Key("storage_path").MustString(filepath.Join(cfg.DataPath, "grafana-apiserver"))
 	o.StorageOptions.Address = apiserverCfg.Key("address").MustString(o.StorageOptions.Address)
 	o.StorageOptions.BlobStoreURL = apiserverCfg.Key("blob_url").MustString(o.StorageOptions.BlobStoreURL)
+	o.StorageOptions.BlobThresholdBytes = apiserverCfg.Key("blob_threshold_bytes").MustInt(o.StorageOptions.BlobThresholdBytes)
 
 	// unified storage configs look like
 	// [unified_storage.<group>.<resource>]

pkg/services/apiserver/options/storage.go

@@ -28,6 +28,8 @@ const (
 
 	// Deprecated: legacy is a shim that is no longer necessary
 	StorageTypeLegacy StorageType = "legacy"
+
+	BlobThresholdDefault int = 0
 )
 
 type StorageOptions struct {
@@ -50,6 +52,9 @@ type StorageOptions struct {
 	// s3://my-bucket?region=us-west-1 (using default credentials)
 	// azblob://my-container
 	BlobStoreURL string
+	// Optional blob storage field. When an object's size in bytes exceeds the threshold
+	// value, it is considered large and gets partially stored in blob storage.
+	BlobThresholdBytes int
 
 	// {resource}.{group} = 1|2|3|4
 	UnifiedStorageConfig map[string]setting.UnifiedStorageConfig
@@ -61,6 +66,7 @@ func NewStorageOptions() *StorageOptions {
 		Address:                                "localhost:10000",
 		GrpcClientAuthenticationTokenNamespace: "*",
 		GrpcClientAuthenticationAllowInsecure:  false,
+		BlobThresholdBytes:                     BlobThresholdDefault,
 	}
 }