UniStore/Large Objects: Make threshold configurable (#101774)

* Make blob threshold configurable
* Test condition for deconstructing large objects
* Refactor blob threshold naming

pkg/storage/unified/apistore/fake_large.go

@@ -0,0 +1,38 @@
+package apistore
+
+import (
+	"context"
+
+	dashboardv1alpha1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1alpha1"
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
+	"github.com/grafana/grafana/pkg/storage/unified/resource"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type LargeObjectSupportFake struct {
+	threshold     int
+	deconstructed bool
+	reconstructed bool
+}
+
+func (s *LargeObjectSupportFake) GroupResource() schema.GroupResource {
+	return dashboardv1alpha1.DashboardResourceInfo.GroupResource()
+}
+
+func (s *LargeObjectSupportFake) Threshold() int {
+	return s.threshold
+}
+
+func (s *LargeObjectSupportFake) MaxSize() int {
+	return 10 * 1024 * 1024
+}
+
+func (s *LargeObjectSupportFake) Deconstruct(ctx context.Context, key *resource.ResourceKey, client resource.BlobStoreClient, obj utils.GrafanaMetaAccessor, raw []byte) error {
+	s.deconstructed = true
+	return nil
+}
+
+func (s *LargeObjectSupportFake) Reconstruct(ctx context.Context, key *resource.ResourceKey, client resource.BlobStoreClient, obj utils.GrafanaMetaAccessor) error {
+	s.reconstructed = true
+	return nil
+}

pkg/storage/unified/apistore/large.go

@@ -38,8 +38,8 @@ var _ LargeObjectSupport = (*BasicLargeObjectSupport)(nil)
 
 type BasicLargeObjectSupport struct {
 	TheGroupResource schema.GroupResource
-	ThresholdSize    int
-	MaxByteSize      int
+	ThresholdBytes   int
+	MaxBytes         int
 
 	// Mutate the spec so it only has the small properties
 	ReduceSpec func(obj runtime.Object) error
@@ -55,12 +55,12 @@ func (s *BasicLargeObjectSupport) GroupResource() schema.GroupResource {
 
 // Threshold implements LargeObjectSupport.
 func (s *BasicLargeObjectSupport) Threshold() int {
-	return s.ThresholdSize
+	return s.ThresholdBytes
 }
 
 // MaxSize implements LargeObjectSupport.
 func (s *BasicLargeObjectSupport) MaxSize() int {
-	return s.MaxByteSize
+	return s.MaxBytes
 }
 
 // Deconstruct implements LargeObjectSupport.

pkg/storage/unified/apistore/prepare.go

@@ -175,12 +175,10 @@ func (s *Storage) prepareObjectForUpdate(ctx context.Context, updateObject runti
 
 func (s *Storage) handleLargeResources(ctx context.Context, obj utils.GrafanaMetaAccessor, buf bytes.Buffer) ([]byte, error) {
 	support := s.opts.LargeObjectSupport
-	if support != nil {
-		size := buf.Len()
-		if size > support.Threshold() {
-			if support.MaxSize() > 0 && size > support.MaxSize() {
-				return nil, fmt.Errorf("request object is too big (%s > %s)", formatBytes(size), formatBytes(support.MaxSize()))
-			}
+	size := buf.Len()
+	if support != nil && size > support.Threshold() {
+		if support.MaxSize() > 0 && size > support.MaxSize() {
+			return nil, fmt.Errorf("request object is too big (%s > %s)", formatBytes(size), formatBytes(support.MaxSize()))
 		}
 
 		key := &resource.ResourceKey{

pkg/storage/unified/apistore/prepare_test.go

@@ -297,3 +297,49 @@ func getPreparedObject(t *testing.T, ctx context.Context, s *Storage, obj runtim
 	require.NoError(t, err)
 	return meta
 }
+
+func TestPrepareLargeObjectForStorage(t *testing.T) {
+	_ = v1alpha1.AddToScheme(scheme)
+	node, err := snowflake.NewNode(rand.Int63n(1024))
+	require.NoError(t, err)
+
+	ctx := authtypes.WithAuthInfo(context.Background(), &identity.StaticRequester{UserID: 1, UserUID: "user-uid", Type: authtypes.TypeUser})
+
+	dashboard := v1alpha1.Dashboard{}
+	dashboard.Name = "test-name"
+	t.Run("Should deconstruct object if size is over threshold", func(t *testing.T) {
+		los := LargeObjectSupportFake{
+			threshold: 0,
+		}
+
+		f := &Storage{
+			codec:     apitesting.TestCodec(codecs, v1alpha1.DashboardResourceInfo.GroupVersion()),
+			snowflake: node,
+			opts: StorageOptions{
+				LargeObjectSupport: &los,
+			},
+		}
+
+		_, err := f.prepareObjectForStorage(ctx, dashboard.DeepCopyObject())
+		require.Nil(t, err)
+		require.True(t, los.deconstructed)
+	})
+
+	t.Run("Should not deconstruct object if size is under threshold", func(t *testing.T) {
+		los := LargeObjectSupportFake{
+			threshold: 1000,
+		}
+
+		f := &Storage{
+			codec:     apitesting.TestCodec(codecs, v1alpha1.DashboardResourceInfo.GroupVersion()),
+			snowflake: node,
+			opts: StorageOptions{
+				LargeObjectSupport: &los,
+			},
+		}
+
+		_, err := f.prepareObjectForStorage(ctx, dashboard.DeepCopyObject())
+		require.Nil(t, err)
+		require.False(t, los.deconstructed)
+	})
+}

pkg/storage/unified/client.go

@@ -55,10 +55,11 @@ func ProvideUnifiedStorageClient(opts *Options, storageMetrics *resource.Storage
 	// See: apiserver.ApplyGrafanaConfig(cfg, features, o)
 	apiserverCfg := opts.Cfg.SectionWithEnvOverrides("grafana-apiserver")
 	client, err := newClient(options.StorageOptions{
-		StorageType:  options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeUnified))),
-		DataPath:     apiserverCfg.Key("storage_path").MustString(filepath.Join(opts.Cfg.DataPath, "grafana-apiserver")),
-		Address:      apiserverCfg.Key("address").MustString(""), // client address
-		BlobStoreURL: apiserverCfg.Key("blob_url").MustString(""),
+		StorageType:        options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeUnified))),
+		DataPath:           apiserverCfg.Key("storage_path").MustString(filepath.Join(opts.Cfg.DataPath, "grafana-apiserver")),
+		Address:            apiserverCfg.Key("address").MustString(""), // client address
+		BlobStoreURL:       apiserverCfg.Key("blob_url").MustString(""),
+		BlobThresholdBytes: apiserverCfg.Key("blob_threshold_bytes").MustInt(options.BlobThresholdDefault),
 	}, opts.Cfg, opts.Features, opts.DB, opts.Tracer, opts.Reg, opts.Authzc, opts.Docs, storageMetrics, indexMetrics)
 	if err == nil {
 		// Used to get the folder stats