Tempo: add custom grpc limits

pkg/services/pluginsintegration/pluginconfig/config.go

@@ -77,6 +77,8 @@ type PluginInstanceCfg struct {
 
 	SigV4AuthEnabled    bool
 	SigV4VerboseLogging bool
+
+	LiveClientQueueMaxSize int
 }
 
 // ProvidePluginInstanceConfig returns a new PluginInstanceCfg.
@@ -124,6 +126,7 @@ func ProvidePluginInstanceConfig(cfg *setting.Cfg, settingProvider setting.Provi
 		ResponseLimit:                       cfg.ResponseLimit,
 		SigV4AuthEnabled:                    cfg.SigV4AuthEnabled,
 		SigV4VerboseLogging:                 cfg.SigV4VerboseLogging,
+		LiveClientQueueMaxSize:              cfg.LiveClientQueueMaxSize,
 	}, nil
 }
 

pkg/services/pluginsintegration/pluginconfig/request.go

@@ -198,5 +198,10 @@ func (s *RequestConfigProvider) PluginRequestConfig(ctx context.Context, pluginI
 		m[backend.AppClientSecret] = externalService.ClientSecret
 	}
 
+	if pluginID == "tempo" {
+		const liveClientQueueMaxSize = "GF_LIVE_CLIENT_QUEUE_MAX_SIZE"
+		m[liveClientQueueMaxSize] = strconv.Itoa(s.cfg.LiveClientQueueMaxSize)
+	}
+
 	return m
 }

pkg/services/pluginsintegration/pluginconfig/request_test.go

@@ -259,6 +259,24 @@ func TestRequestConfigProvider_PluginRequestConfig_concurrentQueryCount(t *testi
 	})
 }
 
+func TestRequestConfigProvider_PluginRequestConfig_liveClientQueueMaxSize(t *testing.T) {
+	t.Run("Sets the live client queue max size only for Tempo", func(t *testing.T) {
+		cfg := setting.NewCfg()
+		cfg.LiveClientQueueMaxSize = 123
+
+		pCfg, err := ProvidePluginInstanceConfig(cfg, setting.ProvideProvider(cfg), featuremgmt.WithFeatures())
+		require.NoError(t, err)
+
+		p := NewRequestConfigProvider(pCfg, &fakeSSOSettingsProvider{})
+
+		require.Subset(t, p.PluginRequestConfig(context.Background(), "tempo", nil), map[string]string{
+			"GF_LIVE_CLIENT_QUEUE_MAX_SIZE": "123",
+		})
+
+		require.NotContains(t, p.PluginRequestConfig(context.Background(), "prometheus", nil), "GF_LIVE_CLIENT_QUEUE_MAX_SIZE")
+	})
+}
+
 func TestRequestConfigProvider_PluginRequestConfig_azureAuthEnabled(t *testing.T) {
 	t.Run("Uses the configured azureAuthEnabled", func(t *testing.T) {
 		cfg := &PluginInstanceCfg{

pkg/tsdb/tempo/grpc.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"google.golang.org/grpc/metadata"
@@ -88,7 +89,21 @@ func getDialOpts(ctx context.Context, settings backend.DataSourceInstanceSetting
 
 	var dialOps []grpc.DialOption
 
-	dialOps = append(dialOps, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(100*1024*1024)))
+	// Default max gRPC receive size is 4MB. Tempo responses can exceed this, so we should increase it.
+	// Prefer `GF_LIVE_CLIENT_QUEUE_MAX_SIZE` (set by Grafana for the Tempo plugin) when it's present and valid.
+	const defaultMaxCallRecvMsgSizeBytes = 4 * 1024 * 1024
+	maxCallRecvMsgSizeBytes := defaultMaxCallRecvMsgSizeBytes
+
+	if v := backend.GrafanaConfigFromContext(ctx).Get("GF_LIVE_CLIENT_QUEUE_MAX_SIZE"); v != "" {
+		parsed, err := strconv.Atoi(v)
+		if err != nil || parsed <= 0 {
+			logger.Debug("Invalid GF_LIVE_CLIENT_QUEUE_MAX_SIZE; using default gRPC max receive size", "value", v, "default", defaultMaxCallRecvMsgSizeBytes, "error", err)
+		} else {
+			maxCallRecvMsgSizeBytes = parsed
+		}
+	}
+
+	dialOps = append(dialOps, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxCallRecvMsgSizeBytes)))
 	dialOps = append(dialOps, grpc.WithChainStreamInterceptor(CustomHeadersStreamInterceptor(opts)))
 	if settings.BasicAuthEnabled {
 		// If basic authentication is enabled, it uses TLS transport credentials and sets the basic authentication header for each RPC call.