diff --git a/pkg/infra/tracing/tracing.go b/pkg/infra/tracing/tracing.go index 73140e1c7cb..55835283b9a 100644 --- a/pkg/infra/tracing/tracing.go +++ b/pkg/infra/tracing/tracing.go @@ -50,9 +50,9 @@ type TracingService struct { Propagation string customAttribs []attribute.KeyValue - sampler string - samplerParam float64 - samplerRemoteURL string + Sampler string + SamplerParam float64 + SamplerRemoteURL string log log.Logger @@ -143,9 +143,9 @@ func (ots *TracingService) parseSettings() error { } } legacyTags = section.Key("always_included_tag").MustString("") - ots.sampler = section.Key("sampler_type").MustString("") - ots.samplerParam = section.Key("sampler_param").MustFloat64(1) - ots.samplerRemoteURL = section.Key("sampling_server_url").MustString("") + ots.Sampler = section.Key("sampler_type").MustString("") + ots.SamplerParam = section.Key("sampler_param").MustFloat64(1) + ots.SamplerRemoteURL = section.Key("sampling_server_url").MustString("") } section := ots.Cfg.Raw.Section("tracing.opentelemetry") var err error @@ -158,17 +158,17 @@ func (ots *TracingService) parseSettings() error { // if sampler_type is set in tracing.opentelemetry, we ignore the config in tracing.jaeger sampler := section.Key("sampler_type").MustString("") if sampler != "" { - ots.sampler = sampler + ots.Sampler = sampler } samplerParam := section.Key("sampler_param").MustFloat64(0) if samplerParam != 0 { - ots.samplerParam = samplerParam + ots.SamplerParam = samplerParam } samplerRemoteURL := section.Key("sampling_server_url").MustString("") if samplerRemoteURL != "" { - ots.samplerRemoteURL = samplerRemoteURL + ots.SamplerRemoteURL = samplerRemoteURL } section = ots.Cfg.Raw.Section("tracing.opentelemetry.jaeger") @@ -272,26 +272,26 @@ func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, e } func (ots *TracingService) initSampler() (tracesdk.Sampler, error) { - switch ots.sampler { + switch ots.Sampler { case "const", "": - if ots.samplerParam >= 1 { + if ots.SamplerParam >= 1 { return tracesdk.AlwaysSample(), nil - } else if ots.samplerParam <= 0 { + } else if ots.SamplerParam <= 0 { return tracesdk.NeverSample(), nil } - return nil, fmt.Errorf("invalid param for const sampler - must be 0 or 1: %f", ots.samplerParam) + return nil, fmt.Errorf("invalid param for const sampler - must be 0 or 1: %f", ots.SamplerParam) case "probabilistic": - return tracesdk.TraceIDRatioBased(ots.samplerParam), nil + return tracesdk.TraceIDRatioBased(ots.SamplerParam), nil case "rateLimiting": - return newRateLimiter(ots.samplerParam), nil + return newRateLimiter(ots.SamplerParam), nil case "remote": return jaegerremote.New("grafana", - jaegerremote.WithSamplingServerURL(ots.samplerRemoteURL), - jaegerremote.WithInitialSampler(tracesdk.TraceIDRatioBased(ots.samplerParam)), + jaegerremote.WithSamplingServerURL(ots.SamplerRemoteURL), + jaegerremote.WithInitialSampler(tracesdk.TraceIDRatioBased(ots.SamplerParam)), ), nil default: - return nil, fmt.Errorf("invalid sampler type: %s", ots.sampler) + return nil, fmt.Errorf("invalid sampler type: %s", ots.Sampler) } } diff --git a/pkg/infra/tracing/tracing_test.go b/pkg/infra/tracing/tracing_test.go index eeaf587d8f9..53a56d897a1 100644 --- a/pkg/infra/tracing/tracing_test.go +++ b/pkg/infra/tracing/tracing_test.go @@ -182,9 +182,9 @@ func TestTracingConfig(t *testing.T) { assert.Equal(t, test.ExpectedAttrs, tracer.customAttribs) if test.ExpectedSampler != "" { - assert.Equal(t, test.ExpectedSampler, tracer.sampler) - assert.Equal(t, test.ExpectedSamplerParam, tracer.samplerParam) - assert.Equal(t, test.ExpectedSamplingServerURL, tracer.samplerRemoteURL) + assert.Equal(t, test.ExpectedSampler, tracer.Sampler) + assert.Equal(t, test.ExpectedSamplerParam, tracer.SamplerParam) + assert.Equal(t, test.ExpectedSamplingServerURL, tracer.SamplerRemoteURL) } }) } @@ -196,29 +196,29 @@ func TestInitSampler(t *testing.T) { require.NoError(t, err) assert.Equal(t, "AlwaysOffSampler", sampler.Description()) - otel.sampler = "bogus" + otel.Sampler = "bogus" _, err = otel.initSampler() require.Error(t, err) - otel.sampler = "const" - otel.samplerParam = 0.5 + otel.Sampler = "const" + otel.SamplerParam = 0.5 _, err = otel.initSampler() require.Error(t, err) - otel.sampler = "const" - otel.samplerParam = 1.0 + otel.Sampler = "const" + otel.SamplerParam = 1.0 sampler, err = otel.initSampler() require.NoError(t, err) assert.Equal(t, "AlwaysOnSampler", sampler.Description()) - otel.sampler = "probabilistic" - otel.samplerParam = 0.5 + otel.Sampler = "probabilistic" + otel.SamplerParam = 0.5 sampler, err = otel.initSampler() require.NoError(t, err) assert.Equal(t, "TraceIDRatioBased{0.5}", sampler.Description()) - otel.sampler = "rateLimiting" - otel.samplerParam = 100.25 + otel.Sampler = "rateLimiting" + otel.SamplerParam = 100.25 sampler, err = otel.initSampler() require.NoError(t, err) assert.Equal(t, "RateLimitingSampler{100.25}", sampler.Description()) diff --git a/pkg/plugins/config/tracing.go b/pkg/plugins/config/tracing.go index 8ea7615d332..d59e1fdf23f 100644 --- a/pkg/plugins/config/tracing.go +++ b/pkg/plugins/config/tracing.go @@ -9,6 +9,10 @@ type Tracing struct { type OpenTelemetryCfg struct { Address string Propagation string + + Sampler string + SamplerParam float64 + SamplerRemoteURL string } // IsEnabled returns true if OTLP tracing is enabled (address set) diff --git a/pkg/plugins/envvars/envvars.go b/pkg/plugins/envvars/envvars.go index 8951b49e46a..184ac3f70a9 100644 --- a/pkg/plugins/envvars/envvars.go +++ b/pkg/plugins/envvars/envvars.go @@ -179,6 +179,10 @@ func (s *Service) tracingEnvVars(plugin *plugins.Plugin) []string { vars := []string{ fmt.Sprintf("GF_INSTANCE_OTLP_ADDRESS=%s", s.cfg.Tracing.OpenTelemetry.Address), fmt.Sprintf("GF_INSTANCE_OTLP_PROPAGATION=%s", s.cfg.Tracing.OpenTelemetry.Propagation), + + fmt.Sprintf("GF_INSTANCE_OTLP_SAMPLER_TYPE=%s", s.cfg.Tracing.OpenTelemetry.Sampler), + fmt.Sprintf("GF_INSTANCE_OTLP_SAMPLER_PARAM=%.6f", s.cfg.Tracing.OpenTelemetry.SamplerParam), + fmt.Sprintf("GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL=%s", s.cfg.Tracing.OpenTelemetry.SamplerRemoteURL), } if plugin.Info.Version != "" { vars = append(vars, fmt.Sprintf("GF_PLUGIN_VERSION=%s", plugin.Info.Version)) diff --git a/pkg/plugins/envvars/envvars_test.go b/pkg/plugins/envvars/envvars_test.go index cdee809c0b4..c158e1e2a0f 100644 --- a/pkg/plugins/envvars/envvars_test.go +++ b/pkg/plugins/envvars/envvars_test.go @@ -173,6 +173,11 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) { OpenTelemetry: config.OpenTelemetryCfg{ Address: "127.0.0.1:4317", Propagation: "w3c", + + // Sensible default values for the sampler set by pkg/infra/tracing while reading config.ini + Sampler: "", + SamplerParam: 1.0, + SamplerRemoteURL: "", }, }, PluginSettings: map[string]map[string]string{ @@ -181,12 +186,15 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) { }, plugin: defaultPlugin, exp: func(t *testing.T, envVars []string) { - assert.Len(t, envVars, 5) + assert.Len(t, envVars, 8) assert.Equal(t, "GF_PLUGIN_TRACING=true", envVars[0]) assert.Equal(t, "GF_VERSION=", envVars[1]) assert.Equal(t, "GF_INSTANCE_OTLP_ADDRESS=127.0.0.1:4317", envVars[2]) assert.Equal(t, "GF_INSTANCE_OTLP_PROPAGATION=w3c", envVars[3]) - assert.Equal(t, "GF_PLUGIN_VERSION=1.0.0", envVars[4]) + assert.Equal(t, "GF_INSTANCE_OTLP_SAMPLER_TYPE=", envVars[4]) + assert.Equal(t, "GF_INSTANCE_OTLP_SAMPLER_PARAM=1.000000", envVars[5]) + assert.Equal(t, "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL=", envVars[6]) + assert.Equal(t, "GF_PLUGIN_VERSION=1.0.0", envVars[7]) }, }, { @@ -196,6 +204,11 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) { OpenTelemetry: config.OpenTelemetryCfg{ Address: "127.0.0.1:4317", Propagation: "w3c,jaeger", + + // Sensible default values for the sampler set by pkg/infra/tracing while reading config.ini + Sampler: "", + SamplerParam: 1.0, + SamplerRemoteURL: "", }, }, PluginSettings: map[string]map[string]string{ @@ -204,12 +217,15 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) { }, plugin: defaultPlugin, exp: func(t *testing.T, envVars []string) { - assert.Len(t, envVars, 5) + assert.Len(t, envVars, 8) assert.Equal(t, "GF_PLUGIN_TRACING=true", envVars[0]) assert.Equal(t, "GF_VERSION=", envVars[1]) assert.Equal(t, "GF_INSTANCE_OTLP_ADDRESS=127.0.0.1:4317", envVars[2]) assert.Equal(t, "GF_INSTANCE_OTLP_PROPAGATION=w3c,jaeger", envVars[3]) - assert.Equal(t, "GF_PLUGIN_VERSION=1.0.0", envVars[4]) + assert.Equal(t, "GF_INSTANCE_OTLP_SAMPLER_TYPE=", envVars[4]) + assert.Equal(t, "GF_INSTANCE_OTLP_SAMPLER_PARAM=1.000000", envVars[5]) + assert.Equal(t, "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL=", envVars[6]) + assert.Equal(t, "GF_PLUGIN_VERSION=1.0.0", envVars[7]) }, }, { @@ -297,6 +313,111 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) { plugin: pluginWithoutVersion, exp: expGfPluginVersionNotPresent, }, + { + name: "no sampling (neversample)", + cfg: &config.Cfg{ + Tracing: config.Tracing{ + OpenTelemetry: config.OpenTelemetryCfg{ + Address: "127.0.0.1:4317", + Propagation: "jaeger", + Sampler: "", + SamplerParam: 0.0, + SamplerRemoteURL: "", + }, + }, + PluginSettings: map[string]map[string]string{pluginID: {"tracing": "true"}}, + }, + plugin: defaultPlugin, + exp: func(t *testing.T, envVars []string) { + require.Empty(t, getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_TYPE")) + require.Equal(t, "0.000000", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_PARAM")) + require.Empty(t, getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL")) + }, + }, + { + name: "empty sampler with param", + cfg: &config.Cfg{ + Tracing: config.Tracing{ + OpenTelemetry: config.OpenTelemetryCfg{ + Address: "127.0.0.1:4317", + Propagation: "jaeger", + Sampler: "", + SamplerParam: 0.5, + SamplerRemoteURL: "", + }, + }, + PluginSettings: map[string]map[string]string{pluginID: {"tracing": "true"}}, + }, + plugin: defaultPlugin, + exp: func(t *testing.T, envVars []string) { + require.Equal(t, "", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_TYPE")) + require.Equal(t, "0.500000", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_PARAM")) + require.Equal(t, "", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL")) + }, + }, + { + name: "const sampler with param", + cfg: &config.Cfg{ + Tracing: config.Tracing{ + OpenTelemetry: config.OpenTelemetryCfg{ + Address: "127.0.0.1:4317", + Propagation: "jaeger", + Sampler: "const", + SamplerParam: 0.5, + SamplerRemoteURL: "", + }, + }, + PluginSettings: map[string]map[string]string{pluginID: {"tracing": "true"}}, + }, + plugin: defaultPlugin, + exp: func(t *testing.T, envVars []string) { + require.Equal(t, "const", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_TYPE")) + require.Equal(t, "0.500000", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_PARAM")) + require.Equal(t, "", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL")) + }, + }, + { + name: "rateLimiting sampler", + cfg: &config.Cfg{ + Tracing: config.Tracing{ + OpenTelemetry: config.OpenTelemetryCfg{ + Address: "127.0.0.1:4317", + Propagation: "jaeger", + Sampler: "rateLimiting", + SamplerParam: 0.5, + SamplerRemoteURL: "", + }, + }, + PluginSettings: map[string]map[string]string{pluginID: {"tracing": "true"}}, + }, + plugin: defaultPlugin, + exp: func(t *testing.T, envVars []string) { + require.Equal(t, "rateLimiting", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_TYPE")) + require.Equal(t, "0.500000", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_PARAM")) + require.Equal(t, "", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL")) + }, + }, + { + name: "remote sampler", + cfg: &config.Cfg{ + Tracing: config.Tracing{ + OpenTelemetry: config.OpenTelemetryCfg{ + Address: "127.0.0.1:4317", + Propagation: "jaeger", + Sampler: "remote", + SamplerParam: 0.5, + SamplerRemoteURL: "127.0.0.1:10001", + }, + }, + PluginSettings: map[string]map[string]string{pluginID: {"tracing": "true"}}, + }, + plugin: defaultPlugin, + exp: func(t *testing.T, envVars []string) { + require.Equal(t, "remote", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_TYPE")) + require.Equal(t, "0.500000", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_PARAM")) + require.Equal(t, "127.0.0.1:10001", getEnvVar(envVars, "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL")) + }, + }, } { t.Run(tc.name, func(t *testing.T) { envVarsProvider := NewProvider(tc.cfg, nil) @@ -306,6 +427,32 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) { } } +// getEnvVarWithExists takes a slice of strings in this format: "K=V" (env vars), and returns the "V" where K = wanted. +// If there's no such key, it returns false as the second argument. +func getEnvVarWithExists(vars []string, wanted string) (string, bool) { + for _, v := range vars { + parts := strings.SplitN(v, "=", 2) + if parts[0] != wanted { + continue + } + var r string + if len(parts) < 2 { + r = "" + } else { + r = parts[1] + } + return r, true + } + return "", false +} + +// getEnvVar is like getEnvVarWithExists, but it returns just one string, without the boolean "ok" value. +// If the wanted environment variable does not exist, it returns an empty string. +func getEnvVar(vars []string, wanted string) string { + v, _ := getEnvVarWithExists(vars, wanted) + return v +} + func TestInitializer_authEnvVars(t *testing.T) { t.Run("backend datasource with auth registration", func(t *testing.T) { p := &plugins.Plugin{ diff --git a/pkg/services/pluginsintegration/config/tracing.go b/pkg/services/pluginsintegration/config/tracing.go index 24e3230bd2b..6dcc3b4abdc 100644 --- a/pkg/services/pluginsintegration/config/tracing.go +++ b/pkg/services/pluginsintegration/config/tracing.go @@ -20,8 +20,11 @@ func newTracingCfg(grafanaCfg *setting.Cfg) (pCfg.Tracing, error) { } return pCfg.Tracing{ OpenTelemetry: pCfg.OpenTelemetryCfg{ - Address: ots.Address, - Propagation: ots.Propagation, + Address: ots.Address, + Propagation: ots.Propagation, + Sampler: ots.Sampler, + SamplerParam: ots.SamplerParam, + SamplerRemoteURL: ots.SamplerRemoteURL, }, }, nil }