diff --git a/conf/defaults.ini b/conf/defaults.ini
index 6b640bf452a..4409523309f 100644
--- a/conf/defaults.ini
+++ b/conf/defaults.ini
@@ -1039,6 +1039,11 @@ zipkin_propagation = false
# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
disable_shared_zipkin_spans = false
+[tracing.opentelemetry]
+
+# attributes that will always be included in when creating new spans. ex (key1:value1,key2:value2)
+custom_attributes =
+
[tracing.opentelemetry.jaeger]
# jaeger destination (ex http://localhost:14268/api/traces)
address =
diff --git a/conf/sample.ini b/conf/sample.ini
index 8a60c1758b3..4bf3e78c300 100644
--- a/conf/sample.ini
+++ b/conf/sample.ini
@@ -1008,6 +1008,10 @@
# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
;disable_shared_zipkin_spans = false
+[tracing.opentelemetry]
+# attributes that will always be included in when creating new spans. ex (key1:value1,key2:value2)
+;custom_attributes = key1:value1,key2:value2
+
[tracing.opentelemetry.jaeger]
# jaeger destination (ex http://localhost:14268/api/traces)
; address = http://localhost:14268/api/traces
diff --git a/docs/sources/setup-grafana/configure-grafana/_index.md b/docs/sources/setup-grafana/configure-grafana/_index.md
index ece945a6895..d1a7e7850eb 100644
--- a/docs/sources/setup-grafana/configure-grafana/_index.md
+++ b/docs/sources/setup-grafana/configure-grafana/_index.md
@@ -1571,6 +1571,18 @@ Setting this to `true` turns off shared RPC spans. Leaving this available is the
+## [tracing.opentelemetry]
+
+Configure general parameters shared between OpenTelemetry providers.
+
+### custom_attributes
+
+Comma-separated list of attributes to include in all new spans, such as `key1:value1,key2:value2`.
+
+Can be set with the environment variable `OTEL_RESOURCE_ATTRIBUTES` (use `=` instead of `:` with the environment variable).
+
+
+
## [tracing.opentelemetry.jaeger]
Configure Grafana's Jaeger client for distributed tracing.
diff --git a/pkg/infra/tracing/opentelemetry_tracing.go b/pkg/infra/tracing/opentelemetry_tracing.go
index b30b4240b14..bfa9dfa90b1 100644
--- a/pkg/infra/tracing/opentelemetry_tracing.go
+++ b/pkg/infra/tracing/opentelemetry_tracing.go
@@ -2,7 +2,9 @@ package tracing
import (
"context"
+ "fmt"
"net/http"
+ "strings"
"time"
"github.com/go-kit/log/level"
@@ -48,10 +50,11 @@ type Span interface {
}
type Opentelemetry struct {
- enabled string
- address string
- propagation string
- log log.Logger
+ enabled string
+ address string
+ propagation string
+ customAttribs []attribute.KeyValue
+ log log.Logger
tracerProvider tracerProvider
tracer trace.Tracer
@@ -89,7 +92,17 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error {
}
func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
- section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
+ section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry")
+ if err != nil {
+ return err
+ }
+
+ ots.customAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(""))
+ if err != nil {
+ return err
+ }
+
+ section, err = ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
if err != nil {
return err
}
@@ -115,6 +128,22 @@ func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
return nil
}
+func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
+ res := []attribute.KeyValue{}
+
+ attribs := strings.Split(s, ",")
+ for _, v := range attribs {
+ parts := strings.SplitN(v, ":", 2)
+ if len(parts) > 1 {
+ res = append(res, attribute.String(parts[0], parts[1]))
+ } else if v != "" {
+ return nil, fmt.Errorf("custom attribute malformed - must be in 'key:value' form: %q", v)
+ }
+ }
+
+ return res, nil
+}
+
func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
// Create the Jaeger exporter
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
@@ -122,13 +151,23 @@ func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider,
return nil, err
}
- tp := tracesdk.NewTracerProvider(
- tracesdk.WithBatcher(exp),
- tracesdk.WithResource(resource.NewWithAttributes(
- semconv.SchemaURL,
+ res, err := resource.New(
+ context.Background(),
+ resource.WithAttributes(
+ // TODO: why are these attributes different from ones added to the
+ // OTLP provider?
semconv.ServiceNameKey.String("grafana"),
attribute.String("environment", "production"),
- )),
+ ),
+ resource.WithAttributes(ots.customAttribs...),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ tp := tracesdk.NewTracerProvider(
+ tracesdk.WithBatcher(exp),
+ tracesdk.WithResource(res),
)
return tp, nil
@@ -147,6 +186,7 @@ func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, er
semconv.ServiceNameKey.String("grafana"),
semconv.ServiceVersionKey.String(version.Version),
),
+ resource.WithAttributes(ots.customAttribs...),
resource.WithProcessRuntimeDescription(),
resource.WithTelemetrySDK(),
)
diff --git a/pkg/infra/tracing/optentelemetry_tracing_test.go b/pkg/infra/tracing/optentelemetry_tracing_test.go
new file mode 100644
index 00000000000..01e817da776
--- /dev/null
+++ b/pkg/infra/tracing/optentelemetry_tracing_test.go
@@ -0,0 +1,88 @@
+package tracing
+
+import (
+ "testing"
+
+ "github.com/grafana/grafana/pkg/setting"
+ "github.com/stretchr/testify/assert"
+ "go.opentelemetry.io/otel/attribute"
+)
+
+func TestSplitCustomAttribs(t *testing.T) {
+ tests := []struct {
+ input string
+ expected []attribute.KeyValue
+ }{
+ {
+ input: "key1:value:1",
+ expected: []attribute.KeyValue{attribute.String("key1", "value:1")},
+ },
+ {
+ input: "key1:value1,key2:value2",
+ expected: []attribute.KeyValue{
+ attribute.String("key1", "value1"),
+ attribute.String("key2", "value2"),
+ },
+ },
+ {
+ input: "",
+ expected: []attribute.KeyValue{},
+ },
+ }
+
+ for _, test := range tests {
+ attribs, err := splitCustomAttribs(test.input)
+ assert.NoError(t, err)
+ assert.EqualValues(t, test.expected, attribs)
+ }
+}
+
+func TestSplitCustomAttribs_Malformed(t *testing.T) {
+ tests := []struct {
+ input string
+ expected []attribute.KeyValue
+ }{
+ {input: "key1=value1"},
+ {input: "key1"},
+ }
+
+ for _, test := range tests {
+ _, err := splitCustomAttribs(test.input)
+ assert.Error(t, err)
+ }
+}
+
+func TestOptentelemetry_ParseSettingsOpentelemetry(t *testing.T) {
+ cfg := setting.NewCfg()
+ otel := &Opentelemetry{Cfg: cfg}
+
+ otelsect := cfg.Raw.Section("tracing.opentelemetry")
+ jaegersect := cfg.Raw.Section("tracing.opentelemetry.jaeger")
+ otlpsect := cfg.Raw.Section("tracing.opentelemetry.otlp")
+
+ assert.NoError(t, otel.parseSettingsOpentelemetry())
+ assert.Equal(t, noopExporter, otel.enabled)
+
+ otelsect.Key("custom_attributes")
+ assert.NoError(t, otel.parseSettingsOpentelemetry())
+ assert.Empty(t, otel.customAttribs)
+
+ otelsect.Key("custom_attributes").SetValue("key1:value1,key2:value2")
+ assert.NoError(t, otel.parseSettingsOpentelemetry())
+ expected := []attribute.KeyValue{
+ attribute.String("key1", "value1"),
+ attribute.String("key2", "value2"),
+ }
+ assert.Equal(t, expected, otel.customAttribs)
+
+ jaegersect.Key("address").SetValue("somehost:6831")
+ assert.NoError(t, otel.parseSettingsOpentelemetry())
+ assert.Equal(t, "somehost:6831", otel.address)
+ assert.Equal(t, jaegerExporter, otel.enabled)
+
+ jaegersect.Key("address").SetValue("")
+ otlpsect.Key("address").SetValue("somehost:4317")
+ assert.NoError(t, otel.parseSettingsOpentelemetry())
+ assert.Equal(t, "somehost:4317", otel.address)
+ assert.Equal(t, otlpExporter, otel.enabled)
+}