diff --git a/devenv/datasources.yaml b/devenv/datasources.yaml index beac2afc408..0cfa7810670 100644 --- a/devenv/datasources.yaml +++ b/devenv/datasources.yaml @@ -311,3 +311,13 @@ datasources: spanEndTimeShift: '-5m' customQuery: true query: '{filename="/var/log/grafana/grafana.log"} |="$${__span.traceId}"' + tracesToProfiles: + datasourceUid: gdev-pyroscope + profileTypeId: "process_cpu:cpu:nanoseconds:cpu:nanoseconds" + + - name: gdev-pyroscope + type: grafana-pyroscope-datasource + uid: gdev-pyroscope + access: proxy + url: http://localhost:4040 + editable: false diff --git a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml index 13fca15e3c3..c7583cd62d3 100644 --- a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml +++ b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml @@ -36,6 +36,8 @@ image: "grafana/pyroscope:latest" ports: - "4040:4040" + extra_hosts: + - "host.docker.internal:host-gateway" agent: image: grafana/agent:v0.37.0 environment: diff --git a/go.mod b/go.mod index 37150bb5183..82bf8585622 100644 --- a/go.mod +++ b/go.mod @@ -106,6 +106,7 @@ require ( // For local development grafana/grafana will always use the local files // Check go.work file for details github.com/grafana/grafana/pkg/promlib v0.0.6 // @grafana/observability-metrics + github.com/grafana/otel-profiling-go v0.5.1 // @grafana/grafana-backend-group github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // @grafana/observability-traces-and-profiling github.com/grafana/pyroscope/api v0.3.0 // @grafana/observability-traces-and-profiling github.com/grafana/tempo v1.5.1-0.20230524121406-1dc1bfe7085b // @grafana/observability-traces-and-profiling @@ -242,6 +243,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect @@ -282,6 +284,7 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/emicklei/proto v1.10.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect @@ -343,6 +346,7 @@ require ( github.com/jonboulle/clockwork v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect + github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.8 // indirect @@ -457,12 +461,6 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect; @grafana-app-platform-squad ) -require ( - github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect - github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect - github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 // indirect -) - // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240523142256-cc370b98af7c diff --git a/go.sum b/go.sum index c8f09ac6e64..8e400cb8e1f 100644 --- a/go.sum +++ b/go.sum @@ -2344,6 +2344,8 @@ github.com/grafana/grafana/pkg/promlib v0.0.6 h1:FuRyHMIgVVXkLuJnCflNfk3gqJflmyi github.com/grafana/grafana/pkg/promlib v0.0.6/go.mod h1:shFkrG1fQ/PPNRGhxAPNMLp0SAeG/jhqaLoG6n2191M= github.com/grafana/grafana/pkg/util/xorm v0.0.1 h1:72QZjxWIWpSeOF8ob4aMV058kfgZyeetkAB8dmeti2o= github.com/grafana/grafana/pkg/util/xorm v0.0.1/go.mod h1:eNfbB9f2jM8o9RfwqwjY8SYm5tvowJ8Ly+iE4P9rXII= +github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= +github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= github.com/grafana/prometheus-alertmanager v0.25.1-0.20240422145632-c33c6b5b6e6b h1:HCbWyVL6vi7gxyO76gQksSPH203oBJ1MJ3JcG1OQlsg= github.com/grafana/prometheus-alertmanager v0.25.1-0.20240422145632-c33c6b5b6e6b/go.mod h1:01sXtHoRwI8W324IPAzuxDFOmALqYLCOhvSC2fUHWXc= github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= diff --git a/pkg/infra/tracing/tracing.go b/pkg/infra/tracing/tracing.go index 1f0f7591957..3f09c215a77 100644 --- a/pkg/infra/tracing/tracing.go +++ b/pkg/infra/tracing/tracing.go @@ -272,6 +272,8 @@ func (ots *TracingService) initOpentelemetryTracer() error { } } + tp = NewProfilingTracerProvider(tp) + // Register our TracerProvider as the global so any imported // instrumentation in the future will default to using it // only if tracing is enabled diff --git a/pkg/infra/tracing/tracing_profiling.go b/pkg/infra/tracing/tracing_profiling.go new file mode 100644 index 00000000000..73fea1da7d8 --- /dev/null +++ b/pkg/infra/tracing/tracing_profiling.go @@ -0,0 +1,27 @@ +package tracing + +import ( + "context" + + otelpyroscope "github.com/grafana/otel-profiling-go" + trace "go.opentelemetry.io/otel/trace" +) + +type profilingTracerProvider struct { + trace.TracerProvider + wrappedTp tracerProvider +} + +// NewProfilingTracerProvider creates a new tracer provider that annotates pprof +// samples with span_id label. This allows to establish a relationship +// between pprof profiles and reported tracing spans. +func NewProfilingTracerProvider(tp tracerProvider) tracerProvider { + return &profilingTracerProvider{ + TracerProvider: otelpyroscope.NewTracerProvider(tp), + wrappedTp: tp, + } +} + +func (tp *profilingTracerProvider) Shutdown(ctx context.Context) error { + return tp.wrappedTp.Shutdown(ctx) +}