From 9453cedb517c20855cb64c0a73b5df93a8436ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Bedi?= Date: Thu, 14 Aug 2025 12:34:41 +0200 Subject: [PATCH] Update self instrumentation to use alloy and bump versions (#109266) * Update self instrumentation to use alloy and bump versions * Update devenv/docker/blocks/self-instrumentation/config.alloy Co-authored-by: Sam DeHaan * Add remote write receiver feature to Prometheus configuration in docker-compose.yaml * Add example for plugin profiling to the doc --------- Co-authored-by: Sam DeHaan --- .../blocks/self-instrumentation/agent.flow | 87 --------------- .../blocks/self-instrumentation/config.alloy | 100 ++++++++++++++++++ .../self-instrumentation/docker-compose.yaml | 60 +++++------ .../self-instrumentation/prometheus.yaml | 6 +- .../blocks/self-instrumentation/promtail.yaml | 18 ---- .../blocks/self-instrumentation/readme.md | 21 +++- .../blocks/self-instrumentation/tempo.yaml | 45 +++++--- 7 files changed, 179 insertions(+), 158 deletions(-) delete mode 100644 devenv/docker/blocks/self-instrumentation/agent.flow create mode 100644 devenv/docker/blocks/self-instrumentation/config.alloy delete mode 100644 devenv/docker/blocks/self-instrumentation/promtail.yaml diff --git a/devenv/docker/blocks/self-instrumentation/agent.flow b/devenv/docker/blocks/self-instrumentation/agent.flow deleted file mode 100644 index d1deb12b5aa..00000000000 --- a/devenv/docker/blocks/self-instrumentation/agent.flow +++ /dev/null @@ -1,87 +0,0 @@ -loki.write "default" { - endpoint { - url = "http://loki:3100/loki/api/v1/push" - } - external_labels = {} -} - -logging { - level = "debug" - format = "json" - write_to = [loki.write.default.receiver] -} - -tracing { - sampling_fraction = 1 - write_to = [otelcol.exporter.otlp.tempo.input] -} - -otelcol.exporter.otlp "tempo" { - client { - endpoint = "tempo:4317" - tls { - insecure = true - insecure_skip_verify = true - } - } -} - -faro.receiver "frontend" { - server { - listen_address = "0.0.0.0" - listen_port = 12347 - cors_allowed_origins = ["*"] - } - - output { - logs = [loki.write.default.receiver] - traces = [otelcol.exporter.otlp.tempo.input] - } -} - -pyroscope.write "default" { - endpoint { - url = "http://pyroscope:4040" - } -} - -pyroscope.scrape "default" { - targets = [ - {"__address__" = "host.docker.internal:6000", "service_name"="grafana"}, - ] - forward_to = [pyroscope.write.default.receiver] - - profiling_config { - profile.process_cpu { - enabled = true - } - - profile.godeltaprof_memory { - enabled = true - } - - profile.memory { // disable memory, use godeltaprof_memory instead - enabled = false - } - - profile.godeltaprof_mutex { - enabled = true - } - - profile.mutex { // disable mutex, use godeltaprof_mutex instead - enabled = false - } - - profile.godeltaprof_block { - enabled = true - } - - profile.block { // disable block, use godeltaprof_block instead - enabled = false - } - - profile.goroutine { - enabled = true - } - } -} diff --git a/devenv/docker/blocks/self-instrumentation/config.alloy b/devenv/docker/blocks/self-instrumentation/config.alloy new file mode 100644 index 00000000000..bf480ddf4b8 --- /dev/null +++ b/devenv/docker/blocks/self-instrumentation/config.alloy @@ -0,0 +1,100 @@ +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} + +// Grafana log file discovery +local.file_match "grafana_logs" { + path_targets = [ + { + __path__ = "/var/log/grafana/*.log", + }, + ] +} + +// Grafana log source +loki.source.file "grafana" { + targets = local.file_match.grafana_logs.targets + forward_to = [loki.relabel.grafana.receiver] +} + +// Add labels to Grafana logs +loki.relabel "grafana" { + rule { + action = "replace" + target_label = "job" + replacement = "grafana" + } + + rule { + action = "replace" + target_label = "service" + replacement = "grafana" + } + + forward_to = [loki.write.default.receiver] +} + +logging { + level = "debug" + format = "json" + // Forward internal logs to the local Loki instance. + write_to = [loki.write.default.receiver] +} + +tracing { + sampling_fraction = 1.0 + + // Forward internal spans to the local Tempo instance. + write_to = [otelcol.exporter.otlp.tempo.input] +} + +otelcol.receiver.otlp "otlp_receiver" { + grpc { + endpoint = "0.0.0.0:4317" + } + + output { + traces = [otelcol.exporter.otlp.tempo.input] + } +} + +otelcol.exporter.otlp "tempo" { + client { + endpoint = "tempo:4317" + tls { + insecure = true + } + } +} + +pyroscope.write "default" { + // Send profiles to a locally running Pyroscope instance. + endpoint { + url = "http://pyroscope:4040" + } +} + + +pyroscope.scrape "default" { + targets = [ + {"__address__" = "pyroscope:4040", "service_name"="pyroscope"}, + {"__address__" = "alloy:12345", "service_name"="alloy"}, + {"__address__" = "host.docker.internal:6000", "service_name"="grafana"}, + ] + forward_to = [pyroscope.write.default.receiver] +} + +faro.receiver "frontend" { + server { + listen_address = "0.0.0.0" + listen_port = 12347 + cors_allowed_origins = ["*"] + } + + output { + logs = [loki.write.default.receiver] + traces = [otelcol.exporter.otlp.tempo.input] + } +} diff --git a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml index c7583cd62d3..fb7f7b62090 100644 --- a/devenv/docker/blocks/self-instrumentation/docker-compose.yaml +++ b/devenv/docker/blocks/self-instrumentation/docker-compose.yaml @@ -1,56 +1,54 @@ prometheus: - image: prom/prometheus:v2.45.0 + image: prom/prometheus:v3.5.0 ports: - - "9090:9090" + - '9090:9090' extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' command: > - --enable-feature=remote-write-receiver --enable-feature=exemplar-storage --enable-feature=native-histograms + --web.enable-remote-write-receiver --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus volumes: - ./docker/blocks/self-instrumentation/prometheus.yaml:/etc/prometheus/prometheus.yml loki: - image: grafana/loki:2.8.2 + image: grafana/loki:3.5.3 ports: - - "3100:3100" + - '3100:3100' command: -config.file=/etc/loki/local-config.yaml - promtail: - image: grafana/promtail:2.8.2 - command: -config.file=/etc/promtail/promtail.yaml - volumes: - - ./docker/blocks/self-instrumentation/promtail.yaml:/etc/promtail/promtail.yaml - - ../data/log:/var/log/grafana tempo: - image: grafana/tempo:2.1.1 + image: grafana/tempo:2.8.1 command: -config.file=/etc/tempo.yaml volumes: - ./docker/blocks/self-instrumentation/tempo.yaml:/etc/tempo.yaml ports: - - "14268:14268" - - "3200:3200" - - "4317:4317" + - '14268:14268' + - '3200:3200' + - '4317:4317' pyroscope: - image: "grafana/pyroscope:latest" + image: 'grafana/pyroscope:1.14.0' ports: - - "4040:4040" + - '4040:4040' extra_hosts: - - "host.docker.internal:host-gateway" - agent: - image: grafana/agent:v0.37.0 - environment: - - AGENT_MODE=flow - entrypoint: - - /bin/grafana-agent + - 'host.docker.internal:host-gateway' + alloy: + image: grafana/alloy:v1.10.0 + command: - run + - /etc/alloy/config.alloy + - --storage.path=/var/lib/alloy/data - --server.http.listen-addr=0.0.0.0:12345 - - /etc/agent/config.river - volumes: - - ./docker/blocks/self-instrumentation/agent.flow:/etc/agent/config.river + - --stability.level=experimental # Enable all functionality ports: - - "12345:12345" - - "12347:12347" + - '12345:12345' + volumes: + - ./docker/blocks/self-instrumentation/config.alloy:/etc/alloy/config.alloy + - ../data/log:/var/log/grafana:ro # Mount Grafana logs directory + depends_on: + - tempo + - loki + - prometheus + - pyroscope extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' diff --git a/devenv/docker/blocks/self-instrumentation/prometheus.yaml b/devenv/docker/blocks/self-instrumentation/prometheus.yaml index 5533f0c0fd6..76db0c65b89 100644 --- a/devenv/docker/blocks/self-instrumentation/prometheus.yaml +++ b/devenv/docker/blocks/self-instrumentation/prometheus.yaml @@ -1,10 +1,10 @@ global: - scrape_interval: 15s + scrape_interval: 15s evaluation_interval: 15s scrape_configs: - job_name: grafana - scrape_classic_histograms: true + always_scrape_classic_histograms: true static_configs: - targets: - - host.docker.internal:3000 + - host.docker.internal:3000 diff --git a/devenv/docker/blocks/self-instrumentation/promtail.yaml b/devenv/docker/blocks/self-instrumentation/promtail.yaml deleted file mode 100644 index eb7787ef789..00000000000 --- a/devenv/docker/blocks/self-instrumentation/promtail.yaml +++ /dev/null @@ -1,18 +0,0 @@ -server: - http_listen_port: 9080 - grpc_listen_port: 0 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://loki:3100/loki/api/v1/push - -scrape_configs: -- job_name: grafana - static_configs: - - targets: - - localhost - labels: - job: grafana - __path__: /var/log/grafana/*log diff --git a/devenv/docker/blocks/self-instrumentation/readme.md b/devenv/docker/blocks/self-instrumentation/readme.md index 07fc5856cf6..c7fe04139b4 100644 --- a/devenv/docker/blocks/self-instrumentation/readme.md +++ b/devenv/docker/blocks/self-instrumentation/readme.md @@ -18,8 +18,9 @@ format = json enabled = true custom_endpoint=http://localhost:12347/collect -[tracing.opentelemetry.jaeger] -address = http://localhost:14268/api/traces +[tracing.opentelemetry.otlp] +address = localhost:4317 +insecure = true ``` To collect profiles with pyroscope, you need to run Grafana with the following env vars: @@ -29,3 +30,19 @@ export GF_DIAGNOSTICS_PROFILING_ENABLED=true export GF_DIAGNOSTICS_PROFILING_ADDR=0.0.0.0 make run ``` + +To enable profiling in your plugin add the following to your `custom.ini`: + +```ini +[plugin.grafana-bigquery-datasource] +profiling_enabled = true +profiling_port = 6161 +profiling_block_rate = 1 +profiling_mutex_rate = 1 +``` + +and the following to your `config.alloy` in the `pyroscope.scrape.targets` section: + +```yaml + {"__address__" = "host.docker.internal:6161", "service_name"="grafana-bigquery-datasource"}, +``` diff --git a/devenv/docker/blocks/self-instrumentation/tempo.yaml b/devenv/docker/blocks/self-instrumentation/tempo.yaml index acc5c1b8ea6..c1eae73fa74 100644 --- a/devenv/docker/blocks/self-instrumentation/tempo.yaml +++ b/devenv/docker/blocks/self-instrumentation/tempo.yaml @@ -1,24 +1,31 @@ +stream_over_http_enabled: true server: http_listen_port: 3200 + log_level: info + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + metadata_slo: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s distributor: - receivers: # this configuration will listen on all ports and protocols that tempo is capable of. - jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can - protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver - thrift_http: # - grpc: # for a production deployment you should only enable the receivers you need! - thrift_binary: - thrift_compact: - zipkin: + receivers: otlp: protocols: - http: grpc: - opencensus: + endpoint: 'tempo:4317' + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally compactor: compaction: - block_retention: 1h + block_retention: 1h # overall Tempo trace retention. set for demo purposes metrics_generator: registry: @@ -26,19 +33,23 @@ metrics_generator: source: tempo cluster: docker-compose storage: - path: /tmp/tempo/generator/wal + path: /var/tempo/generator/wal remote_write: - url: http://prometheus:9090/api/v1/write send_exemplars: true + traces_storage: + path: /var/tempo/generator/traces storage: trace: - backend: local # backend configuration to use + backend: local # backend configuration to use wal: - path: /tmp/tempo/wal # where to store the wal locally + path: /var/tempo/wal # where to store the wal locally local: - path: /tmp/tempo/blocks + path: /var/tempo/blocks overrides: - metrics_generator_processors: [service-graphs, span-metrics] - \ No newline at end of file + defaults: + metrics_generator: + processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator + generate_native_histograms: both