public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update self instrumentation to use alloy and bump versions (#109266)

Browse Source 
* Update self instrumentation to use alloy and bump versions

* Update devenv/docker/blocks/self-instrumentation/config.alloy

Co-authored-by: Sam DeHaan <dehaansa@gmail.com>

* Add remote write receiver feature to Prometheus configuration in docker-compose.yaml

* Add example for plugin profiling to the doc

---------

Co-authored-by: Sam DeHaan <dehaansa@gmail.com>
This commit is contained in:
Zoltán Bedi
2025-08-14 12:34:41 +02:00
committed by GitHub
co-authored by Sam DeHaan
parent 9b38ac1c24
commit 9453cedb51
7 changed files with 179 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devenv/docker/blocks/self-instrumentation/agent.flow
-87
View File
@@ -1,87 +0,0 @@
loki.write "default" {
endpoint {
url = "http://loki:3100/loki/api/v1/push"
}
external_labels = {}
}
logging {
level = "debug"
format = "json"
write_to = [loki.write.default.receiver]
}
tracing {
sampling_fraction = 1
write_to = [otelcol.exporter.otlp.tempo.input]
}
otelcol.exporter.otlp "tempo" {
client {
endpoint = "tempo:4317"
tls {
insecure = true
insecure_skip_verify = true
}
}
}
faro.receiver "frontend" {
server {
listen_address = "0.0.0.0"
listen_port = 12347
cors_allowed_origins = ["*"]
}
output {
logs = [loki.write.default.receiver]
traces = [otelcol.exporter.otlp.tempo.input]
}
}
pyroscope.write "default" {
endpoint {
url = "http://pyroscope:4040"
}
}
pyroscope.scrape "default" {
targets = [
{"__address__" = "host.docker.internal:6000", "service_name"="grafana"},
]
forward_to = [pyroscope.write.default.receiver]
profiling_config {
profile.process_cpu {
enabled = true
}
profile.godeltaprof_memory {
enabled = true
}
profile.memory { // disable memory, use godeltaprof_memory instead
enabled = false
}
profile.godeltaprof_mutex {
enabled = true
}
profile.mutex { // disable mutex, use godeltaprof_mutex instead
enabled = false
}
profile.godeltaprof_block {
enabled = true
}
profile.block { // disable block, use godeltaprof_block instead
enabled = false
}
profile.goroutine {
enabled = true
}
}
}
devenv/docker/blocks/self-instrumentation/config.alloy
+100
View File
@@ -0,0 +1,100 @@
loki.write "default" {
endpoint {
url = "http://loki:3100/loki/api/v1/push"
}
}
// Grafana log file discovery
local.file_match "grafana_logs" {
path_targets = [
{
__path__ = "/var/log/grafana/*.log",
},
]
}
// Grafana log source
loki.source.file "grafana" {
targets = local.file_match.grafana_logs.targets
forward_to = [loki.relabel.grafana.receiver]
}
// Add labels to Grafana logs
loki.relabel "grafana" {
rule {
action = "replace"
target_label = "job"
replacement = "grafana"
}
rule {
action = "replace"
target_label = "service"
replacement = "grafana"
}
forward_to = [loki.write.default.receiver]
}
logging {
level = "debug"
format = "json"
// Forward internal logs to the local Loki instance.
write_to = [loki.write.default.receiver]
}
tracing {
sampling_fraction = 1.0
// Forward internal spans to the local Tempo instance.
write_to = [otelcol.exporter.otlp.tempo.input]
}
otelcol.receiver.otlp "otlp_receiver" {
grpc {
endpoint = "0.0.0.0:4317"
}
output {
traces = [otelcol.exporter.otlp.tempo.input]
}
}
otelcol.exporter.otlp "tempo" {
client {
endpoint = "tempo:4317"
tls {
insecure = true
}
}
}
pyroscope.write "default" {
// Send profiles to a locally running Pyroscope instance.
endpoint {
url = "http://pyroscope:4040"
}
}
pyroscope.scrape "default" {
targets = [
{"__address__" = "pyroscope:4040", "service_name"="pyroscope"},
{"__address__" = "alloy:12345", "service_name"="alloy"},
{"__address__" = "host.docker.internal:6000", "service_name"="grafana"},
]
forward_to = [pyroscope.write.default.receiver]
}
faro.receiver "frontend" {
server {
listen_address = "0.0.0.0"
listen_port = 12347
cors_allowed_origins = ["*"]
}
output {
logs = [loki.write.default.receiver]
traces = [otelcol.exporter.otlp.tempo.input]
}
}
devenv/docker/blocks/self-instrumentation/docker-compose.yaml
+29 -31
View File
@@ -1,56 +1,54 @@
prometheus:
image: prom/prometheus:v2.45.0
image: prom/prometheus:v3.5.0
ports:
- "9090:9090"
- '9090:9090'
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
command: >
--enable-feature=remote-write-receiver
--enable-feature=exemplar-storage
--enable-feature=native-histograms
--web.enable-remote-write-receiver
--config.file=/etc/prometheus/prometheus.yml
--storage.tsdb.path=/prometheus
volumes:
- ./docker/blocks/self-instrumentation/prometheus.yaml:/etc/prometheus/prometheus.yml
loki:
image: grafana/loki:2.8.2
image: grafana/loki:3.5.3
ports:
- "3100:3100"
- '3100:3100'
command: -config.file=/etc/loki/local-config.yaml
promtail:
image: grafana/promtail:2.8.2
command: -config.file=/etc/promtail/promtail.yaml
volumes:
- ./docker/blocks/self-instrumentation/promtail.yaml:/etc/promtail/promtail.yaml
- ../data/log:/var/log/grafana
tempo:
image: grafana/tempo:2.1.1
image: grafana/tempo:2.8.1
command: -config.file=/etc/tempo.yaml
volumes:
- ./docker/blocks/self-instrumentation/tempo.yaml:/etc/tempo.yaml
ports:
- "14268:14268"
- "3200:3200"
- "4317:4317"
- '14268:14268'
- '3200:3200'
- '4317:4317'
pyroscope:
image: "grafana/pyroscope:latest"
image: 'grafana/pyroscope:1.14.0'
ports:
- "4040:4040"
- '4040:4040'
extra_hosts:
- "host.docker.internal:host-gateway"
agent:
image: grafana/agent:v0.37.0
environment:
- AGENT_MODE=flow
entrypoint:
- /bin/grafana-agent
- 'host.docker.internal:host-gateway'
alloy:
image: grafana/alloy:v1.10.0
command:
- run
- /etc/alloy/config.alloy
- --storage.path=/var/lib/alloy/data
- --server.http.listen-addr=0.0.0.0:12345
- /etc/agent/config.river
volumes:
- ./docker/blocks/self-instrumentation/agent.flow:/etc/agent/config.river
- --stability.level=experimental # Enable all functionality
ports:
- "12345:12345"
- "12347:12347"
- '12345:12345'
volumes:
- ./docker/blocks/self-instrumentation/config.alloy:/etc/alloy/config.alloy
- ../data/log:/var/log/grafana:ro # Mount Grafana logs directory
depends_on:
- tempo
- loki
- prometheus
- pyroscope
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
devenv/docker/blocks/self-instrumentation/prometheus.yaml
+3 -3
View File
@@ -1,10 +1,10 @@
global:
scrape_interval: 15s
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: grafana
scrape_classic_histograms: true
always_scrape_classic_histograms: true
static_configs:
- targets:
- host.docker.internal:3000
- host.docker.internal:3000
devenv/docker/blocks/self-instrumentation/promtail.yaml
-18
View File
@@ -1,18 +0,0 @@
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
scrape_configs:
- job_name: grafana
static_configs:
- targets:
- localhost
labels:
job: grafana
__path__: /var/log/grafana/*log
devenv/docker/blocks/self-instrumentation/readme.md
+19 -2
View File
@@ -18,8 +18,9 @@ format = json
enabled = true
custom_endpoint=http://localhost:12347/collect
[tracing.opentelemetry.jaeger]
address = http://localhost:14268/api/traces
[tracing.opentelemetry.otlp]
address = localhost:4317
insecure = true
```
To collect profiles with pyroscope, you need to run Grafana with the following env vars:
@@ -29,3 +30,19 @@ export GF_DIAGNOSTICS_PROFILING_ENABLED=true
export GF_DIAGNOSTICS_PROFILING_ADDR=0.0.0.0
make run
```
To enable profiling in your plugin add the following to your `custom.ini`:
```ini
[plugin.grafana-bigquery-datasource]
profiling_enabled = true
profiling_port = 6161
profiling_block_rate = 1
profiling_mutex_rate = 1
```
and the following to your `config.alloy` in the `pyroscope.scrape.targets` section:
```yaml
{"__address__" = "host.docker.internal:6161", "service_name"="grafana-bigquery-datasource"},
```
devenv/docker/blocks/self-instrumentation/tempo.yaml
+28 -17
View File
@@ -1,24 +1,31 @@
stream_over_http_enabled: true
server:
http_listen_port: 3200
log_level: info
query_frontend:
search:
duration_slo: 5s
throughput_bytes_slo: 1.073741824e+09
metadata_slo:
duration_slo: 5s
throughput_bytes_slo: 1.073741824e+09
trace_by_id:
duration_slo: 5s
distributor:
receivers: # this configuration will listen on all ports and protocols that tempo is capable of.
jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can
protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
thrift_http: #
grpc: # for a production deployment you should only enable the receivers you need!
thrift_binary:
thrift_compact:
zipkin:
receivers:
otlp:
protocols:
http:
grpc:
opencensus:
endpoint: 'tempo:4317'
ingester:
max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
compactor:
compaction:
block_retention: 1h
block_retention: 1h # overall Tempo trace retention. set for demo purposes
metrics_generator:
registry:
@@ -26,19 +33,23 @@ metrics_generator:
source: tempo
cluster: docker-compose
storage:
path: /tmp/tempo/generator/wal
path: /var/tempo/generator/wal
remote_write:
- url: http://prometheus:9090/api/v1/write
send_exemplars: true
traces_storage:
path: /var/tempo/generator/traces
storage:
trace:
backend: local # backend configuration to use
backend: local # backend configuration to use
wal:
path: /tmp/tempo/wal # where to store the wal locally
path: /var/tempo/wal # where to store the wal locally
local:
path: /tmp/tempo/blocks
path: /var/tempo/blocks
overrides:
metrics_generator_processors: [service-graphs, span-metrics]
defaults:
metrics_generator:
processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
generate_native_histograms: both