diff --git a/devenv/frontend-service/Tiltfile b/devenv/frontend-service/Tiltfile index 885d8eddaf3..a753e475e80 100644 --- a/devenv/frontend-service/Tiltfile +++ b/devenv/frontend-service/Tiltfile @@ -69,6 +69,8 @@ dc_resource("frontend-service", ) dc_resource("alloy", labels=["observability"]) dc_resource("loki", labels=["observability"]) +dc_resource("tempo-init", labels=["observability"]) +dc_resource("tempo", labels=["observability"]) # paths in tilt files are confusing.... # - if tilt is dealing the the path, it is relative to the Tiltfile @@ -111,10 +113,10 @@ docker_build('grafana-proxy', # Path relative to the docker context (this folder) only=[ - "./nginx.conf", + "./configs/nginx.conf", ], live_update = [ - sync('./nginx.conf', '/etc/nginx/conf.d/default.conf'), + sync('./configs/nginx.conf', '/etc/nginx/conf.d/default.conf'), restart_container() ] ) diff --git a/devenv/frontend-service/configs/alloy/_databases.alloy b/devenv/frontend-service/configs/alloy/_databases.alloy index eb1766c1346..9e050d70fd9 100644 --- a/devenv/frontend-service/configs/alloy/_databases.alloy +++ b/devenv/frontend-service/configs/alloy/_databases.alloy @@ -1,3 +1,5 @@ +// +// Logs loki.relabel "publish_logs" { rule { action = "replace" @@ -13,3 +15,31 @@ loki.write "tilt_loki" { url = "http://loki:3100/loki/api/v1/push" } } + +// +// Traces +otelcol.processor.transform "publish_traces" { + error_mode = "ignore" + + trace_statements { + context = "resource" + statements = [ + string.format(`set(attributes["instance"], "%s")`, constants.hostname), + ] + } + + output { + traces = [otelcol.exporter.otlp.tilt_tempo.input] + } +} + +otelcol.exporter.otlp "tilt_tempo" { + client { + endpoint = "tempo:4317" + + tls { + insecure = true + insecure_skip_verify = true + } + } +} diff --git a/devenv/frontend-service/configs/alloy/otel-receiver.alloy b/devenv/frontend-service/configs/alloy/otel-receiver.alloy new file mode 100644 index 00000000000..df6b7d53178 --- /dev/null +++ b/devenv/frontend-service/configs/alloy/otel-receiver.alloy @@ -0,0 +1,13 @@ +// +// Traces +otelcol.receiver.otlp "grafana_dev" { + output { + metrics = [] + logs = [] + traces = [otelcol.processor.transform.publish_traces.input] + } + + grpc { + endpoint = "0.0.0.0:4317" + } +} diff --git a/devenv/frontend-service/nginx.conf b/devenv/frontend-service/configs/nginx.conf similarity index 83% rename from devenv/frontend-service/nginx.conf rename to devenv/frontend-service/configs/nginx.conf index 5e2ad228eb3..0636ff0a231 100644 --- a/devenv/frontend-service/nginx.conf +++ b/devenv/frontend-service/configs/nginx.conf @@ -1,3 +1,9 @@ +otel_exporter { + endpoint alloy:4317; +} + +otel_service_name "proxy"; + ### # Instance ### @@ -14,10 +20,19 @@ map "$request_method:$cookie_fs_unavailable" $reject_login { "POST:1" 1; } +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + server { listen 80; server_name _; + otel_trace on; + otel_trace_context inject; + otel_span_name "$request_method Request"; + location ~ ^/-/down/?$ { add_header Set-Cookie "fs_unavailable=true; Max-Age=60; Path=/; HttpOnly" always; return 302 $scheme://$http_host/; @@ -58,9 +73,11 @@ server { } # API calls go to the backend - # Cheat with app plugin paths and route them to the backend. These should come from - # the Plugin CDN location ~ ^/(api|apis|avatar|bootdata|render|logout|public\/plugins) { + # Add debug headers to the response + add_header Nginx-Trace-Id $otel_trace_id always; + add_header Nginx-Route "backend" always; + if ($cookie_fs_unavailable) { add_header Content-Type application/json always; return 503 '{"code":"Loading", "message": "Soon!"}'; @@ -71,12 +88,18 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Upgrade $http_upgrade; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; } # Everything else to the frontend location / { + # Add debug headers to the response + add_header Nginx-Trace-Id $otel_trace_id always; + add_header Nginx-Route "frontend" always; + proxy_pass http://frontend; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/devenv/frontend-service/configs/tempo.yaml b/devenv/frontend-service/configs/tempo.yaml new file mode 100644 index 00000000000..58457e755c8 --- /dev/null +++ b/devenv/frontend-service/configs/tempo.yaml @@ -0,0 +1,48 @@ +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + log_level: info + +distributor: + usage: + cost_attribution: + enabled: true + receivers: + otlp: + protocols: + grpc: + endpoint: 'tempo:4317' + http: + endpoint: 'tempo:4318' + +storage: + trace: + backend: local # Specify local storage explicitly + + # The Write-Ahead Log (WAL) is a logging mechanism used to ensure data integrity. In the context of Tempo, + # the WAL is used to record changes to trace data before they are committed to the main storage. + #This helps in recovering data in case of a failure. The WAL stores these changes in a sequential log file, + #which can be replayed to reconstruct the state of the system. + wal: + path: /var/lib/tempo/wal + # indicates that the storage backend is local, meaning the data will be stored on the local filesystem. + local: + path: /var/lib/tempo/blocks # Path for block storage + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /var/lib/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + traces_storage: + path: /var/lib/tempo/generator/traces + processor: + local_blocks: + filter_server_spans: false + flush_to_storage: true diff --git a/devenv/frontend-service/docker-compose.yaml b/devenv/frontend-service/docker-compose.yaml index 9e6939d6e97..3a238ba890d 100644 --- a/devenv/frontend-service/docker-compose.yaml +++ b/devenv/frontend-service/docker-compose.yaml @@ -30,15 +30,18 @@ services: - ./provisioning/dashboards:/grafana/conf/provisioning/dashboards - ../dev-dashboards:/grafana/conf/dev-dashboards environment: + OTEL_BSP_SCHEDULE_DELAY: 500 GF_DEFAULT_APP_MODE: development GF_PANELS_ENABLE_ALPHA: true GF_SERVER_CDN_URL: http://localhost:3010 GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend - GF_DATABASE_TYPE: postgres - GF_DATABASE_HOST: postgres - GF_DATABASE_NAME: grafana - GF_DATABASE_USER: grafana - GF_DATABASE_PASSWORD: grafana + GF_DATABASE_URL: postgres://grafana:grafana@postgres:5432/grafana + GF_SERVER_ROUTER_LOGGING: true + GF_LOG_LEVEL: info + GF_AUTH_LOGIN_COOKIE_NAME: grafana_fs_dev_login # set a custom cookie name to not conflict with other instances running on localhost + OTEL_SERVICE_NAME: grafana-api + GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: 'alloy:4317' + GF_TRACING_OPENTELEMETRY_OTLP_PROPAGATION: jaeger,w3c ports: - '3011:3000' labels: @@ -55,10 +58,16 @@ services: labels: - 'alloy.logs=true' environment: + OTEL_BSP_SCHEDULE_DELAY: 500 GF_DEFAULT_APP_MODE: development GF_DEFAULT_TARGET: frontend-server GF_SECURITY_CONTENT_SECURITY_POLICY: false GF_SERVER_CDN_URL: http://localhost:3010 + GF_SERVER_ROUTER_LOGGING: true + GF_LOG_LEVEL: info + OTEL_SERVICE_NAME: frontend-service + GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: 'alloy:4317' + GF_TRACING_OPENTELEMETRY_OTLP_PROPAGATION: jaeger,w3c postgres: image: postgres:16.1-alpine3.19 @@ -95,8 +104,30 @@ services: labels: - 'alloy.logs=true' + tempo-init: + image: grafana/tempo + user: root + entrypoint: + - 'chown' + - '10001:10001' + - '/var/tempo' + volumes: + - tempo-data:/var/tempo + + tempo: + image: grafana/tempo + volumes: + - tempo-data:/var/lib/tempo + - ./configs/tempo.yaml:/etc/tempo/tempo.yaml + command: ['-config.file=/etc/tempo/tempo.yaml'] + + depends_on: + tempo-init: + condition: service_completed_successfully + volumes: backend-data: postgres-data: alloy-data: loki-data: + tempo-data: diff --git a/devenv/frontend-service/provisioning/datasources/default.yaml b/devenv/frontend-service/provisioning/datasources/default.yaml index c99a93fd0b2..005bd631498 100644 --- a/devenv/frontend-service/provisioning/datasources/default.yaml +++ b/devenv/frontend-service/provisioning/datasources/default.yaml @@ -12,3 +12,16 @@ datasources: uid: fs-loki type: loki url: http://loki:3100 + + - name: Tempo Traces + uid: fs-tempo + type: tempo + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + # Field with an internal link pointing to a logs data source in Grafana. + # datasourceUid value must match the uid value of the logs data source. + datasourceUid: 'fs-loki' + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + filterByTraceID: true diff --git a/devenv/frontend-service/proxy.dockerfile b/devenv/frontend-service/proxy.dockerfile index cd66cfd1187..9e0d8a4ceea 100644 --- a/devenv/frontend-service/proxy.dockerfile +++ b/devenv/frontend-service/proxy.dockerfile @@ -1,5 +1,18 @@ -# Runtime stage - just a simple nginx to serve static files -# We bake the config into the image so we can live-update it with Tilt when it changes -FROM nginx:alpine +FROM nginx:1.29.0-alpine -COPY nginx.conf /etc/nginx/conf.d/default.conf +RUN apk add --no-cache openssl curl ca-certificates + +RUN printf "%s%s%s%s\n" \ + "@nginx " \ + "http://nginx.org/packages/mainline/alpine/v" \ + `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` \ + "/main" \ + | tee -a /etc/apk/repositories + +RUN curl -o /tmp/nginx_signing.rsa.pub https://nginx.org/keys/nginx_signing.rsa.pub +RUN mv /tmp/nginx_signing.rsa.pub /etc/apk/keys/ +RUN apk add --no-cache nginx-module-otel@nginx --force-overwrite + +RUN sed -i '1iload_module modules/ngx_otel_module.so;' /etc/nginx/nginx.conf + +COPY configs/nginx.conf /etc/nginx/conf.d/default.conf