From 0f40b2d206410309e6866d420882a373b98c16fb Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 8 Feb 2019 17:28:15 +0100 Subject: [PATCH 1/5] devenv: trying to make fluentd with inoffical fluent-plugin-loki work --- devenv/docker/ha_test/docker-compose.yaml | 26 ++++++++++++++++ devenv/docker/ha_test/fluentd/Dockerfile | 30 +++++++++++++++++++ devenv/docker/ha_test/fluentd/Gemfile | 1 + devenv/docker/ha_test/fluentd/fluent.conf | 19 ++++++++++++ .../provisioning/datasources/datasources.yaml | 7 ++++- 5 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 devenv/docker/ha_test/fluentd/Dockerfile create mode 100644 devenv/docker/ha_test/fluentd/Gemfile create mode 100644 devenv/docker/ha_test/fluentd/fluent.conf diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index 8087894c56e..91f5eead697 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -7,6 +7,10 @@ services: - "80:80" volumes: - /var/run/docker.sock:/tmp/docker.sock:ro + logging: + driver: "fluentd" + options: + tag: nginx db: image: mysql:5.6 @@ -73,6 +77,10 @@ services: depends_on: db: condition: service_healthy + logging: + driver: "fluentd" + options: + tag: grafana prometheus: image: prom/prometheus:v2.4.2 @@ -82,3 +90,21 @@ services: - VIRTUAL_HOST=prometheus.loc ports: - 9090 + + loki: + image: grafana/loki:master + environment: + - VIRTUAL_HOST=loki.loc + ports: + - 3100 + command: -config.file=/etc/loki/local-config.yaml + + fluentd: + build: ./fluentd + volumes: + - ./fluentd/fluent.conf:/fluentd/etc/fluent.conf + links: + - loki + ports: + - "24224:24224" + - "24224:24224/udp" diff --git a/devenv/docker/ha_test/fluentd/Dockerfile b/devenv/docker/ha_test/fluentd/Dockerfile new file mode 100644 index 00000000000..75956b7d572 --- /dev/null +++ b/devenv/docker/ha_test/fluentd/Dockerfile @@ -0,0 +1,30 @@ +FROM fluent/fluentd:v1.3.3-debian-1.0 + +# USER root + +# RUN apk add --no-cache --update --virtual .build-deps \ +# sudo build-base ruby-dev \ +# && sudo gem install bundler \ +# && gem 'loki', git: 'https://github.com/grafana/loki.git', glob: 'fluentd/fluent-plugin-loki/*.gemspec' \ +# && sudo gem sources --clear-all \ +# && apk del .build-deps \ +# && rm -rf /home/fluent/.gem/ruby/2.5.0/cache/*.gem + +# USER fluent + +USER root + +COPY Gemfile . +RUN ["gem", "install", "fluent-plugin-loki"] + +# RUN apt-get -y update \ +# && apt-get -y install git \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man/?? /usr/share/man/??_* + +# RUN gem install bundler \ +# && bundler install + +# COPY fluent.conf /fluentd/etc + +USER fluent diff --git a/devenv/docker/ha_test/fluentd/Gemfile b/devenv/docker/ha_test/fluentd/Gemfile new file mode 100644 index 00000000000..20b4d4aebec --- /dev/null +++ b/devenv/docker/ha_test/fluentd/Gemfile @@ -0,0 +1 @@ +source 'https://rubygems.org' diff --git a/devenv/docker/ha_test/fluentd/fluent.conf b/devenv/docker/ha_test/fluentd/fluent.conf new file mode 100644 index 00000000000..44091934be1 --- /dev/null +++ b/devenv/docker/ha_test/fluentd/fluent.conf @@ -0,0 +1,19 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type stdout + format json + + + + @type loki + endpoint_url "http://loki:3100" + labels {"env":"dev"} + flush_interval 10s + flush_at_shutdown true + buffer_chunk_limit 1m + \ No newline at end of file diff --git a/devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml b/devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml index 8d59793be16..3cf02b6a9b6 100644 --- a/devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml +++ b/devenv/docker/ha_test/grafana/provisioning/datasources/datasources.yaml @@ -8,4 +8,9 @@ datasources: jsonData: timeInterval: 10s queryTimeout: 30s - httpMethod: POST \ No newline at end of file + httpMethod: POST + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 From 7b1249ad472bcd87af6c9af847ef642aa1e38fd4 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 12 Feb 2019 00:48:27 +0100 Subject: [PATCH 2/5] devenv: use grafana/fluent-plugin-loki --- devenv/docker/ha_test/docker-compose.yaml | 12 ++++---- devenv/docker/ha_test/fluentd/Dockerfile | 30 ------------------- devenv/docker/ha_test/fluentd/Gemfile | 1 - .../fluentd/{fluent.conf => fluentd.conf} | 13 ++++---- .../docker/ha_test/prometheus/prometheus.yml | 8 +++++ 5 files changed, 18 insertions(+), 46 deletions(-) delete mode 100644 devenv/docker/ha_test/fluentd/Dockerfile delete mode 100644 devenv/docker/ha_test/fluentd/Gemfile rename devenv/docker/ha_test/fluentd/{fluent.conf => fluentd.conf} (52%) diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index 91f5eead697..3ca23035e09 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -7,10 +7,6 @@ services: - "80:80" volumes: - /var/run/docker.sock:/tmp/docker.sock:ro - logging: - driver: "fluentd" - options: - tag: nginx db: image: mysql:5.6 @@ -51,7 +47,7 @@ services: # retries: 10 grafana: - image: grafana/grafana:dev + image: grafana/grafana:6.0.0-beta2 volumes: - ./grafana/provisioning/:/etc/grafana/provisioning/ environment: @@ -70,6 +66,8 @@ services: # - GF_DATABASE_SSL_MODE=disable # - GF_SESSION_PROVIDER=postgres # - GF_SESSION_PROVIDER_CONFIG=user=grafana password=password host=db port=5432 dbname=grafana sslmode=disable + - GF_SERVER_ROUTER_LOGGING=true + - GF_LOG_CONSOLE_FORMAT=json - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug,auth:debug - GF_AUTH_TOKEN_ROTATION_INTERVAL_MINUTES=2 ports: @@ -100,9 +98,9 @@ services: command: -config.file=/etc/loki/local-config.yaml fluentd: - build: ./fluentd + image: grafana/fluent-plugin-loki:master volumes: - - ./fluentd/fluent.conf:/fluentd/etc/fluent.conf + - ./fluentd/fluentd.conf:/fluentd/etc/fluentd.conf links: - loki ports: diff --git a/devenv/docker/ha_test/fluentd/Dockerfile b/devenv/docker/ha_test/fluentd/Dockerfile deleted file mode 100644 index 75956b7d572..00000000000 --- a/devenv/docker/ha_test/fluentd/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM fluent/fluentd:v1.3.3-debian-1.0 - -# USER root - -# RUN apk add --no-cache --update --virtual .build-deps \ -# sudo build-base ruby-dev \ -# && sudo gem install bundler \ -# && gem 'loki', git: 'https://github.com/grafana/loki.git', glob: 'fluentd/fluent-plugin-loki/*.gemspec' \ -# && sudo gem sources --clear-all \ -# && apk del .build-deps \ -# && rm -rf /home/fluent/.gem/ruby/2.5.0/cache/*.gem - -# USER fluent - -USER root - -COPY Gemfile . -RUN ["gem", "install", "fluent-plugin-loki"] - -# RUN apt-get -y update \ -# && apt-get -y install git \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man/?? /usr/share/man/??_* - -# RUN gem install bundler \ -# && bundler install - -# COPY fluent.conf /fluentd/etc - -USER fluent diff --git a/devenv/docker/ha_test/fluentd/Gemfile b/devenv/docker/ha_test/fluentd/Gemfile deleted file mode 100644 index 20b4d4aebec..00000000000 --- a/devenv/docker/ha_test/fluentd/Gemfile +++ /dev/null @@ -1 +0,0 @@ -source 'https://rubygems.org' diff --git a/devenv/docker/ha_test/fluentd/fluent.conf b/devenv/docker/ha_test/fluentd/fluentd.conf similarity index 52% rename from devenv/docker/ha_test/fluentd/fluent.conf rename to devenv/docker/ha_test/fluentd/fluentd.conf index 44091934be1..1a037407519 100644 --- a/devenv/docker/ha_test/fluentd/fluent.conf +++ b/devenv/docker/ha_test/fluentd/fluentd.conf @@ -4,15 +4,12 @@ bind 0.0.0.0 - - @type stdout - format json - - - + @type loki - endpoint_url "http://loki:3100" - labels {"env":"dev"} + url "http://loki:3100" + extra_labels {"app":"grafana"} + label_keys "container_name,container_id" + line_format "json" flush_interval 10s flush_at_shutdown true buffer_chunk_limit 1m diff --git a/devenv/docker/ha_test/prometheus/prometheus.yml b/devenv/docker/ha_test/prometheus/prometheus.yml index 0950bad6f7d..ab1711e9c6e 100644 --- a/devenv/docker/ha_test/prometheus/prometheus.yml +++ b/devenv/docker/ha_test/prometheus/prometheus.yml @@ -36,4 +36,12 @@ scrape_configs: - 'mysqld-exporter' type: 'A' port: 9104 + refresh_interval: 10s + + - job_name: 'loki' + dns_sd_configs: + - names: + - 'loki' + type: 'A' + port: 3100 refresh_interval: 10s \ No newline at end of file From d55a6deb9735b75621a164accbf01e0cf321b9f6 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 12 Feb 2019 17:14:07 +0100 Subject: [PATCH 3/5] devenv: proper fluentd conf for grafana and loki --- devenv/docker/ha_test/fluentd/fluentd.conf | 40 +++++++++++++++++----- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/devenv/docker/ha_test/fluentd/fluentd.conf b/devenv/docker/ha_test/fluentd/fluentd.conf index 1a037407519..6d7dc3daf3f 100644 --- a/devenv/docker/ha_test/fluentd/fluentd.conf +++ b/devenv/docker/ha_test/fluentd/fluentd.conf @@ -4,13 +4,37 @@ bind 0.0.0.0 + + @type parser + + @type json + json_parser json + + replace_invalid_sequence true + emit_invalid_record_to_error false + key_name log + reserve_data true + remove_key_name_field true + + + + @type record_transformer + remove_keys "source,t" + + - @type loki - url "http://loki:3100" - extra_labels {"app":"grafana"} - label_keys "container_name,container_id" - line_format "json" - flush_interval 10s - flush_at_shutdown true - buffer_chunk_limit 1m + @type copy + + @type stdout + output_type json + + + @type loki + url "http://loki:3100" + extra_labels {"app":"grafana"} + label_keys "container_name,container_id,logger" + flush_interval 10s + flush_at_shutdown true + buffer_chunk_limit 1m + \ No newline at end of file From e61616d829d4c3de4039b0137e540f2396af20e4 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 13 Feb 2019 11:16:53 +0100 Subject: [PATCH 4/5] devenv: send nginx logs to loki in ha test --- devenv/docker/ha_test/docker-compose.yaml | 7 +++++ devenv/docker/ha_test/fluentd/fluentd.conf | 35 +++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index 3ca23035e09..9772827251a 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -7,6 +7,13 @@ services: - "80:80" volumes: - /var/run/docker.sock:/tmp/docker.sock:ro + depends_on: + db: + condition: service_healthy + logging: + driver: "fluentd" + options: + tag: nginx db: image: mysql:5.6 diff --git a/devenv/docker/ha_test/fluentd/fluentd.conf b/devenv/docker/ha_test/fluentd/fluentd.conf index 6d7dc3daf3f..58859441d96 100644 --- a/devenv/docker/ha_test/fluentd/fluentd.conf +++ b/devenv/docker/ha_test/fluentd/fluentd.conf @@ -17,7 +17,16 @@ remove_key_name_field true - + + @type parser + format /^.*m(?[^ ]*) (?[^ ]*) \- \- \[(? + + @type record_transformer remove_keys "source,t" @@ -37,4 +46,28 @@ flush_at_shutdown true buffer_chunk_limit 1m + + + + @type record_transformer + + lvl "info" + + + + + @type copy + + @type stdout + output_type json + + + @type loki + url "http://loki:3100" + extra_labels {"app":"nginx"} + label_keys "container_name,container_id" + flush_interval 10s + flush_at_shutdown true + buffer_chunk_limit 1m + \ No newline at end of file From 0f758fd1ffb800d00ed89e888e28701a7c17372b Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 13 Feb 2019 11:18:02 +0100 Subject: [PATCH 5/5] devenv: use grafana:dev image in ha test per default --- devenv/docker/ha_test/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index 9772827251a..a215513c568 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -54,7 +54,7 @@ services: # retries: 10 grafana: - image: grafana/grafana:6.0.0-beta2 + image: grafana/grafana:dev volumes: - ./grafana/provisioning/:/etc/grafana/provisioning/ environment: