diff --git a/docker/blocks/graphite/Dockerfile b/docker/blocks/graphite/Dockerfile new file mode 100644 index 00000000000..0c85798dd8a --- /dev/null +++ b/docker/blocks/graphite/Dockerfile @@ -0,0 +1,68 @@ +from ubuntu:14.10 + +run apt-get -y update + +run apt-get -y install software-properties-common + +run apt-get -y install python-software-properties &&\ + add-apt-repository ppa:chris-lea/node.js &&\ + apt-get -y update + +run apt-get -y install python-django-tagging python-simplejson python-memcache \ + python-ldap python-cairo python-django python-twisted \ + python-pysqlite2 python-support python-pip gunicorn \ + supervisor nginx-light nodejs git wget curl + +# Install statsd +run mkdir /src && git clone https://github.com/etsy/statsd.git /src/statsd + +run cd /usr/local/src && git clone https://github.com/graphite-project/graphite-web.git +run cd /usr/local/src && git clone https://github.com/graphite-project/carbon.git +run cd /usr/local/src && git clone https://github.com/graphite-project/whisper.git + +run cd /usr/local/src/whisper && git checkout master && python setup.py install +run cd /usr/local/src/carbon && git checkout 0.9.x && python setup.py install +run cd /usr/local/src/graphite-web && git checkout 0.9.x && python check-dependencies.py; python setup.py install + +# statsd +add ./files/statsd_config.js /src/statsd/config.js + +# Add graphite config +add ./files/initial_data.json /opt/graphite/webapp/graphite/initial_data.json +add ./files/local_settings.py /opt/graphite/webapp/graphite/local_settings.py +add ./files/carbon.conf /opt/graphite/conf/carbon.conf +add ./files/storage-schemas.conf /opt/graphite/conf/storage-schemas.conf +add ./files/storage-aggregation.conf /opt/graphite/conf/storage-aggregation.conf +add ./files/events_views.py /opt/graphite/webapp/graphite/events/views.py + +run mkdir -p /opt/graphite/storage/whisper +run touch /opt/graphite/storage/graphite.db /opt/graphite/storage/index +run chown -R www-data /opt/graphite/storage +run chmod 0775 /opt/graphite/storage /opt/graphite/storage/whisper +run chmod 0664 /opt/graphite/storage/graphite.db +run cd /opt/graphite/webapp/graphite && python manage.py syncdb --noinput + +# Add system service config +add ./files/nginx.conf /etc/nginx/nginx.conf +add ./files/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +# graphite +expose 80 + +# Carbon line receiver port +expose 2003 +# Carbon cache query port +expose 7002 + +# Statsd UDP port +expose 8125/udp +# Statsd Management port +expose 8126 + +VOLUME ["/var/lib/elasticsearch"] +VOLUME ["/opt/graphite/storage/whisper"] +VOLUME ["/var/lib/log/supervisor"] + +cmd ["/usr/bin/supervisord"] + +# vim:ts=8:noet: diff --git a/docker/blocks/graphite/fig b/docker/blocks/graphite/fig new file mode 100644 index 00000000000..1f6381e6847 --- /dev/null +++ b/docker/blocks/graphite/fig @@ -0,0 +1,4 @@ +graphite: + build: blocks/docker_graphite + ports: + - "8776:80" diff --git a/docker/blocks/graphite/files/carbon.conf b/docker/blocks/graphite/files/carbon.conf new file mode 100644 index 00000000000..50762b3fff5 --- /dev/null +++ b/docker/blocks/graphite/files/carbon.conf @@ -0,0 +1,76 @@ +[cache] +LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ + +# Specify the user to drop privileges to +# If this is blank carbon runs as the user that invokes it +# This user must have write access to the local data directory +USER = + +# Limit the size of the cache to avoid swapping or becoming CPU bound. +# Sorts and serving cache queries gets more expensive as the cache grows. +# Use the value "inf" (infinity) for an unlimited cache size. +MAX_CACHE_SIZE = inf + +# Limits the number of whisper update_many() calls per second, which effectively +# means the number of write requests sent to the disk. This is intended to +# prevent over-utilizing the disk and thus starving the rest of the system. +# When the rate of required updates exceeds this, then carbon's caching will +# take effect and increase the overall throughput accordingly. +MAX_UPDATES_PER_SECOND = 1000 + +# Softly limits the number of whisper files that get created each minute. +# Setting this value low (like at 50) is a good way to ensure your graphite +# system will not be adversely impacted when a bunch of new metrics are +# sent to it. The trade off is that it will take much longer for those metrics' +# database files to all get created and thus longer until the data becomes usable. +# Setting this value high (like "inf" for infinity) will cause graphite to create +# the files quickly but at the risk of slowing I/O down considerably for a while. +MAX_CREATES_PER_MINUTE = inf + +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +CACHE_QUERY_INTERFACE = 0.0.0.0 +CACHE_QUERY_PORT = 7002 + +LOG_UPDATES = False + +# Enable AMQP if you want to receve metrics using an amqp broker +# ENABLE_AMQP = False + +# Verbose means a line will be logged for every metric received +# useful for testing +# AMQP_VERBOSE = False + +# AMQP_HOST = localhost +# AMQP_PORT = 5672 +# AMQP_VHOST = / +# AMQP_USER = guest +# AMQP_PASSWORD = guest +# AMQP_EXCHANGE = graphite + +# Patterns for all of the metrics this machine will store. Read more at +# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings +# +# Example: store all sales, linux servers, and utilization metrics +# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization +# +# Example: store everything +# BIND_PATTERNS = # + +# NOTE: you cannot run both a cache and a relay on the same server +# with the default configuration, you have to specify a distinict +# interfaces and ports for the listeners. + +[relay] +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +CACHE_SERVERS = server1, server2, server3 +MAX_QUEUE_SIZE = 10000 diff --git a/docker/blocks/graphite/files/events_views.py b/docker/blocks/graphite/files/events_views.py new file mode 100644 index 00000000000..6c12987af3c --- /dev/null +++ b/docker/blocks/graphite/files/events_views.py @@ -0,0 +1,102 @@ +import datetime +import time + +from django.utils.timezone import get_current_timezone +from django.core.urlresolvers import get_script_prefix +from django.http import HttpResponse +from django.shortcuts import render_to_response, get_object_or_404 +from pytz import timezone + +from graphite.util import json +from graphite.events import models +from graphite.render.attime import parseATTime + + +def to_timestamp(dt): + return time.mktime(dt.timetuple()) + + +class EventEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime.datetime): + return to_timestamp(obj) + return json.JSONEncoder.default(self, obj) + + +def view_events(request): + if request.method == "GET": + context = { 'events' : fetch(request), + 'slash' : get_script_prefix() + } + return render_to_response("events.html", context) + else: + return post_event(request) + +def detail(request, event_id): + e = get_object_or_404(models.Event, pk=event_id) + context = { 'event' : e, + 'slash' : get_script_prefix() + } + return render_to_response("event.html", context) + + +def post_event(request): + if request.method == 'POST': + event = json.loads(request.body) + assert isinstance(event, dict) + + values = {} + values["what"] = event["what"] + values["tags"] = event.get("tags", None) + values["when"] = datetime.datetime.fromtimestamp( + event.get("when", time.time())) + if "data" in event: + values["data"] = event["data"] + + e = models.Event(**values) + e.save() + + return HttpResponse(status=200) + else: + return HttpResponse(status=405) + +def get_data(request): + if 'jsonp' in request.REQUEST: + response = HttpResponse( + "%s(%s)" % (request.REQUEST.get('jsonp'), + json.dumps(fetch(request), cls=EventEncoder)), + mimetype='text/javascript') + else: + response = HttpResponse( + json.dumps(fetch(request), cls=EventEncoder), + mimetype="application/json") + return response + +def fetch(request): + #XXX we need to move to USE_TZ=True to get rid of naive-time conversions + def make_naive(dt): + if 'tz' in request.GET: + tz = timezone(request.GET['tz']) + else: + tz = get_current_timezone() + local_dt = dt.astimezone(tz) + if hasattr(local_dt, 'normalize'): + local_dt = local_dt.normalize() + return local_dt.replace(tzinfo=None) + + if request.GET.get("from", None) is not None: + time_from = make_naive(parseATTime(request.GET["from"])) + else: + time_from = datetime.datetime.fromtimestamp(0) + + if request.GET.get("until", None) is not None: + time_until = make_naive(parseATTime(request.GET["until"])) + else: + time_until = datetime.datetime.now() + + tags = request.GET.get("tags", None) + if tags is not None: + tags = request.GET.get("tags").split(" ") + + return [x.as_dict() for x in + models.Event.find_events(time_from, time_until, tags=tags)] diff --git a/docker/blocks/graphite/files/initial_data.json b/docker/blocks/graphite/files/initial_data.json new file mode 100644 index 00000000000..b3ac9b1ebb0 --- /dev/null +++ b/docker/blocks/graphite/files/initial_data.json @@ -0,0 +1,20 @@ +[ + { + "pk": 1, + "model": "auth.user", + "fields": { + "username": "admin", + "first_name": "", + "last_name": "", + "is_active": true, + "is_superuser": true, + "is_staff": true, + "last_login": "2011-09-20 17:02:14", + "groups": [], + "user_permissions": [], + "password": "sha1$1b11b$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236", + "email": "root@example.com", + "date_joined": "2011-09-20 17:02:14" + } + } +] diff --git a/docker/blocks/graphite/files/local_settings.py b/docker/blocks/graphite/files/local_settings.py new file mode 100644 index 00000000000..177d674e9dd --- /dev/null +++ b/docker/blocks/graphite/files/local_settings.py @@ -0,0 +1,42 @@ +# Edit this file to override the default graphite settings, do not edit settings.py + +# Turn on debugging and restart apache if you ever see an "Internal Server Error" page +#DEBUG = True + +# Set your local timezone (django will try to figure this out automatically) +TIME_ZONE = 'UTC' + +# Setting MEMCACHE_HOSTS to be empty will turn off use of memcached entirely +#MEMCACHE_HOSTS = ['127.0.0.1:11211'] + +# Sometimes you need to do a lot of rendering work but cannot share your storage mount +#REMOTE_RENDERING = True +#RENDERING_HOSTS = ['fastserver01','fastserver02'] +#LOG_RENDERING_PERFORMANCE = True +#LOG_CACHE_PERFORMANCE = True + +# If you've got more than one backend server they should all be listed here +#CLUSTER_SERVERS = [] + +# Override this if you need to provide documentation specific to your graphite deployment +#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite" + +# Enable email-related features +#SMTP_SERVER = "mail.mycompany.com" + +# LDAP / ActiveDirectory authentication setup +#USE_LDAP_AUTH = True +#LDAP_SERVER = "ldap.mycompany.com" +#LDAP_PORT = 389 +#LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com" +#LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com" +#LDAP_BASE_PASS = "readonly_account_password" +#LDAP_USER_QUERY = "(username=%s)" #For Active Directory use "(sAMAccountName=%s)" + +# If sqlite won't cut it, configure your real database here (don't forget to run manage.py syncdb!) +#DATABASE_ENGINE = 'mysql' # or 'postgres' +#DATABASE_NAME = 'graphite' +#DATABASE_USER = 'graphite' +#DATABASE_PASSWORD = 'graphite-is-awesome' +#DATABASE_HOST = 'mysql.mycompany.com' +#DATABASE_PORT = '3306' diff --git a/docker/blocks/graphite/files/nginx.conf b/docker/blocks/graphite/files/nginx.conf new file mode 100644 index 00000000000..5b5c562ce9d --- /dev/null +++ b/docker/blocks/graphite/files/nginx.conf @@ -0,0 +1,70 @@ +daemon off; +user www-data; +worker_processes 1; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + server_names_hash_bucket_size 32; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + gzip on; + gzip_disable "msie6"; + + server { + listen 80 default_server; + server_name _; + + open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m; + + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header Host $host; + + client_max_body_size 10m; + client_body_buffer_size 128k; + + proxy_connect_timeout 90; + proxy_send_timeout 90; + proxy_read_timeout 90; + + proxy_buffer_size 4k; + proxy_buffers 4 32k; + proxy_busy_buffers_size 64k; + proxy_temp_file_write_size 64k; + } + + add_header Access-Control-Allow-Origin "*"; + add_header Access-Control-Allow-Methods "GET, OPTIONS"; + add_header Access-Control-Allow-Headers "origin, authorization, accept"; + + location /content { + alias /opt/graphite/webapp/content; + + } + + location /media { + alias /usr/share/pyshared/django/contrib/admin/media; + } + } +} diff --git a/docker/blocks/graphite/files/statsd_config.js b/docker/blocks/graphite/files/statsd_config.js new file mode 100644 index 00000000000..a03003dcd79 --- /dev/null +++ b/docker/blocks/graphite/files/statsd_config.js @@ -0,0 +1,8 @@ +{ + graphitePort: 2003, + graphiteHost: "127.0.0.1", + port: 8125, + mgmt_port: 8126, + backends: ['./backends/graphite'], + debug: true +} diff --git a/docker/blocks/graphite/files/storage-aggregation.conf b/docker/blocks/graphite/files/storage-aggregation.conf new file mode 100644 index 00000000000..5c4bc19fde1 --- /dev/null +++ b/docker/blocks/graphite/files/storage-aggregation.conf @@ -0,0 +1,19 @@ +[min] +pattern = \.min$ +xFilesFactor = 0.1 +aggregationMethod = min + +[max] +pattern = \.max$ +xFilesFactor = 0.1 +aggregationMethod = max + +[sum] +pattern = \.count$ +xFilesFactor = 0 +aggregationMethod = sum + +[default_average] +pattern = .* +xFilesFactor = 0.5 +aggregationMethod = average diff --git a/docker/blocks/graphite/files/storage-schemas.conf b/docker/blocks/graphite/files/storage-schemas.conf new file mode 100644 index 00000000000..7e28d3a6649 --- /dev/null +++ b/docker/blocks/graphite/files/storage-schemas.conf @@ -0,0 +1,16 @@ +[carbon] +pattern = ^carbon\..* +retentions = 1m:31d,10m:1y,1h:5y + +[highres] +pattern = ^highres.* +retentions = 1s:1d,1m:7d + +[statsd] +pattern = ^statsd.* +retentions = 1m:7d,10m:1y + +[default] +pattern = .* +retentions = 10s:1d,1m:7d,10m:1y + diff --git a/docker/blocks/graphite/files/supervisord.conf b/docker/blocks/graphite/files/supervisord.conf new file mode 100644 index 00000000000..25ba39c8819 --- /dev/null +++ b/docker/blocks/graphite/files/supervisord.conf @@ -0,0 +1,33 @@ +[supervisord] +nodaemon = true +environment = GRAPHITE_STORAGE_DIR='/opt/graphite/storage',GRAPHITE_CONF_DIR='/opt/graphite/conf' + +[program:nginx] +command = /usr/sbin/nginx +stdout_logfile = /var/log/supervisor/%(program_name)s.log +stderr_logfile = /var/log/supervisor/%(program_name)s.log +autorestart = true + +[program:carbon-cache] +;user = www-data +command = /opt/graphite/bin/carbon-cache.py --debug start +stdout_logfile = /var/log/supervisor/%(program_name)s.log +stderr_logfile = /var/log/supervisor/%(program_name)s.log +autorestart = true + +[program:graphite-webapp] +;user = www-data +directory = /opt/graphite/webapp +environment = PYTHONPATH='/opt/graphite/webapp' +command = /usr/bin/gunicorn_django -b127.0.0.1:8000 -w2 graphite/settings.py +stdout_logfile = /var/log/supervisor/%(program_name)s.log +stderr_logfile = /var/log/supervisor/%(program_name)s.log +autorestart = true + +[program:statsd] +;user = www-data +command = /usr/bin/node /src/statsd/stats.js /src/statsd/config.js +stdout_logfile = /var/log/supervisor/%(program_name)s.log +stderr_logfile = /var/log/supervisor/%(program_name)s.log +autorestart = true + diff --git a/docker/blocks/mysql/config b/docker/blocks/mysql/config new file mode 100644 index 00000000000..53f8949d9bc --- /dev/null +++ b/docker/blocks/mysql/config @@ -0,0 +1,6 @@ +[database] +DB_TYPE = mysql +HOST = ${DB_1_PORT_3306_TCP_ADDR}:${DB_1_PORT_3306_TCP_PORT} +NAME = ${DB_1_ENV_MYSQL_DATABASE} +USER = ${DB_1_ENV_MYSQL_USER} +PASSWD = ${DB_1_ENV_MYSQL_PASSWORD} diff --git a/docker/blocks/mysql/fig b/docker/blocks/mysql/fig new file mode 100644 index 00000000000..5adc10a8c10 --- /dev/null +++ b/docker/blocks/mysql/fig @@ -0,0 +1,7 @@ +db: + image: mysql:latest + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: grafana + MYSQL_USER: grafana + MYSQL_PASSWORD: password diff --git a/docker/build_fig.sh b/docker/build_fig.sh new file mode 100755 index 00000000000..aca4f68413f --- /dev/null +++ b/docker/build_fig.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +blocks_dir=blocks +docker_dir=docker +template_dir=templates + +grafana_config_file=conf.tmp +grafana_config=config + +fig_file=fig.yml +fig_config=fig + +if [ "$#" == 0 ]; then + blocks=`ls $blocks_dir` + if [ -z "$blocks" ]; then + echo "No Blocks available in $blocks_dir" + else + echo "Available Blocks:" + for block in $blocks; do + echo " $block" + done + fi + exit 0 +fi + +for file in $gogs_config_file $fig_file; do + if [ -e $file ]; then + echo "Deleting $file" + rm $file + fi +done + +for dir in $@; do + current_dir=$blocks_dir/$dir + if [ ! -d "$current_dir" ]; then + echo "$current_dir is not a directory" + exit 1 + fi + + if [ -e $current_dir/$grafana_config ]; then + echo "Adding $current_dir/$grafana_config to $grafana_config_file" + cat $current_dir/$grafana_config >> $grafana_config_file + echo "" >> $grafana_config_file + fi + + if [ -e $current_dir/$fig_config ]; then + echo "Adding $current_dir/$fig_config to $fig_file" + cat $current_dir/fig >> $fig_file + echo "" >> $fig_file + fi +done + diff --git a/docker/fig.yml b/docker/fig.yml index 78246136dfb..989b4d90600 100644 --- a/docker/fig.yml +++ b/docker/fig.yml @@ -1,9 +1,5 @@ -db: - image: mysql:latest +graphite: + build: blocks/docker_graphite ports: - - "3306:3306" - environment: - MYSQL_ROOT_PASSWORD: rootpass - MYSQL_DATABASE: grafana - MYSQL_USER: grafana - MYSQL_PASSWORD: password + - "8776:80" + diff --git a/grafana b/grafana index 979349388ce..f332f22bed9 160000 --- a/grafana +++ b/grafana @@ -1 +1 @@ -Subproject commit 979349388cee6f9f35f4b85b473cfe05e86ef57f +Subproject commit f332f22bed9176c7a6292a2230fded2498039432 diff --git a/pkg/api/api.go b/pkg/api/api.go index 2fa504c02f3..434535c6891 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -16,8 +16,8 @@ func Register(m *macaron.Macaron) { m.Post("/login", LoginPost) // login - m.Get("/login", Index) m.Get("/login/:name", OAuthLogin) + m.Get("/login", Index) // account m.Get("/account/", auth, Index) diff --git a/pkg/cmd/web.go b/pkg/cmd/web.go index 92ea3b09283..3ba64fb136c 100644 --- a/pkg/cmd/web.go +++ b/pkg/cmd/web.go @@ -44,7 +44,7 @@ func newMacaron() *macaron.Macaron { mapStatic(m, "app", "app") mapStatic(m, "css", "css") mapStatic(m, "img", "img") - mapStatic(m, "font", "font") + mapStatic(m, "fonts", "fonts") m.Use(session.Sessioner(setting.SessionOptions))