moves metric package to /infra

ref #14679

pkg/infra/metrics/graphitebridge/graphite.go

@@ -0,0 +1,409 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package graphite provides a bridge to push Prometheus metrics to a Graphite
+// server.
+package graphitebridge
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"sort"
+	"strings"
+	"time"
+
+	"context"
+
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+
+	dto "github.com/prometheus/client_model/go"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	defaultInterval       = 15 * time.Second
+	millisecondsPerSecond = 1000
+)
+
+// HandlerErrorHandling defines how a Handler serving metrics will handle
+// errors.
+type HandlerErrorHandling int
+
+// These constants cause handlers serving metrics to behave as described if
+// errors are encountered.
+const (
+	// Ignore errors and try to push as many metrics to Graphite as possible.
+	ContinueOnError HandlerErrorHandling = iota
+
+	// Abort the push to Graphite upon the first error encountered.
+	AbortOnError
+)
+
+var metricCategoryPrefix = []string{
+	"proxy_",
+	"api_",
+	"page_",
+	"alerting_",
+	"aws_",
+	"db_",
+	"stat_",
+	"go_",
+	"process_"}
+
+var trimMetricPrefix = []string{"grafana_"}
+
+// Config defines the Graphite bridge config.
+type Config struct {
+	// The url to push data to. Required.
+	URL string
+
+	// The prefix for the pushed Graphite metrics. Defaults to empty string.
+	Prefix string
+
+	// The interval to use for pushing data to Graphite. Defaults to 15 seconds.
+	Interval time.Duration
+
+	// The timeout for pushing metrics to Graphite. Defaults to 15 seconds.
+	Timeout time.Duration
+
+	// The Gatherer to use for metrics. Defaults to prometheus.DefaultGatherer.
+	Gatherer prometheus.Gatherer
+
+	// The logger that messages are written to. Defaults to no logging.
+	Logger Logger
+
+	// ErrorHandling defines how errors are handled. Note that errors are
+	// logged regardless of the configured ErrorHandling provided Logger
+	// is not nil.
+	ErrorHandling HandlerErrorHandling
+
+	// Graphite does not support ever increasing counter the same way
+	// prometheus does. Rollups and ingestion might cannot handle ever
+	// increasing counters. This option allows enabled the caller to
+	// calculate the delta by saving the last sent counter in memory
+	// and subtraction it from the collected value before sending.
+	CountersAsDelta bool
+}
+
+// Bridge pushes metrics to the configured Graphite server.
+type Bridge struct {
+	url              string
+	prefix           string
+	countersAsDetlas bool
+	interval         time.Duration
+	timeout          time.Duration
+
+	errorHandling HandlerErrorHandling
+	logger        Logger
+
+	g prometheus.Gatherer
+
+	lastValue map[model.Fingerprint]float64
+}
+
+// Logger is the minimal interface Bridge needs for logging. Note that
+// log.Logger from the standard library implements this interface, and it is
+// easy to implement by custom loggers, if they don't do so already anyway.
+type Logger interface {
+	Println(v ...interface{})
+}
+
+// NewBridge returns a pointer to a new Bridge struct.
+func NewBridge(c *Config) (*Bridge, error) {
+	b := &Bridge{}
+
+	if c.URL == "" {
+		return nil, errors.New("missing URL")
+	}
+	b.url = c.URL
+
+	if c.Gatherer == nil {
+		b.g = prometheus.DefaultGatherer
+	} else {
+		b.g = c.Gatherer
+	}
+
+	if c.Logger != nil {
+		b.logger = c.Logger
+	}
+
+	if c.Prefix != "" {
+		b.prefix = c.Prefix
+	}
+
+	var z time.Duration
+	if c.Interval == z {
+		b.interval = defaultInterval
+	} else {
+		b.interval = c.Interval
+	}
+
+	if c.Timeout == z {
+		b.timeout = defaultInterval
+	} else {
+		b.timeout = c.Timeout
+	}
+
+	b.errorHandling = c.ErrorHandling
+	b.lastValue = map[model.Fingerprint]float64{}
+	b.countersAsDetlas = c.CountersAsDelta
+
+	return b, nil
+}
+
+// Run starts the event loop that pushes Prometheus metrics to Graphite at the
+// configured interval.
+func (b *Bridge) Run(ctx context.Context) {
+	ticker := time.NewTicker(b.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			if err := b.Push(); err != nil && b.logger != nil {
+				b.logger.Println("error pushing to Graphite:", err)
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// Push pushes Prometheus metrics to the configured Graphite server.
+func (b *Bridge) Push() error {
+	mfs, err := b.g.Gather()
+	if err != nil || len(mfs) == 0 {
+		switch b.errorHandling {
+		case AbortOnError:
+			return err
+		case ContinueOnError:
+			if b.logger != nil {
+				b.logger.Println("continue on error:", err)
+			}
+		default:
+			panic("unrecognized error handling value")
+		}
+	}
+
+	conn, err := net.DialTimeout("tcp", b.url, b.timeout)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	return b.writeMetrics(conn, mfs, b.prefix, model.Now())
+}
+
+func (b *Bridge) writeMetrics(w io.Writer, mfs []*dto.MetricFamily, prefix string, now model.Time) error {
+	for _, mf := range mfs {
+		vec, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{
+			Timestamp: now,
+		}, mf)
+		if err != nil {
+			return err
+		}
+
+		buf := bufio.NewWriter(w)
+		for _, s := range vec {
+			if math.IsNaN(float64(s.Value)) {
+				continue
+			}
+
+			if err := writePrefix(buf, prefix); err != nil {
+				return err
+			}
+
+			if err := writeMetric(buf, s.Metric, mf); err != nil {
+				return err
+			}
+
+			value := b.replaceCounterWithDelta(mf, s.Metric, s.Value)
+			if _, err := fmt.Fprintf(buf, " %g %d\n", value, int64(s.Timestamp)/millisecondsPerSecond); err != nil {
+				return err
+			}
+			if err := buf.Flush(); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func writeMetric(buf *bufio.Writer, m model.Metric, mf *dto.MetricFamily) error {
+	metricName, hasName := m[model.MetricNameLabel]
+	numLabels := len(m) - 1
+	if !hasName {
+		numLabels = len(m)
+	}
+
+	for _, v := range trimMetricPrefix {
+		if strings.HasPrefix(string(metricName), v) {
+			metricName = model.LabelValue(strings.Replace(string(metricName), v, "", 1))
+		}
+	}
+
+	for _, v := range metricCategoryPrefix {
+		if strings.HasPrefix(string(metricName), v) {
+			group := strings.Replace(v, "_", " ", 1)
+			metricName = model.LabelValue(strings.Replace(string(metricName), v, group, 1))
+		}
+	}
+
+	labelStrings := make([]string, 0, numLabels)
+	for label, value := range m {
+		if label != model.MetricNameLabel {
+			labelStrings = append(labelStrings, fmt.Sprintf("%s %s", string(label), string(value)))
+		}
+	}
+
+	var err error
+	switch numLabels {
+	case 0:
+		if hasName {
+			if err := writeSanitized(buf, string(metricName)); err != nil {
+				return err
+			}
+		}
+	default:
+		sort.Strings(labelStrings)
+		if err = writeSanitized(buf, string(metricName)); err != nil {
+			return err
+		}
+		for _, s := range labelStrings {
+			if err = buf.WriteByte('.'); err != nil {
+				return err
+			}
+			if err = writeSanitized(buf, s); err != nil {
+				return err
+			}
+		}
+	}
+
+	return addExtentionConventionForRollups(buf, mf, m)
+}
+
+func addExtentionConventionForRollups(buf *bufio.Writer, mf *dto.MetricFamily, m model.Metric) error {
+	// Adding `.count` `.sum` suffix makes it possible to configure
+	// different rollup strategies based on metric type
+
+	mfType := mf.GetType()
+	var err error
+	if mfType == dto.MetricType_COUNTER {
+		if _, err = fmt.Fprint(buf, ".count"); err != nil {
+			return err
+		}
+	}
+
+	if mfType == dto.MetricType_SUMMARY || mfType == dto.MetricType_HISTOGRAM {
+		if strings.HasSuffix(string(m[model.MetricNameLabel]), "_count") {
+			if _, err = fmt.Fprint(buf, ".count"); err != nil {
+				return err
+			}
+		}
+	}
+	if mfType == dto.MetricType_HISTOGRAM {
+		if strings.HasSuffix(string(m[model.MetricNameLabel]), "_sum") {
+			if _, err = fmt.Fprint(buf, ".sum"); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func writePrefix(buf *bufio.Writer, s string) error {
+	for _, c := range s {
+		if _, err := buf.WriteRune(replaceInvalid(c)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func writeSanitized(buf *bufio.Writer, s string) error {
+	prevUnderscore := false
+
+	for _, c := range s {
+		c = replaceInvalidRune(c)
+		if c == '_' {
+			if prevUnderscore {
+				continue
+			}
+			prevUnderscore = true
+		} else {
+			prevUnderscore = false
+		}
+		if _, err := buf.WriteRune(c); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func replaceInvalid(c rune) rune {
+	if c == ' ' || c == '.' {
+		return '.'
+	}
+	return replaceInvalidRune(c)
+}
+
+func replaceInvalidRune(c rune) rune {
+	if c == ' ' {
+		return '.'
+	}
+	if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '-' || c == '_' || c == ':' || (c >= '0' && c <= '9')) {
+		return '_'
+	}
+	return c
+}
+
+func (b *Bridge) replaceCounterWithDelta(mf *dto.MetricFamily, metric model.Metric, value model.SampleValue) float64 {
+	if !b.countersAsDetlas {
+		return float64(value)
+	}
+
+	mfType := mf.GetType()
+	if mfType == dto.MetricType_COUNTER {
+		return b.returnDelta(metric, value)
+	}
+
+	if mfType == dto.MetricType_SUMMARY {
+		if strings.HasSuffix(string(metric[model.MetricNameLabel]), "_count") {
+			return b.returnDelta(metric, value)
+		}
+	}
+
+	return float64(value)
+}
+
+func (b *Bridge) returnDelta(metric model.Metric, value model.SampleValue) float64 {
+	key := metric.Fingerprint()
+	_, exists := b.lastValue[key]
+	if !exists {
+		b.lastValue[key] = 0
+	}
+
+	delta := float64(value) - b.lastValue[key]
+	b.lastValue[key] = float64(value)
+
+	return delta
+}

pkg/infra/metrics/graphitebridge/graphite_test.go

@@ -0,0 +1,568 @@
+package graphitebridge
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"net"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/model"
+)
+
+func TestCountersAsDelta(t *testing.T) {
+	b, _ := NewBridge(&Config{
+		URL:             "localhost:12345",
+		CountersAsDelta: true,
+	})
+	ty := dto.MetricType(0)
+	mf := &dto.MetricFamily{
+		Type:   &ty,
+		Metric: []*dto.Metric{},
+	}
+	m := model.Metric{}
+
+	var want float64
+	var got float64
+	want = float64(1)
+	got = b.replaceCounterWithDelta(mf, m, model.SampleValue(1))
+	if got != want {
+		t.Fatalf("want %v got %v", want, got)
+	}
+
+	got = b.replaceCounterWithDelta(mf, m, model.SampleValue(2))
+	if got != want {
+		t.Fatalf("want %v got %v", want, got)
+	}
+}
+
+func TestCountersAsDeltaDisabled(t *testing.T) {
+	b, _ := NewBridge(&Config{
+		URL:             "localhost:12345",
+		CountersAsDelta: false,
+	})
+	ty := dto.MetricType(0)
+	mf := &dto.MetricFamily{
+		Type:   &ty,
+		Metric: []*dto.Metric{},
+	}
+	m := model.Metric{}
+
+	var want float64
+	var got float64
+	want = float64(1)
+	got = b.replaceCounterWithDelta(mf, m, model.SampleValue(1))
+	if got != want {
+		t.Fatalf("want %v got %v", want, got)
+	}
+
+	want = float64(2)
+	got = b.replaceCounterWithDelta(mf, m, model.SampleValue(2))
+	if got != want {
+		t.Fatalf("want %v got %v", want, got)
+	}
+}
+
+func TestSanitize(t *testing.T) {
+	testCases := []struct {
+		in, out string
+	}{
+		{in: "hello", out: "hello"},
+		{in: "hE/l1o", out: "hE_l1o"},
+		{in: "he,*ll(.o", out: "he_ll_o"},
+		{in: "hello_there%^&", out: "hello_there_"},
+	}
+
+	var buf bytes.Buffer
+	w := bufio.NewWriter(&buf)
+
+	for i, tc := range testCases {
+		if err := writeSanitized(w, tc.in); err != nil {
+			t.Fatalf("write failed: %v", err)
+		}
+		if err := w.Flush(); err != nil {
+			t.Fatalf("flush failed: %v", err)
+		}
+
+		if want, got := tc.out, buf.String(); want != got {
+			t.Fatalf("test case index %d: got sanitized string %s, want %s", i, got, want)
+		}
+
+		buf.Reset()
+	}
+}
+
+func TestSanitizePrefix(t *testing.T) {
+	testCases := []struct {
+		in, out string
+	}{
+		{in: "service.prod.", out: "service.prod."},
+		{in: "service.prod", out: "service.prod"},
+	}
+
+	var buf bytes.Buffer
+	w := bufio.NewWriter(&buf)
+
+	for i, tc := range testCases {
+		if err := writePrefix(w, tc.in); err != nil {
+			t.Fatalf("write failed: %v", err)
+		}
+		if err := w.Flush(); err != nil {
+			t.Fatalf("flush failed: %v", err)
+		}
+
+		if want, got := tc.out, buf.String(); want != got {
+			t.Fatalf("test case index %d: got sanitized string %s, want %s", i, got, want)
+		}
+
+		buf.Reset()
+	}
+}
+
+func TestWriteSummary(t *testing.T) {
+	sumVec := prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:        "name",
+			Help:        "docstring",
+			Namespace:   "grafana",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+			Objectives:  map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"labelname"},
+	)
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(sumVec)
+
+	b, err := NewBridge(&Config{
+		URL:             "localhost:8080",
+		Gatherer:        reg,
+		CountersAsDelta: true,
+	})
+	if err != nil {
+		t.Fatalf("cannot create bridge. err: %v", err)
+	}
+
+	sumVec.WithLabelValues("val1").Observe(float64(10))
+	sumVec.WithLabelValues("val1").Observe(float64(20))
+	sumVec.WithLabelValues("val1").Observe(float64(30))
+	sumVec.WithLabelValues("val2").Observe(float64(20))
+	sumVec.WithLabelValues("val2").Observe(float64(30))
+	sumVec.WithLabelValues("val2").Observe(float64(40))
+
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	now := model.Time(1477043083)
+	var buf bytes.Buffer
+	err = b.writeMetrics(&buf, mfs, "prefix.", now)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want := `prefix.name.constname.constvalue.labelname.val1.quantile.0_5 20 1477043
+prefix.name.constname.constvalue.labelname.val1.quantile.0_9 30 1477043
+prefix.name.constname.constvalue.labelname.val1.quantile.0_99 30 1477043
+prefix.name_sum.constname.constvalue.labelname.val1 60 1477043
+prefix.name_count.constname.constvalue.labelname.val1.count 3 1477043
+prefix.name.constname.constvalue.labelname.val2.quantile.0_5 30 1477043
+prefix.name.constname.constvalue.labelname.val2.quantile.0_9 40 1477043
+prefix.name.constname.constvalue.labelname.val2.quantile.0_99 40 1477043
+prefix.name_sum.constname.constvalue.labelname.val2 90 1477043
+prefix.name_count.constname.constvalue.labelname.val2.count 3 1477043
+`
+
+	if got := buf.String(); want != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want, got)
+	}
+}
+
+func TestWriteHistogram(t *testing.T) {
+	histVec := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:        "name",
+			Help:        "docstring",
+			Namespace:   "grafana",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+			Buckets:     []float64{0.01, 0.02, 0.05, 0.1},
+		},
+		[]string{"labelname"},
+	)
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(histVec)
+
+	b, err := NewBridge(&Config{
+		URL:             "localhost:8080",
+		Gatherer:        reg,
+		CountersAsDelta: true,
+	})
+	if err != nil {
+		t.Fatalf("error creating bridge: %v", err)
+	}
+
+	histVec.WithLabelValues("val1").Observe(float64(10))
+	histVec.WithLabelValues("val1").Observe(float64(20))
+	histVec.WithLabelValues("val1").Observe(float64(30))
+	histVec.WithLabelValues("val2").Observe(float64(20))
+	histVec.WithLabelValues("val2").Observe(float64(30))
+	histVec.WithLabelValues("val2").Observe(float64(40))
+
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	now := model.Time(1477043083)
+	var buf bytes.Buffer
+	err = b.writeMetrics(&buf, mfs, "prefix.", now)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want := `prefix.name_bucket.constname.constvalue.labelname.val1.le.0_01 0 1477043
+prefix.name_bucket.constname.constvalue.labelname.val1.le.0_02 0 1477043
+prefix.name_bucket.constname.constvalue.labelname.val1.le.0_05 0 1477043
+prefix.name_bucket.constname.constvalue.labelname.val1.le.0_1 0 1477043
+prefix.name_sum.constname.constvalue.labelname.val1.sum 60 1477043
+prefix.name_count.constname.constvalue.labelname.val1.count 3 1477043
+prefix.name_bucket.constname.constvalue.labelname.val1.le._Inf 3 1477043
+prefix.name_bucket.constname.constvalue.labelname.val2.le.0_01 0 1477043
+prefix.name_bucket.constname.constvalue.labelname.val2.le.0_02 0 1477043
+prefix.name_bucket.constname.constvalue.labelname.val2.le.0_05 0 1477043
+prefix.name_bucket.constname.constvalue.labelname.val2.le.0_1 0 1477043
+prefix.name_sum.constname.constvalue.labelname.val2.sum 90 1477043
+prefix.name_count.constname.constvalue.labelname.val2.count 3 1477043
+prefix.name_bucket.constname.constvalue.labelname.val2.le._Inf 3 1477043
+`
+	if got := buf.String(); want != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want, got)
+	}
+}
+
+func TestCounterVec(t *testing.T) {
+	cntVec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name:        "page_response",
+			Namespace:   "grafana",
+			Help:        "docstring",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+		},
+		[]string{"labelname"},
+	)
+
+	apicntVec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name:        "api_response",
+			Namespace:   "grafana",
+			Help:        "docstring",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+		},
+		[]string{"labelname"},
+	)
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(cntVec)
+	reg.MustRegister(apicntVec)
+
+	cntVec.WithLabelValues("val1").Inc()
+	cntVec.WithLabelValues("val2").Inc()
+	apicntVec.WithLabelValues("val1").Inc()
+	apicntVec.WithLabelValues("val2").Inc()
+
+	b, err := NewBridge(&Config{
+		URL:             "localhost:8080",
+		Gatherer:        reg,
+		CountersAsDelta: true,
+	})
+	if err != nil {
+		t.Fatalf("error creating bridge: %v", err)
+	}
+
+	// first collect
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	err = b.writeMetrics(&buf, mfs, "prefix.", model.Time(1477043083))
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want := `prefix.api.response.constname.constvalue.labelname.val1.count 1 1477043
+prefix.api.response.constname.constvalue.labelname.val2.count 1 1477043
+prefix.page.response.constname.constvalue.labelname.val1.count 1 1477043
+prefix.page.response.constname.constvalue.labelname.val2.count 1 1477043
+`
+	if got := buf.String(); want != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want, got)
+	}
+
+	//next collect
+	cntVec.WithLabelValues("val1").Inc()
+	cntVec.WithLabelValues("val2").Inc()
+	apicntVec.WithLabelValues("val1").Inc()
+	apicntVec.WithLabelValues("val2").Inc()
+
+	mfs, err = reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	buf = bytes.Buffer{}
+	err = b.writeMetrics(&buf, mfs, "prefix.", model.Time(1477053083))
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want2 := `prefix.api.response.constname.constvalue.labelname.val1.count 1 1477053
+prefix.api.response.constname.constvalue.labelname.val2.count 1 1477053
+prefix.page.response.constname.constvalue.labelname.val1.count 1 1477053
+prefix.page.response.constname.constvalue.labelname.val2.count 1 1477053
+`
+	if got := buf.String(); want2 != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want2, got)
+	}
+}
+
+func TestCounter(t *testing.T) {
+	cntVec := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "page_response",
+			Help:        "docstring",
+			Namespace:   "grafana",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+		})
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(cntVec)
+
+	cntVec.Inc()
+
+	b, err := NewBridge(&Config{
+		URL:             "localhost:8080",
+		Gatherer:        reg,
+		CountersAsDelta: true,
+	})
+	if err != nil {
+		t.Fatalf("error creating bridge: %v", err)
+	}
+
+	// first collect
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	err = b.writeMetrics(&buf, mfs, "prefix.", model.Time(1477043083))
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want := "prefix.page.response.constname.constvalue.count 1 1477043\n"
+	if got := buf.String(); want != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want, got)
+	}
+
+	//next collect
+	cntVec.Inc()
+
+	mfs, err = reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	buf = bytes.Buffer{}
+	err = b.writeMetrics(&buf, mfs, "prefix.", model.Time(1477053083))
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want2 := "prefix.page.response.constname.constvalue.count 1 1477053\n"
+	if got := buf.String(); want2 != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want2, got)
+	}
+}
+
+func TestTrimGrafanaNamespace(t *testing.T) {
+	cntVec := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name:        "http_request_total",
+			Help:        "docstring",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+		})
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(cntVec)
+
+	cntVec.Inc()
+
+	b, err := NewBridge(&Config{
+		URL:             "localhost:8080",
+		Gatherer:        reg,
+		CountersAsDelta: true,
+	})
+	if err != nil {
+		t.Fatalf("error creating bridge: %v", err)
+	}
+
+	// first collect
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	err = b.writeMetrics(&buf, mfs, "prefix.", model.Time(1477043083))
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want := "prefix.http_request_total.constname.constvalue.count 1 1477043\n"
+	if got := buf.String(); want != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want, got)
+	}
+}
+
+func TestSkipNanValues(t *testing.T) {
+	cntVec := prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Name:        "http_request_total",
+			Help:        "docstring",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+		})
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(cntVec)
+
+	b, err := NewBridge(&Config{
+		URL:             "localhost:8080",
+		Gatherer:        reg,
+		CountersAsDelta: true,
+	})
+	if err != nil {
+		t.Fatalf("error creating bridge: %v", err)
+	}
+
+	// first collect
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	err = b.writeMetrics(&buf, mfs, "prefix.", model.Time(1477043083))
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	want := `prefix.http_request_total_sum.constname.constvalue 0 1477043
+prefix.http_request_total_count.constname.constvalue.count 0 1477043
+`
+
+	if got := buf.String(); want != got {
+		t.Fatalf("wanted \n%s\n, got \n%s\n", want, got)
+	}
+}
+
+func TestPush(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	cntVec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name:        "name",
+			Help:        "docstring",
+			Namespace:   "grafana",
+			ConstLabels: prometheus.Labels{"constname": "constvalue"},
+		},
+		[]string{"labelname"},
+	)
+	cntVec.WithLabelValues("val1").Inc()
+	cntVec.WithLabelValues("val2").Inc()
+	reg.MustRegister(cntVec)
+
+	host := "localhost"
+	port := ":56789"
+	b, err := NewBridge(&Config{
+		URL:      host + port,
+		Gatherer: reg,
+		Prefix:   "prefix.",
+	})
+	if err != nil {
+		t.Fatalf("error creating bridge: %v", err)
+	}
+
+	nmg, err := newMockGraphite(port)
+	if err != nil {
+		t.Fatalf("error creating mock graphite: %v", err)
+	}
+	defer nmg.Close()
+
+	err = b.Push()
+	if err != nil {
+		t.Fatalf("error pushing: %v", err)
+	}
+
+	wants := []string{
+		"prefix.name.constname.constvalue.labelname.val1.count 1",
+		"prefix.name.constname.constvalue.labelname.val2.count 1",
+	}
+
+	select {
+	case got := <-nmg.readc:
+		for _, want := range wants {
+			matched, err := regexp.MatchString(want, got)
+			if err != nil {
+				t.Fatalf("error pushing: %v", err)
+			}
+			if !matched {
+				t.Fatalf("missing metric:\nno match for %s received by server:\n%s", want, got)
+			}
+		}
+		return
+	case err := <-nmg.errc:
+		t.Fatalf("error reading push: %v", err)
+	case <-time.After(50 * time.Millisecond):
+		t.Fatalf("no result from graphite server")
+	}
+}
+
+func newMockGraphite(port string) (*mockGraphite, error) {
+	readc := make(chan string)
+	errc := make(chan error)
+	ln, err := net.Listen("tcp", port)
+	if err != nil {
+		return nil, err
+	}
+
+	go func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			errc <- err
+		}
+		var b bytes.Buffer
+		io.Copy(&b, conn)
+		readc <- b.String()
+	}()
+
+	return &mockGraphite{
+		readc:    readc,
+		errc:     errc,
+		Listener: ln,
+	}, nil
+}
+
+type mockGraphite struct {
+	readc chan string
+	errc  chan error
+
+	net.Listener
+}

pkg/infra/metrics/metrics.go

@@ -0,0 +1,361 @@
+package metrics
+
+import (
+	"runtime"
+
+	"github.com/grafana/grafana/pkg/setting"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const exporterName = "grafana"
+
+var (
+	M_Instance_Start       prometheus.Counter
+	M_Page_Status          *prometheus.CounterVec
+	M_Api_Status           *prometheus.CounterVec
+	M_Proxy_Status         *prometheus.CounterVec
+	M_Http_Request_Total   *prometheus.CounterVec
+	M_Http_Request_Summary *prometheus.SummaryVec
+
+	M_Api_User_SignUpStarted   prometheus.Counter
+	M_Api_User_SignUpCompleted prometheus.Counter
+	M_Api_User_SignUpInvite    prometheus.Counter
+	M_Api_Dashboard_Save       prometheus.Summary
+	M_Api_Dashboard_Get        prometheus.Summary
+	M_Api_Dashboard_Search     prometheus.Summary
+	M_Api_Admin_User_Create    prometheus.Counter
+	M_Api_Login_Post           prometheus.Counter
+	M_Api_Login_OAuth          prometheus.Counter
+	M_Api_Org_Create           prometheus.Counter
+
+	M_Api_Dashboard_Snapshot_Create      prometheus.Counter
+	M_Api_Dashboard_Snapshot_External    prometheus.Counter
+	M_Api_Dashboard_Snapshot_Get         prometheus.Counter
+	M_Api_Dashboard_Insert               prometheus.Counter
+	M_Alerting_Result_State              *prometheus.CounterVec
+	M_Alerting_Notification_Sent         *prometheus.CounterVec
+	M_Aws_CloudWatch_GetMetricStatistics prometheus.Counter
+	M_Aws_CloudWatch_ListMetrics         prometheus.Counter
+	M_Aws_CloudWatch_GetMetricData       prometheus.Counter
+	M_DB_DataSource_QueryById            prometheus.Counter
+
+	// Timers
+	M_DataSource_ProxyReq_Timer prometheus.Summary
+	M_Alerting_Execution_Time   prometheus.Summary
+
+	// StatTotals
+	M_Alerting_Active_Alerts prometheus.Gauge
+	M_StatTotal_Dashboards   prometheus.Gauge
+	M_StatTotal_Users        prometheus.Gauge
+	M_StatActive_Users       prometheus.Gauge
+	M_StatTotal_Orgs         prometheus.Gauge
+	M_StatTotal_Playlists    prometheus.Gauge
+
+	// M_Grafana_Version is a gauge that contains build info about this binary
+	//
+	// Deprecated: use M_Grafana_Build_Version instead.
+	M_Grafana_Version *prometheus.GaugeVec
+
+	// grafanaBuildVersion is a gauge that contains build info about this binary
+	grafanaBuildVersion *prometheus.GaugeVec
+)
+
+func init() {
+	M_Instance_Start = prometheus.NewCounter(prometheus.CounterOpts{
+		Name:      "instance_start_total",
+		Help:      "counter for started instances",
+		Namespace: exporterName,
+	})
+
+	httpStatusCodes := []string{"200", "404", "500", "unknown"}
+	M_Page_Status = newCounterVecStartingAtZero(
+		prometheus.CounterOpts{
+			Name:      "page_response_status_total",
+			Help:      "page http response status",
+			Namespace: exporterName,
+		}, []string{"code"}, httpStatusCodes...)
+
+	M_Api_Status = newCounterVecStartingAtZero(
+		prometheus.CounterOpts{
+			Name:      "api_response_status_total",
+			Help:      "api http response status",
+			Namespace: exporterName,
+		}, []string{"code"}, httpStatusCodes...)
+
+	M_Proxy_Status = newCounterVecStartingAtZero(
+		prometheus.CounterOpts{
+			Name:      "proxy_response_status_total",
+			Help:      "proxy http response status",
+			Namespace: exporterName,
+		}, []string{"code"}, httpStatusCodes...)
+
+	M_Http_Request_Total = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_request_total",
+			Help: "http request counter",
+		},
+		[]string{"handler", "statuscode", "method"},
+	)
+
+	M_Http_Request_Summary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "http_request_duration_milliseconds",
+			Help: "http request summary",
+		},
+		[]string{"handler", "statuscode", "method"},
+	)
+
+	M_Api_User_SignUpStarted = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_user_signup_started_total",
+		Help:      "amount of users who started the signup flow",
+		Namespace: exporterName,
+	})
+
+	M_Api_User_SignUpCompleted = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_user_signup_completed_total",
+		Help:      "amount of users who completed the signup flow",
+		Namespace: exporterName,
+	})
+
+	M_Api_User_SignUpInvite = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_user_signup_invite_total",
+		Help:      "amount of users who have been invited",
+		Namespace: exporterName,
+	})
+
+	M_Api_Dashboard_Save = prometheus.NewSummary(prometheus.SummaryOpts{
+		Name:      "api_dashboard_save_milliseconds",
+		Help:      "summary for dashboard save duration",
+		Namespace: exporterName,
+	})
+
+	M_Api_Dashboard_Get = prometheus.NewSummary(prometheus.SummaryOpts{
+		Name:      "api_dashboard_get_milliseconds",
+		Help:      "summary for dashboard get duration",
+		Namespace: exporterName,
+	})
+
+	M_Api_Dashboard_Search = prometheus.NewSummary(prometheus.SummaryOpts{
+		Name:      "api_dashboard_search_milliseconds",
+		Help:      "summary for dashboard search duration",
+		Namespace: exporterName,
+	})
+
+	M_Api_Admin_User_Create = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_admin_user_created_total",
+		Help:      "api admin user created counter",
+		Namespace: exporterName,
+	})
+
+	M_Api_Login_Post = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_login_post_total",
+		Help:      "api login post counter",
+		Namespace: exporterName,
+	})
+
+	M_Api_Login_OAuth = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_login_oauth_total",
+		Help:      "api login oauth counter",
+		Namespace: exporterName,
+	})
+
+	M_Api_Org_Create = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_org_create_total",
+		Help:      "api org created counter",
+		Namespace: exporterName,
+	})
+
+	M_Api_Dashboard_Snapshot_Create = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_dashboard_snapshot_create_total",
+		Help:      "dashboard snapshots created",
+		Namespace: exporterName,
+	})
+
+	M_Api_Dashboard_Snapshot_External = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_dashboard_snapshot_external_total",
+		Help:      "external dashboard snapshots created",
+		Namespace: exporterName,
+	})
+
+	M_Api_Dashboard_Snapshot_Get = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_dashboard_snapshot_get_total",
+		Help:      "loaded dashboards",
+		Namespace: exporterName,
+	})
+
+	M_Api_Dashboard_Insert = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "api_models_dashboard_insert_total",
+		Help:      "dashboards inserted ",
+		Namespace: exporterName,
+	})
+
+	M_Alerting_Result_State = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name:      "alerting_result_total",
+		Help:      "alert execution result counter",
+		Namespace: exporterName,
+	}, []string{"state"})
+
+	M_Alerting_Notification_Sent = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name:      "alerting_notification_sent_total",
+		Help:      "counter for how many alert notifications been sent",
+		Namespace: exporterName,
+	}, []string{"type"})
+
+	M_Aws_CloudWatch_GetMetricStatistics = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "aws_cloudwatch_get_metric_statistics_total",
+		Help:      "counter for getting metric statistics from aws",
+		Namespace: exporterName,
+	})
+
+	M_Aws_CloudWatch_ListMetrics = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "aws_cloudwatch_list_metrics_total",
+		Help:      "counter for getting list of metrics from aws",
+		Namespace: exporterName,
+	})
+
+	M_Aws_CloudWatch_GetMetricData = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "aws_cloudwatch_get_metric_data_total",
+		Help:      "counter for getting metric data time series from aws",
+		Namespace: exporterName,
+	})
+
+	M_DB_DataSource_QueryById = newCounterStartingAtZero(prometheus.CounterOpts{
+		Name:      "db_datasource_query_by_id_total",
+		Help:      "counter for getting datasource by id",
+		Namespace: exporterName,
+	})
+
+	M_DataSource_ProxyReq_Timer = prometheus.NewSummary(prometheus.SummaryOpts{
+		Name:      "api_dataproxy_request_all_milliseconds",
+		Help:      "summary for dataproxy request duration",
+		Namespace: exporterName,
+	})
+
+	M_Alerting_Execution_Time = prometheus.NewSummary(prometheus.SummaryOpts{
+		Name:      "alerting_execution_time_milliseconds",
+		Help:      "summary of alert exeuction duration",
+		Namespace: exporterName,
+	})
+
+	M_Alerting_Active_Alerts = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "alerting_active_alerts",
+		Help:      "amount of active alerts",
+		Namespace: exporterName,
+	})
+
+	M_StatTotal_Dashboards = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_totals_dashboard",
+		Help:      "total amount of dashboards",
+		Namespace: exporterName,
+	})
+
+	M_StatTotal_Users = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_total_users",
+		Help:      "total amount of users",
+		Namespace: exporterName,
+	})
+
+	M_StatActive_Users = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_active_users",
+		Help:      "number of active users",
+		Namespace: exporterName,
+	})
+
+	M_StatTotal_Orgs = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_total_orgs",
+		Help:      "total amount of orgs",
+		Namespace: exporterName,
+	})
+
+	M_StatTotal_Playlists = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "stat_total_playlists",
+		Help:      "total amount of playlists",
+		Namespace: exporterName,
+	})
+
+	M_Grafana_Version = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name:      "info",
+		Help:      "Information about the Grafana. This metric is deprecated. please use `grafana_build_info`",
+		Namespace: exporterName,
+	}, []string{"version"})
+
+	grafanaBuildVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name:      "build_info",
+		Help:      "A metric with a constant '1' value labeled by version, revision, branch, and goversion from which Grafana was built.",
+		Namespace: exporterName,
+	}, []string{"version", "revision", "branch", "goversion", "edition"})
+}
+
+// SetBuildInformation sets the build information for this binary
+func SetBuildInformation(version, revision, branch string) {
+	// We export this info twice for backwards compatibility.
+	// Once this have been released for some time we should be able to remote `M_Grafana_Version`
+	// The reason we added a new one is that its common practice in the prometheus community
+	// to name this metric `*_build_info` so its easy to do aggregation on all programs.
+	edition := "oss"
+	if setting.IsEnterprise {
+		edition = "enterprise"
+	}
+
+	M_Grafana_Version.WithLabelValues(version).Set(1)
+	grafanaBuildVersion.WithLabelValues(version, revision, branch, runtime.Version(), edition).Set(1)
+}
+
+func initMetricVars() {
+	prometheus.MustRegister(
+		M_Instance_Start,
+		M_Page_Status,
+		M_Api_Status,
+		M_Proxy_Status,
+		M_Http_Request_Total,
+		M_Http_Request_Summary,
+		M_Api_User_SignUpStarted,
+		M_Api_User_SignUpCompleted,
+		M_Api_User_SignUpInvite,
+		M_Api_Dashboard_Save,
+		M_Api_Dashboard_Get,
+		M_Api_Dashboard_Search,
+		M_DataSource_ProxyReq_Timer,
+		M_Alerting_Execution_Time,
+		M_Api_Admin_User_Create,
+		M_Api_Login_Post,
+		M_Api_Login_OAuth,
+		M_Api_Org_Create,
+		M_Api_Dashboard_Snapshot_Create,
+		M_Api_Dashboard_Snapshot_External,
+		M_Api_Dashboard_Snapshot_Get,
+		M_Api_Dashboard_Insert,
+		M_Alerting_Result_State,
+		M_Alerting_Notification_Sent,
+		M_Aws_CloudWatch_GetMetricStatistics,
+		M_Aws_CloudWatch_ListMetrics,
+		M_Aws_CloudWatch_GetMetricData,
+		M_DB_DataSource_QueryById,
+		M_Alerting_Active_Alerts,
+		M_StatTotal_Dashboards,
+		M_StatTotal_Users,
+		M_StatActive_Users,
+		M_StatTotal_Orgs,
+		M_StatTotal_Playlists,
+		M_Grafana_Version,
+		grafanaBuildVersion)
+
+}
+
+func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec {
+	counter := prometheus.NewCounterVec(opts, labels)
+
+	for _, label := range labelValues {
+		counter.WithLabelValues(label).Add(0)
+	}
+
+	return counter
+}
+
+func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter {
+	counter := prometheus.NewCounter(opts)
+	counter.Add(0)
+
+	return counter
+}

pkg/infra/metrics/service.go

@@ -0,0 +1,53 @@
+package metrics
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/infra/metrics/graphitebridge"
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/registry"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+var metricsLogger log.Logger = log.New("metrics")
+
+type logWrapper struct {
+	logger log.Logger
+}
+
+func (lw *logWrapper) Println(v ...interface{}) {
+	lw.logger.Info("graphite metric bridge", v...)
+}
+
+func init() {
+	registry.RegisterService(&InternalMetricsService{})
+	initMetricVars()
+}
+
+type InternalMetricsService struct {
+	Cfg *setting.Cfg `inject:""`
+
+	intervalSeconds int64
+	graphiteCfg     *graphitebridge.Config
+}
+
+func (im *InternalMetricsService) Init() error {
+	return im.readSettings()
+}
+
+func (im *InternalMetricsService) Run(ctx context.Context) error {
+	// Start Graphite Bridge
+	if im.graphiteCfg != nil {
+		bridge, err := graphitebridge.NewBridge(im.graphiteCfg)
+		if err != nil {
+			metricsLogger.Error("failed to create graphite bridge", "error", err)
+		} else {
+			go bridge.Run(ctx)
+		}
+	}
+
+	M_Instance_Start.Inc()
+
+	<-ctx.Done()
+	return ctx.Err()
+}

pkg/infra/metrics/settings.go

@@ -0,0 +1,62 @@
+package metrics
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/metrics/graphitebridge"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func (im *InternalMetricsService) readSettings() error {
+	var section, err = im.Cfg.Raw.GetSection("metrics")
+	if err != nil {
+		return fmt.Errorf("Unable to find metrics config section %v", err)
+	}
+
+	im.intervalSeconds = section.Key("interval_seconds").MustInt64(10)
+
+	if err := im.parseGraphiteSettings(); err != nil {
+		return fmt.Errorf("Unable to parse metrics graphite section, %v", err)
+	}
+
+	return nil
+}
+
+func (im *InternalMetricsService) parseGraphiteSettings() error {
+	graphiteSection, err := im.Cfg.Raw.GetSection("metrics.graphite")
+
+	if err != nil {
+		return nil
+	}
+
+	address := graphiteSection.Key("address").String()
+	if address == "" {
+		return nil
+	}
+
+	bridgeCfg := &graphitebridge.Config{
+		URL:             address,
+		Prefix:          graphiteSection.Key("prefix").MustString("prod.grafana.%(instance_name)s"),
+		CountersAsDelta: true,
+		Gatherer:        prometheus.DefaultGatherer,
+		Interval:        time.Duration(im.intervalSeconds) * time.Second,
+		Timeout:         10 * time.Second,
+		Logger:          &logWrapper{logger: metricsLogger},
+		ErrorHandling:   graphitebridge.ContinueOnError,
+	}
+
+	safeInstanceName := strings.Replace(setting.InstanceName, ".", "_", -1)
+	prefix := graphiteSection.Key("prefix").Value()
+
+	if prefix == "" {
+		prefix = "prod.grafana.%(instance_name)s."
+	}
+
+	bridgeCfg.Prefix = strings.Replace(prefix, "%(instance_name)s", safeInstanceName, -1)
+
+	im.graphiteCfg = bridgeCfg
+	return nil
+}

pkg/infra/usagestats/usage_stats.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/grafana/grafana/pkg/metrics"
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/setting"