From d987532262435937215e72664101257058d2057e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Sun, 22 Mar 2015 15:14:00 -0400 Subject: [PATCH] Added server metrics --- conf/defaults.ini | 7 ++ pkg/api/admin_users.go | 3 + pkg/api/dashboard.go | 5 ++ pkg/api/index.go | 2 +- pkg/api/login.go | 4 +- pkg/api/login_oauth.go | 3 + pkg/api/org.go | 3 + pkg/api/signup.go | 3 + pkg/cmd/web.go | 5 ++ pkg/metrics/counter.go | 72 ++++++++++++++++++++ pkg/metrics/metric_ref.go | 39 +++++++++++ pkg/metrics/metrics.go | 25 +++++++ pkg/metrics/registry.go | 102 +++++++++++++++++++++++++++++ pkg/metrics/report_usage.go | 60 +++++++++++++++++ pkg/middleware/middleware.go | 12 ++++ pkg/models/stats.go | 11 ++++ pkg/services/sqlstore/dashboard.go | 2 + pkg/services/sqlstore/stats.go | 36 ++++++++++ pkg/setting/setting.go | 4 ++ 19 files changed, 396 insertions(+), 2 deletions(-) create mode 100644 pkg/metrics/counter.go create mode 100644 pkg/metrics/metric_ref.go create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/metrics/registry.go create mode 100644 pkg/metrics/report_usage.go create mode 100644 pkg/models/stats.go create mode 100644 pkg/services/sqlstore/stats.go diff --git a/conf/defaults.ini b/conf/defaults.ini index 5318e89b2ad..095457dd2a2 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1,6 +1,13 @@ app_name = Grafana app_mode = production +# Once every 24 hours Grafana will report anonymous data to +# stats.grafana.org (https). No ip addresses are being tracked. +# only simple counters to track running instances, dashboard +# count and errors. It is very helpful to us. +# Change this option to false to disable reporting. +reporting-enabled = true + [server] ; protocol (http or https) protocol = http diff --git a/pkg/api/admin_users.go b/pkg/api/admin_users.go index d3fa111d333..f7e8fca2b5e 100644 --- a/pkg/api/admin_users.go +++ b/pkg/api/admin_users.go @@ -3,6 +3,7 @@ package api import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/util" @@ -64,6 +65,8 @@ func AdminCreateUser(c *middleware.Context, form dtos.AdminCreateUserForm) { return } + metrics.M_Api_Admin_User_Create.Inc(1) + c.JsonOK("User created") } diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 8cde5a8bc8a..278264f22d7 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -27,6 +28,8 @@ func isDasboardStarredByUser(c *middleware.Context, dashId int64) (bool, error) } func GetDashboard(c *middleware.Context) { + metrics.M_Api_Dashboard_Get.Inc(1) + slug := c.Params(":slug") query := m.GetDashboardQuery{Slug: slug, OrgId: c.OrgId} @@ -88,6 +91,8 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) { return } + metrics.M_Api_Dashboard_Post.Inc(1) + c.JSON(200, util.DynMap{"status": "success", "slug": cmd.Result.Slug, "version": cmd.Result.Version}) } diff --git a/pkg/api/index.go b/pkg/api/index.go index 3006c54e8ab..4af66f18133 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -47,7 +47,7 @@ func Index(c *middleware.Context) { func NotFound(c *middleware.Context) { if c.IsApiRequest() { - c.JsonApiErr(200, "Not found", nil) + c.JsonApiErr(404, "Not found", nil) return } diff --git a/pkg/api/login.go b/pkg/api/login.go index e7707c53138..56a61697cb9 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -75,7 +76,6 @@ func LoginView(c *middleware.Context) { } func LoginPost(c *middleware.Context, cmd dtos.LoginCommand) { - userQuery := m.GetUserByLoginQuery{LoginOrEmail: cmd.User} err := bus.Dispatch(&userQuery) @@ -112,6 +112,8 @@ func LoginPost(c *middleware.Context, cmd dtos.LoginCommand) { c.SetCookie("redirect_to", "", -1, setting.AppSubUrl+"/") } + metrics.M_Api_Login_Post.Inc(1) + c.JSON(200, result) } diff --git a/pkg/api/login_oauth.go b/pkg/api/login_oauth.go index 9ccb8f0b60d..a234ef02bf3 100644 --- a/pkg/api/login_oauth.go +++ b/pkg/api/login_oauth.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -81,5 +82,7 @@ func OAuthLogin(ctx *middleware.Context) { // login loginUserWithUser(userQuery.Result, ctx) + metrics.M_Api_Login_OAuth.Inc(1) + ctx.Redirect(setting.AppSubUrl + "/") } diff --git a/pkg/api/org.go b/pkg/api/org.go index 8b41b0e3f5f..ed180b1af77 100644 --- a/pkg/api/org.go +++ b/pkg/api/org.go @@ -2,6 +2,7 @@ package api import ( "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" ) @@ -35,6 +36,8 @@ func CreateOrg(c *middleware.Context, cmd m.CreateOrgCommand) { return } + metrics.M_Api_Org_Create.Inc(1) + c.JsonOK("Organization created") } diff --git a/pkg/api/signup.go b/pkg/api/signup.go index 74f00509b98..63bb34c72ac 100644 --- a/pkg/api/signup.go +++ b/pkg/api/signup.go @@ -2,6 +2,7 @@ package api import ( "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -26,4 +27,6 @@ func SignUp(c *middleware.Context, cmd m.CreateUserCommand) { loginUserWithUser(&user, c) c.JsonOK("User created and logged in") + + metrics.M_Api_User_SignUp.Inc(1) } diff --git a/pkg/cmd/web.go b/pkg/cmd/web.go index 6619e5b1e0e..e5516fb52d9 100644 --- a/pkg/cmd/web.go +++ b/pkg/cmd/web.go @@ -21,6 +21,7 @@ import ( "github.com/grafana/grafana/pkg/api" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/eventpublisher" @@ -88,6 +89,10 @@ func runWeb(c *cli.Context) { m := newMacaron() api.Register(m) + if setting.ReportingEnabled { + go metrics.StartUsageReportLoop() + } + listenAddr := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort) log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubUrl) switch setting.Protocol { diff --git a/pkg/metrics/counter.go b/pkg/metrics/counter.go new file mode 100644 index 00000000000..1a4a88be37b --- /dev/null +++ b/pkg/metrics/counter.go @@ -0,0 +1,72 @@ +package metrics + +import "sync/atomic" + +// Counters hold an int64 value that can be incremented and decremented. +type Counter interface { + Clear() + Count() int64 + Dec(int64) + Inc(int64) + Snapshot() Counter +} + +// NewCounter constructs a new StandardCounter. +func NewCounter() Counter { + return &StandardCounter{0} +} + +// CounterSnapshot is a read-only copy of another Counter. +type CounterSnapshot int64 + +// Clear panics. +func (CounterSnapshot) Clear() { + panic("Clear called on a CounterSnapshot") +} + +// Count returns the count at the time the snapshot was taken. +func (c CounterSnapshot) Count() int64 { return int64(c) } + +// Dec panics. +func (CounterSnapshot) Dec(int64) { + panic("Dec called on a CounterSnapshot") +} + +// Inc panics. +func (CounterSnapshot) Inc(int64) { + panic("Inc called on a CounterSnapshot") +} + +// Snapshot returns the snapshot. +func (c CounterSnapshot) Snapshot() Counter { return c } + +// StandardCounter is the standard implementation of a Counter and uses the +// sync/atomic package to manage a single int64 value. +type StandardCounter struct { + count int64 +} + +// Clear sets the counter to zero. +func (c *StandardCounter) Clear() { + atomic.StoreInt64(&c.count, 0) +} + +// Count returns the current count. +func (c *StandardCounter) Count() int64 { + return atomic.LoadInt64(&c.count) +} + +// Dec decrements the counter by the given amount. +func (c *StandardCounter) Dec(i int64) { + atomic.AddInt64(&c.count, -i) +} + +// Inc increments the counter by the given amount. +func (c *StandardCounter) Inc(i int64) { + atomic.AddInt64(&c.count, i) +} + +// Snapshot returns a read-only copy of the counter. +func (c *StandardCounter) Snapshot() Counter { + return CounterSnapshot(c.Count()) +} diff --git a/pkg/metrics/metric_ref.go b/pkg/metrics/metric_ref.go new file mode 100644 index 00000000000..f9e5d693d4c --- /dev/null +++ b/pkg/metrics/metric_ref.go @@ -0,0 +1,39 @@ +package metrics + +type comboCounterRef struct { + usageCounter Counter + metricCounter Counter +} + +func NewComboCounterRef(name string) Counter { + cr := &comboCounterRef{} + cr.usageCounter = UsageStats.GetOrRegister(name, NewCounter).(Counter) + cr.metricCounter = MetricStats.GetOrRegister(name, NewCounter).(Counter) + return cr +} + +func (c comboCounterRef) Clear() { + c.usageCounter.Clear() + c.metricCounter.Clear() +} + +func (c comboCounterRef) Count() int64 { + panic("Count called on a combocounter ref") +} + +// Dec panics. +func (c comboCounterRef) Dec(i int64) { + c.usageCounter.Dec(i) + c.metricCounter.Dec(i) +} + +// Inc panics. +func (c comboCounterRef) Inc(i int64) { + c.usageCounter.Inc(i) + c.metricCounter.Inc(i) +} + +// Snapshot returns the snapshot. +func (c comboCounterRef) Snapshot() Counter { + panic("snapshot called on a combocounter ref") +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 00000000000..45b964fb56e --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,25 @@ +package metrics + +var UsageStats = NewRegistry() +var MetricStats = NewRegistry() + +var ( + M_Instance_Start = NewComboCounterRef("instance.start") + + M_Page_Status_200 = NewComboCounterRef("page.status.200") + M_Page_Status_500 = NewComboCounterRef("page.status.500") + M_Page_Status_404 = NewComboCounterRef("page.status.404") + + M_Api_Status_500 = NewComboCounterRef("api.status.500") + M_Api_Status_404 = NewComboCounterRef("api.status.404") + + M_Api_User_SignUp = NewComboCounterRef("api.user.signup") + M_Api_Dashboard_Get = NewComboCounterRef("api.dashboard.get") + M_Api_Dashboard_Post = NewComboCounterRef("api.dashboard.post") + M_Api_Admin_User_Create = NewComboCounterRef("api.admin.user_create") + M_Api_Login_Post = NewComboCounterRef("api.login.post") + M_Api_Login_OAuth = NewComboCounterRef("api.login.oauth") + M_Api_Org_Create = NewComboCounterRef("api.org.create") + + M_Models_Dashboard_Insert = NewComboCounterRef("models.dashboard.insert") +) diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go new file mode 100644 index 00000000000..9e1618f3691 --- /dev/null +++ b/pkg/metrics/registry.go @@ -0,0 +1,102 @@ +package metrics + +import ( + "fmt" + "reflect" + "sync" +) + +// DuplicateMetric is the error returned by Registry.Register when a metric +// already exists. If you mean to Register that metric you must first +// Unregister the existing metric. +type DuplicateMetric string + +func (err DuplicateMetric) Error() string { + return fmt.Sprintf("duplicate metric: %s", string(err)) +} + +type Registry interface { + // Call the given function for each registered metric. + Each(func(string, interface{})) + + // Get the metric by the given name or nil if none is registered. + Get(string) interface{} + + // Gets an existing metric or registers the given one. + // The interface can be the metric to register if not found in registry, + // or a function returning the metric for lazy instantiation. + GetOrRegister(string, interface{}) interface{} + + // Register the given metric under the given name. + Register(string, interface{}) error +} + +// The standard implementation of a Registry is a mutex-protected map +// of names to metrics. +type StandardRegistry struct { + metrics map[string]interface{} + mutex sync.Mutex +} + +// Create a new registry. +func NewRegistry() Registry { + return &StandardRegistry{metrics: make(map[string]interface{})} +} + +// Call the given function for each registered metric. +func (r *StandardRegistry) Each(f func(string, interface{})) { + for name, i := range r.registered() { + f(name, i) + } +} + +// Get the metric by the given name or nil if none is registered. +func (r *StandardRegistry) Get(name string) interface{} { + r.mutex.Lock() + defer r.mutex.Unlock() + return r.metrics[name] +} + +// Gets an existing metric or creates and registers a new one. Threadsafe +// alternative to calling Get and Register on failure. +// The interface can be the metric to register if not found in registry, +// or a function returning the metric for lazy instantiation. +func (r *StandardRegistry) GetOrRegister(name string, i interface{}) interface{} { + r.mutex.Lock() + defer r.mutex.Unlock() + if metric, ok := r.metrics[name]; ok { + return metric + } + if v := reflect.ValueOf(i); v.Kind() == reflect.Func { + i = v.Call(nil)[0].Interface() + } + r.register(name, i) + return i +} + +// Register the given metric under the given name. Returns a DuplicateMetric +// if a metric by the given name is already registered. +func (r *StandardRegistry) Register(name string, i interface{}) error { + r.mutex.Lock() + defer r.mutex.Unlock() + return r.register(name, i) +} + +func (r *StandardRegistry) register(name string, i interface{}) error { + if _, ok := r.metrics[name]; ok { + return DuplicateMetric(name) + } + + r.metrics[name] = i + return nil +} + +func (r *StandardRegistry) registered() map[string]interface{} { + metrics := make(map[string]interface{}, len(r.metrics)) + r.mutex.Lock() + defer r.mutex.Unlock() + for name, i := range r.metrics { + metrics[name] = i + } + return metrics +} diff --git a/pkg/metrics/report_usage.go b/pkg/metrics/report_usage.go new file mode 100644 index 00000000000..af6552b5c1e --- /dev/null +++ b/pkg/metrics/report_usage.go @@ -0,0 +1,60 @@ +package metrics + +import ( + "bytes" + "encoding/json" + "net/http" + "time" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/setting" +) + +func StartUsageReportLoop() chan struct{} { + M_Instance_Start.Inc(1) + + ticker := time.NewTicker(10 * time.Minute) + for { + select { + case <-ticker.C: + sendUsageStats() + } + } +} + +func sendUsageStats() { + log.Trace("Sending anonymous usage stats to stats.grafana.org") + + metrics := map[string]interface{}{} + report := map[string]interface{}{ + "version": setting.BuildVersion, + "metrics": metrics, + } + + // statsQuery := m.GetSystemStatsQuery{} + // if err := bus.Dispatch(&statsQuery); err != nil { + // log.Error(3, "Failed to get system stats", err) + // return + // } + + UsageStats.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case Counter: + if metric.Count() > 0 { + metrics[name+".count"] = metric.Count() + metric.Clear() + } + } + }) + + // metrics["stats.dashboards.count"] = statsQuery.Result.DashboardCount + // metrics["stats.users.count"] = statsQuery.Result.UserCount + // metrics["stats.orgs.count"] = statsQuery.Result.OrgCount + + out, _ := json.Marshal(report) + data := bytes.NewBuffer(out) + + client := http.Client{Timeout: time.Duration(5 * time.Second)} + + go client.Post("http://stats.grafana.org/grafana-usage-report", "application/json", data) +} diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index a15fd075fca..20e1eb196e5 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/apikeygen" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" ) @@ -99,6 +100,15 @@ func (ctx *Context) Handle(status int, title string, err error) { } } + switch status { + case 200: + metrics.M_Page_Status_200.Inc(1) + case 404: + metrics.M_Page_Status_404.Inc(1) + case 500: + metrics.M_Page_Status_500.Inc(1) + } + ctx.Data["Title"] = title ctx.HTML(status, strconv.Itoa(status)) } @@ -128,7 +138,9 @@ func (ctx *Context) JsonApiErr(status int, message string, err error) { switch status { case 404: resp["message"] = "Not Found" + metrics.M_Api_Status_500.Inc(1) case 500: + metrics.M_Api_Status_404.Inc(1) resp["message"] = "Internal Server Error" } diff --git a/pkg/models/stats.go b/pkg/models/stats.go new file mode 100644 index 00000000000..0d83882e666 --- /dev/null +++ b/pkg/models/stats.go @@ -0,0 +1,11 @@ +package models + +type SystemStats struct { + DashboardCount int + UserCount int + OrgCount int +} + +type GetSystemStatsQuery struct { + Result *SystemStats +} diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index bf748b600f4..0384a5bb6e6 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -6,6 +6,7 @@ import ( "github.com/go-xorm/xorm" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/metrics" m "github.com/grafana/grafana/pkg/models" ) @@ -48,6 +49,7 @@ func SaveDashboard(cmd *m.SaveDashboardCommand) error { } if dash.Id == 0 { + metrics.M_Models_Dashboard_Insert.Inc(1) _, err = sess.Insert(dash) } else { dash.Version += 1 diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go new file mode 100644 index 00000000000..7995dd43f38 --- /dev/null +++ b/pkg/services/sqlstore/stats.go @@ -0,0 +1,36 @@ +package sqlstore + +import ( + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" +) + +func init() { + bus.AddHandler("sql", GetSystemStats) +} + +func GetSystemStats(query *m.GetSystemStatsQuery) error { + var rawSql = `SELECT + ( + SELECT COUNT(*) + FROM ` + dialect.Quote("user") + ` + ) AS user_count, + ( + SELECT COUNT(*) + FROM ` + dialect.Quote("org") + ` + ) AS org_count, + ( + SELECT COUNT(*) + FROM ` + dialect.Quote("dashboard") + ` + ) AS dashboard_count + ` + + var stats m.SystemStats + _, err := x.Sql(rawSql).Get(&stats) + if err != nil { + return err + } + + query.Result = &stats + return err +} diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index b8d038dbc29..defa8311e8c 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -96,6 +96,8 @@ var ( PhantomDir string configFiles []string + + ReportingEnabled bool ) func init() { @@ -233,6 +235,8 @@ func NewConfigContext(config string) { ImagesDir = "data/png" PhantomDir = "vendor/phantomjs" + ReportingEnabled = Cfg.Section("").Key("reporting-enabled").MustBool(true) + readSessionConfig() }