diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 83505826910..3d3cfc2e1b6 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -332,6 +332,8 @@ func updateTotalStats() { M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs)) } +var usageStatsURL = "https://stats.grafana.org/grafana-usage-report" + func sendUsageStats() { if !setting.ReportingEnabled { return @@ -366,6 +368,12 @@ func sendUsageStats() { metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers metrics["stats.datasources.count"] = statsQuery.Result.Datasources metrics["stats.stars.count"] = statsQuery.Result.Stars + metrics["stats.folders.count"] = statsQuery.Result.Folders + metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions + metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions + metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards + metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots + metrics["stats.teams.count"] = statsQuery.Result.Teams dsStats := models.GetDataSourceStatsQuery{} if err := bus.Dispatch(&dsStats); err != nil { @@ -386,9 +394,38 @@ func sendUsageStats() { } metrics["stats.ds.other.count"] = dsOtherCount + dsAccessStats := models.GetDataSourceAccessStatsQuery{} + if err := bus.Dispatch(&dsAccessStats); err != nil { + metricsLogger.Error("Failed to get datasource access stats", "error", err) + return + } + + // send access counters for each data source + // but ignore any custom data sources + // as sending that name could be sensitive information + dsAccessOtherCount := make(map[string]int64) + for _, dsAccessStat := range dsAccessStats.Result { + if dsAccessStat.Access == "" { + continue + } + + access := strings.ToLower(dsAccessStat.Access) + + if models.IsKnownDataSourcePlugin(dsAccessStat.Type) { + metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count + } else { + old := dsAccessOtherCount[access] + dsAccessOtherCount[access] = old + dsAccessStat.Count + } + } + + for access, count := range dsAccessOtherCount { + metrics["stats.ds_access.other."+access+".count"] = count + } + out, _ := json.MarshalIndent(report, "", " ") data := bytes.NewBuffer(out) client := http.Client{Timeout: 5 * time.Second} - go client.Post("https://stats.grafana.org/grafana-usage-report", "application/json", data) + go client.Post(usageStatsURL, "application/json", data) } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 00000000000..8d88e03d106 --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,222 @@ +package metrics + +import ( + "bytes" + "io/ioutil" + "runtime" + "sync" + "testing" + "time" + + "net/http" + "net/http/httptest" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" +) + +func TestMetrics(t *testing.T) { + Convey("Test send usage stats", t, func() { + var getSystemStatsQuery *models.GetSystemStatsQuery + bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error { + query.Result = &models.SystemStats{ + Dashboards: 1, + Datasources: 2, + Users: 3, + ActiveUsers: 4, + Orgs: 5, + Playlists: 6, + Alerts: 7, + Stars: 8, + Folders: 9, + DashboardPermissions: 10, + FolderPermissions: 11, + ProvisionedDashboards: 12, + Snapshots: 13, + Teams: 14, + } + getSystemStatsQuery = query + return nil + }) + + var getDataSourceStatsQuery *models.GetDataSourceStatsQuery + bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error { + query.Result = []*models.DataSourceStats{ + { + Type: models.DS_ES, + Count: 9, + }, + { + Type: models.DS_PROMETHEUS, + Count: 10, + }, + { + Type: "unknown_ds", + Count: 11, + }, + { + Type: "unknown_ds2", + Count: 12, + }, + } + getDataSourceStatsQuery = query + return nil + }) + + var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery + bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error { + query.Result = []*models.DataSourceAccessStats{ + { + Type: models.DS_ES, + Access: "direct", + Count: 1, + }, + { + Type: models.DS_ES, + Access: "proxy", + Count: 2, + }, + { + Type: models.DS_PROMETHEUS, + Access: "proxy", + Count: 3, + }, + { + Type: "unknown_ds", + Access: "proxy", + Count: 4, + }, + { + Type: "unknown_ds2", + Access: "", + Count: 5, + }, + { + Type: "unknown_ds3", + Access: "direct", + Count: 6, + }, + { + Type: "unknown_ds4", + Access: "direct", + Count: 7, + }, + { + Type: "unknown_ds5", + Access: "proxy", + Count: 8, + }, + } + getDataSourceAccessStatsQuery = query + return nil + }) + + var wg sync.WaitGroup + var responseBuffer *bytes.Buffer + var req *http.Request + ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + req = r + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("Failed to read response body, err=%v", err) + } + responseBuffer = bytes.NewBuffer(buf) + wg.Done() + })) + usageStatsURL = ts.URL + + sendUsageStats() + + Convey("Given reporting not enabled and sending usage stats", func() { + setting.ReportingEnabled = false + sendUsageStats() + + Convey("Should not gather stats or call http endpoint", func() { + So(getSystemStatsQuery, ShouldBeNil) + So(getDataSourceStatsQuery, ShouldBeNil) + So(getDataSourceAccessStatsQuery, ShouldBeNil) + So(req, ShouldBeNil) + }) + }) + + Convey("Given reporting enabled and sending usage stats", func() { + setting.ReportingEnabled = true + setting.BuildVersion = "5.0.0" + wg.Add(1) + sendUsageStats() + + Convey("Should gather stats and call http endpoint", func() { + if waitTimeout(&wg, 2*time.Second) { + t.Fatalf("Timed out waiting for http request") + } + + So(getSystemStatsQuery, ShouldNotBeNil) + So(getDataSourceStatsQuery, ShouldNotBeNil) + So(getDataSourceAccessStatsQuery, ShouldNotBeNil) + So(req, ShouldNotBeNil) + So(req.Method, ShouldEqual, http.MethodPost) + So(req.Header.Get("Content-Type"), ShouldEqual, "application/json") + + So(responseBuffer, ShouldNotBeNil) + + j, err := simplejson.NewFromReader(responseBuffer) + So(err, ShouldBeNil) + + So(j.Get("version").MustString(), ShouldEqual, "5_0_0") + So(j.Get("os").MustString(), ShouldEqual, runtime.GOOS) + So(j.Get("arch").MustString(), ShouldEqual, runtime.GOARCH) + + metrics := j.Get("metrics") + So(metrics.Get("stats.dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Dashboards) + So(metrics.Get("stats.users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Users) + So(metrics.Get("stats.orgs.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Orgs) + So(metrics.Get("stats.playlist.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Playlists) + So(metrics.Get("stats.plugins.apps.count").MustInt(), ShouldEqual, len(plugins.Apps)) + So(metrics.Get("stats.plugins.panels.count").MustInt(), ShouldEqual, len(plugins.Panels)) + So(metrics.Get("stats.plugins.datasources.count").MustInt(), ShouldEqual, len(plugins.DataSources)) + So(metrics.Get("stats.alerts.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Alerts) + So(metrics.Get("stats.active_users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ActiveUsers) + So(metrics.Get("stats.datasources.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Datasources) + So(metrics.Get("stats.stars.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Stars) + So(metrics.Get("stats.folders.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Folders) + So(metrics.Get("stats.dashboard_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.DashboardPermissions) + So(metrics.Get("stats.folder_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.FolderPermissions) + So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards) + So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots) + So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams) + + So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9) + So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10) + So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12) + + So(metrics.Get("stats.ds_access."+models.DS_ES+".direct.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.ds_access."+models.DS_ES+".proxy.count").MustInt(), ShouldEqual, 2) + So(metrics.Get("stats.ds_access."+models.DS_PROMETHEUS+".proxy.count").MustInt(), ShouldEqual, 3) + So(metrics.Get("stats.ds_access.other.direct.count").MustInt(), ShouldEqual, 6+7) + So(metrics.Get("stats.ds_access.other.proxy.count").MustInt(), ShouldEqual, 4+8) + }) + }) + + Reset(func() { + ts.Close() + }) + }) +} + +func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() + }() + select { + case <-c: + return false // completed normally + case <-time.After(timeout): + return true // timed out + } +} diff --git a/pkg/models/stats.go b/pkg/models/stats.go index e132d88c030..4cd50d37463 100644 --- a/pkg/models/stats.go +++ b/pkg/models/stats.go @@ -1,14 +1,20 @@ package models type SystemStats struct { - Dashboards int64 - Datasources int64 - Users int64 - ActiveUsers int64 - Orgs int64 - Playlists int64 - Alerts int64 - Stars int64 + Dashboards int64 + Datasources int64 + Users int64 + ActiveUsers int64 + Orgs int64 + Playlists int64 + Alerts int64 + Stars int64 + Snapshots int64 + Teams int64 + DashboardPermissions int64 + FolderPermissions int64 + Folders int64 + ProvisionedDashboards int64 } type DataSourceStats struct { @@ -24,6 +30,16 @@ type GetDataSourceStatsQuery struct { Result []*DataSourceStats } +type DataSourceAccessStats struct { + Type string + Access string + Count int64 +} + +type GetDataSourceAccessStatsQuery struct { + Result []*DataSourceAccessStats +} + type AdminStats struct { Users int `json:"users"` Orgs int `json:"orgs"` @@ -40,3 +56,11 @@ type AdminStats struct { type GetAdminStatsQuery struct { Result *AdminStats } + +type SystemUserCountStats struct { + Count int64 +} + +type GetSystemUserCountStatsQuery struct { + Result *SystemUserCountStats +} diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 39aa2cb7ead..6af56f2f169 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -86,13 +86,13 @@ func (ss *SqlStore) Init() error { } func (ss *SqlStore) ensureAdminUser() error { - statsQuery := m.GetSystemStatsQuery{} + systemUserCountQuery := m.GetSystemUserCountStatsQuery{} - if err := bus.Dispatch(&statsQuery); err != nil { + if err := bus.Dispatch(&systemUserCountQuery); err != nil { fmt.Errorf("Could not determine if admin user exists: %v", err) } - if statsQuery.Result.Users > 0 { + if systemUserCountQuery.Result.Count > 0 { return nil } diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go index 173a1e56634..3e3e83c4014 100644 --- a/pkg/services/sqlstore/stats.go +++ b/pkg/services/sqlstore/stats.go @@ -10,7 +10,9 @@ import ( func init() { bus.AddHandler("sql", GetSystemStats) bus.AddHandler("sql", GetDataSourceStats) + bus.AddHandler("sql", GetDataSourceAccessStats) bus.AddHandler("sql", GetAdminStats) + bus.AddHandler("sql", GetSystemUserCountStats) } var activeUserTimeLimit = time.Hour * 24 * 30 @@ -22,43 +24,51 @@ func GetDataSourceStats(query *m.GetDataSourceStatsQuery) error { return err } +func GetDataSourceAccessStats(query *m.GetDataSourceAccessStatsQuery) error { + var rawSql = `SELECT COUNT(*) as count, type, access FROM data_source GROUP BY type, access` + query.Result = make([]*m.DataSourceAccessStats, 0) + err := x.SQL(rawSql).Find(&query.Result) + return err +} + func GetSystemStats(query *m.GetSystemStatsQuery) error { - var rawSql = `SELECT - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("user") + ` - ) AS users, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("org") + ` - ) AS orgs, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("dashboard") + ` - ) AS dashboards, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("data_source") + ` - ) AS datasources, - ( - SELECT COUNT(*) FROM ` + dialect.Quote("star") + ` - ) AS stars, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("playlist") + ` - ) AS playlists, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("alert") + ` - ) AS alerts, - ( - SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` where last_seen_at > ? - ) as active_users - ` + sb := &SqlBuilder{} + sb.Write("SELECT ") + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + `) AS users,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("dashboard") + `) AS dashboards,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("alert") + `) AS alerts,`) activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit) + sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` where last_seen_at > ?) AS active_users,`, activeUserDeadlineDate) + + sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` where is_folder = ?) AS folders,`, dialect.BooleanStr(true)) + + sb.Write(`( + SELECT COUNT(acl.id) + FROM `+dialect.Quote("dashboard_acl")+` as acl + inner join `+dialect.Quote("dashboard")+` as d + on d.id = acl.dashboard_id + WHERE d.is_folder = ? + ) AS dashboard_permissions,`, dialect.BooleanStr(false)) + + sb.Write(`( + SELECT COUNT(acl.id) + FROM `+dialect.Quote("dashboard_acl")+` as acl + inner join `+dialect.Quote("dashboard")+` as d + on d.id = acl.dashboard_id + WHERE d.is_folder = ? + ) AS folder_permissions,`, dialect.BooleanStr(true)) + + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams`) + var stats m.SystemStats - _, err := x.SQL(rawSql, activeUserDeadlineDate).Get(&stats) + _, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats) if err != nil { return err } @@ -122,3 +132,16 @@ func GetAdminStats(query *m.GetAdminStatsQuery) error { query.Result = &stats return err } + +func GetSystemUserCountStats(query *m.GetSystemUserCountStatsQuery) error { + var rawSql = `SELECT COUNT(id) AS Count FROM ` + dialect.Quote("user") + var stats m.SystemUserCountStats + _, err := x.SQL(rawSql).Get(&stats) + if err != nil { + return err + } + + query.Result = &stats + + return err +} diff --git a/pkg/services/sqlstore/stats_test.go b/pkg/services/sqlstore/stats_test.go new file mode 100644 index 00000000000..97f0ca0c43e --- /dev/null +++ b/pkg/services/sqlstore/stats_test.go @@ -0,0 +1,39 @@ +package sqlstore + +import ( + "testing" + + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestStatsDataAccess(t *testing.T) { + + Convey("Testing Stats Data Access", t, func() { + InitTestDB(t) + + Convey("Get system stats should not results in error", func() { + query := m.GetSystemStatsQuery{} + err := GetSystemStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get system user count stats should not results in error", func() { + query := m.GetSystemUserCountStatsQuery{} + err := GetSystemUserCountStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get datasource stats should not results in error", func() { + query := m.GetDataSourceStatsQuery{} + err := GetDataSourceStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get datasource access stats should not results in error", func() { + query := m.GetDataSourceAccessStatsQuery{} + err := GetDataSourceAccessStats(&query) + So(err, ShouldBeNil) + }) + }) +}