public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #12071 from grafana/12056_usage_stats

Browse Source 
Additional anonymous usage stats
This commit is contained in:
Carl Bergquist
2018-05-28 11:23:16 +02:00
committed by GitHub
parent a2ed0b15da c9e9f25699
commit 519e58a267
6 changed files with 390 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/metrics/metrics.go
+38 -1
View File
@@ -332,6 +332,8 @@ func updateTotalStats() {
M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs))
}
var usageStatsURL = "https://stats.grafana.org/grafana-usage-report"
func sendUsageStats() {
if !setting.ReportingEnabled {
return
@@ -366,6 +368,12 @@ func sendUsageStats() {
metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
metrics["stats.datasources.count"] = statsQuery.Result.Datasources
metrics["stats.stars.count"] = statsQuery.Result.Stars
metrics["stats.folders.count"] = statsQuery.Result.Folders
metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions
metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions
metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards
metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots
metrics["stats.teams.count"] = statsQuery.Result.Teams
dsStats := models.GetDataSourceStatsQuery{}
if err := bus.Dispatch(&dsStats); err != nil {
@@ -386,9 +394,38 @@ func sendUsageStats() {
}
metrics["stats.ds.other.count"] = dsOtherCount
dsAccessStats := models.GetDataSourceAccessStatsQuery{}
if err := bus.Dispatch(&dsAccessStats); err != nil {
metricsLogger.Error("Failed to get datasource access stats", "error", err)
return
}
// send access counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsAccessOtherCount := make(map[string]int64)
for _, dsAccessStat := range dsAccessStats.Result {
if dsAccessStat.Access == "" {
continue
}
access := strings.ToLower(dsAccessStat.Access)
if models.IsKnownDataSourcePlugin(dsAccessStat.Type) {
metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count
} else {
old := dsAccessOtherCount[access]
dsAccessOtherCount[access] = old + dsAccessStat.Count
}
}
for access, count := range dsAccessOtherCount {
metrics["stats.ds_access.other."+access+".count"] = count
}
out, _ := json.MarshalIndent(report, "", " ")
data := bytes.NewBuffer(out)
client := http.Client{Timeout: 5 * time.Second}
go client.Post("https://stats.grafana.org/grafana-usage-report", "application/json", data)
go client.Post(usageStatsURL, "application/json", data)
}
pkg/metrics/metrics_test.go
+222
View File
@@ -0,0 +1,222 @@
package metrics
import (
"bytes"
"io/ioutil"
"runtime"
"sync"
"testing"
"time"
"net/http"
"net/http/httptest"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
. "github.com/smartystreets/goconvey/convey"
)
func TestMetrics(t *testing.T) {
Convey("Test send usage stats", t, func() {
var getSystemStatsQuery *models.GetSystemStatsQuery
bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error {
query.Result = &models.SystemStats{
Dashboards: 1,
Datasources: 2,
Users: 3,
ActiveUsers: 4,
Orgs: 5,
Playlists: 6,
Alerts: 7,
Stars: 8,
Folders: 9,
DashboardPermissions: 10,
FolderPermissions: 11,
ProvisionedDashboards: 12,
Snapshots: 13,
Teams: 14,
}
getSystemStatsQuery = query
return nil
})
var getDataSourceStatsQuery *models.GetDataSourceStatsQuery
bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error {
query.Result = []*models.DataSourceStats{
{
Type: models.DS_ES,
Count: 9,
},
{
Type: models.DS_PROMETHEUS,
Count: 10,
},
{
Type: "unknown_ds",
Count: 11,
},
{
Type: "unknown_ds2",
Count: 12,
},
}
getDataSourceStatsQuery = query
return nil
})
var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery
bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error {
query.Result = []*models.DataSourceAccessStats{
{
Type: models.DS_ES,
Access: "direct",
Count: 1,
},
{
Type: models.DS_ES,
Access: "proxy",
Count: 2,
},
{
Type: models.DS_PROMETHEUS,
Access: "proxy",
Count: 3,
},
{
Type: "unknown_ds",
Access: "proxy",
Count: 4,
},
{
Type: "unknown_ds2",
Access: "",
Count: 5,
},
{
Type: "unknown_ds3",
Access: "direct",
Count: 6,
},
{
Type: "unknown_ds4",
Access: "direct",
Count: 7,
},
{
Type: "unknown_ds5",
Access: "proxy",
Count: 8,
},
}
getDataSourceAccessStatsQuery = query
return nil
})
var wg sync.WaitGroup
var responseBuffer *bytes.Buffer
var req *http.Request
ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
req = r
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
t.Fatalf("Failed to read response body, err=%v", err)
}
responseBuffer = bytes.NewBuffer(buf)
wg.Done()
}))
usageStatsURL = ts.URL
sendUsageStats()
Convey("Given reporting not enabled and sending usage stats", func() {
setting.ReportingEnabled = false
sendUsageStats()
Convey("Should not gather stats or call http endpoint", func() {
So(getSystemStatsQuery, ShouldBeNil)
So(getDataSourceStatsQuery, ShouldBeNil)
So(getDataSourceAccessStatsQuery, ShouldBeNil)
So(req, ShouldBeNil)
})
})
Convey("Given reporting enabled and sending usage stats", func() {
setting.ReportingEnabled = true
setting.BuildVersion = "5.0.0"
wg.Add(1)
sendUsageStats()
Convey("Should gather stats and call http endpoint", func() {
if waitTimeout(&wg, 2*time.Second) {
t.Fatalf("Timed out waiting for http request")
}
So(getSystemStatsQuery, ShouldNotBeNil)
So(getDataSourceStatsQuery, ShouldNotBeNil)
So(getDataSourceAccessStatsQuery, ShouldNotBeNil)
So(req, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost)
So(req.Header.Get("Content-Type"), ShouldEqual, "application/json")
So(responseBuffer, ShouldNotBeNil)
j, err := simplejson.NewFromReader(responseBuffer)
So(err, ShouldBeNil)
So(j.Get("version").MustString(), ShouldEqual, "5_0_0")
So(j.Get("os").MustString(), ShouldEqual, runtime.GOOS)
So(j.Get("arch").MustString(), ShouldEqual, runtime.GOARCH)
metrics := j.Get("metrics")
So(metrics.Get("stats.dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Dashboards)
So(metrics.Get("stats.users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Users)
So(metrics.Get("stats.orgs.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Orgs)
So(metrics.Get("stats.playlist.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Playlists)
So(metrics.Get("stats.plugins.apps.count").MustInt(), ShouldEqual, len(plugins.Apps))
So(metrics.Get("stats.plugins.panels.count").MustInt(), ShouldEqual, len(plugins.Panels))
So(metrics.Get("stats.plugins.datasources.count").MustInt(), ShouldEqual, len(plugins.DataSources))
So(metrics.Get("stats.alerts.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Alerts)
So(metrics.Get("stats.active_users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ActiveUsers)
So(metrics.Get("stats.datasources.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Datasources)
So(metrics.Get("stats.stars.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Stars)
So(metrics.Get("stats.folders.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Folders)
So(metrics.Get("stats.dashboard_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.DashboardPermissions)
So(metrics.Get("stats.folder_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.FolderPermissions)
So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards)
So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots)
So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams)
So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9)
So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10)
So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12)
So(metrics.Get("stats.ds_access."+models.DS_ES+".direct.count").MustInt(), ShouldEqual, 1)
So(metrics.Get("stats.ds_access."+models.DS_ES+".proxy.count").MustInt(), ShouldEqual, 2)
So(metrics.Get("stats.ds_access."+models.DS_PROMETHEUS+".proxy.count").MustInt(), ShouldEqual, 3)
So(metrics.Get("stats.ds_access.other.direct.count").MustInt(), ShouldEqual, 6+7)
So(metrics.Get("stats.ds_access.other.proxy.count").MustInt(), ShouldEqual, 4+8)
})
})
Reset(func() {
ts.Close()
})
})
}
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
c := make(chan struct{})
go func() {
defer close(c)
wg.Wait()
}()
select {
case <-c:
return false // completed normally
case <-time.After(timeout):
return true // timed out
}
}
pkg/models/stats.go
+32 -8
View File
@@ -1,14 +1,20 @@
package models
type SystemStats struct {
Dashboards int64
Datasources int64
Users int64
ActiveUsers int64
Orgs int64
Playlists int64
Alerts int64
Stars int64
Dashboards int64
Datasources int64
Users int64
ActiveUsers int64
Orgs int64
Playlists int64
Alerts int64
Stars int64
Snapshots int64
Teams int64
DashboardPermissions int64
FolderPermissions int64
Folders int64
ProvisionedDashboards int64
}
type DataSourceStats struct {
@@ -24,6 +30,16 @@ type GetDataSourceStatsQuery struct {
Result []*DataSourceStats
}
type DataSourceAccessStats struct {
Type string
Access string
Count int64
}
type GetDataSourceAccessStatsQuery struct {
Result []*DataSourceAccessStats
}
type AdminStats struct {
Users int `json:"users"`
Orgs int `json:"orgs"`
@@ -40,3 +56,11 @@ type AdminStats struct {
type GetAdminStatsQuery struct {
Result *AdminStats
}
type SystemUserCountStats struct {
Count int64
}
type GetSystemUserCountStatsQuery struct {
Result *SystemUserCountStats
}
pkg/services/sqlstore/sqlstore.go
+3 -3
View File
@@ -86,13 +86,13 @@ func (ss *SqlStore) Init() error {
}
func (ss *SqlStore) ensureAdminUser() error {
statsQuery := m.GetSystemStatsQuery{}
systemUserCountQuery := m.GetSystemUserCountStatsQuery{}
if err := bus.Dispatch(&statsQuery); err != nil {
if err := bus.Dispatch(&systemUserCountQuery); err != nil {
fmt.Errorf("Could not determine if admin user exists: %v", err)
}
if statsQuery.Result.Users > 0 {
if systemUserCountQuery.Result.Count > 0 {
return nil
}
pkg/services/sqlstore/stats.go
+56 -33
View File
@@ -10,7 +10,9 @@ import (
func init() {
bus.AddHandler("sql", GetSystemStats)
bus.AddHandler("sql", GetDataSourceStats)
bus.AddHandler("sql", GetDataSourceAccessStats)
bus.AddHandler("sql", GetAdminStats)
bus.AddHandler("sql", GetSystemUserCountStats)
}
var activeUserTimeLimit = time.Hour * 24 * 30
@@ -22,43 +24,51 @@ func GetDataSourceStats(query *m.GetDataSourceStatsQuery) error {
return err
}
func GetDataSourceAccessStats(query *m.GetDataSourceAccessStatsQuery) error {
var rawSql = `SELECT COUNT(*) as count, type, access FROM data_source GROUP BY type, access`
query.Result = make([]*m.DataSourceAccessStats, 0)
err := x.SQL(rawSql).Find(&query.Result)
return err
}
func GetSystemStats(query *m.GetSystemStatsQuery) error {
var rawSql = `SELECT
(
SELECT COUNT(*)
FROM ` + dialect.Quote("user") + `
) AS users,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("org") + `
) AS orgs,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("dashboard") + `
) AS dashboards,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("data_source") + `
) AS datasources,
(
SELECT COUNT(*) FROM ` + dialect.Quote("star") + `
) AS stars,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("playlist") + `
) AS playlists,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("alert") + `
) AS alerts,
(
SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` where last_seen_at > ?
) as active_users
`
sb := &SqlBuilder{}
sb.Write("SELECT ")
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + `) AS users,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("dashboard") + `) AS dashboards,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("alert") + `) AS alerts,`)
activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit)
sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` where last_seen_at > ?) AS active_users,`, activeUserDeadlineDate)
sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` where is_folder = ?) AS folders,`, dialect.BooleanStr(true))
sb.Write(`(
SELECT COUNT(acl.id)
FROM `+dialect.Quote("dashboard_acl")+` as acl
inner join `+dialect.Quote("dashboard")+` as d
on d.id = acl.dashboard_id
WHERE d.is_folder = ?
) AS dashboard_permissions,`, dialect.BooleanStr(false))
sb.Write(`(
SELECT COUNT(acl.id)
FROM `+dialect.Quote("dashboard_acl")+` as acl
inner join `+dialect.Quote("dashboard")+` as d
on d.id = acl.dashboard_id
WHERE d.is_folder = ?
) AS folder_permissions,`, dialect.BooleanStr(true))
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams`)
var stats m.SystemStats
_, err := x.SQL(rawSql, activeUserDeadlineDate).Get(&stats)
_, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats)
if err != nil {
return err
}
@@ -122,3 +132,16 @@ func GetAdminStats(query *m.GetAdminStatsQuery) error {
query.Result = &stats
return err
}
func GetSystemUserCountStats(query *m.GetSystemUserCountStatsQuery) error {
var rawSql = `SELECT COUNT(id) AS Count FROM ` + dialect.Quote("user")
var stats m.SystemUserCountStats
_, err := x.SQL(rawSql).Get(&stats)
if err != nil {
return err
}
query.Result = &stats
return err
}
pkg/services/sqlstore/stats_test.go
+39
View File
@@ -0,0 +1,39 @@
package sqlstore
import (
"testing"
m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestStatsDataAccess(t *testing.T) {
Convey("Testing Stats Data Access", t, func() {
InitTestDB(t)
Convey("Get system stats should not results in error", func() {
query := m.GetSystemStatsQuery{}
err := GetSystemStats(&query)
So(err, ShouldBeNil)
})
Convey("Get system user count stats should not results in error", func() {
query := m.GetSystemUserCountStatsQuery{}
err := GetSystemUserCountStats(&query)
So(err, ShouldBeNil)
})
Convey("Get datasource stats should not results in error", func() {
query := m.GetDataSourceStatsQuery{}
err := GetDataSourceStats(&query)
So(err, ShouldBeNil)
})
Convey("Get datasource access stats should not results in error", func() {
query := m.GetDataSourceAccessStatsQuery{}
err := GetDataSourceAccessStats(&query)
So(err, ShouldBeNil)
})
})
}