moves usage stats sender to new package
This commit is contained in:
bergquist
@@ -0,0 +1,54 @@
|
||||
package usagestats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/services/auth"
|
||||
"github.com/grafana/grafana/pkg/social"
|
||||
|
||||
"github.com/grafana/grafana/pkg/log"
|
||||
"github.com/grafana/grafana/pkg/registry"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
var metricsLogger log.Logger = log.New("metrics")
|
||||
|
||||
func init() {
|
||||
registry.RegisterService(&UsageStatsService{})
|
||||
}
|
||||
|
||||
type UsageStatsService struct {
|
||||
Cfg *setting.Cfg `inject:""`
|
||||
TokenService *auth.UserAuthTokenService `inject:""`
|
||||
Bus bus.Bus `inject:""`
|
||||
|
||||
oauthProviders map[string]bool
|
||||
}
|
||||
|
||||
func (uss *UsageStatsService) Init() error {
|
||||
|
||||
uss.oauthProviders = social.GetOAuthProviders(uss.Cfg)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (uss *UsageStatsService) Run(ctx context.Context) error {
|
||||
uss.updateTotalStats()
|
||||
|
||||
onceEveryDayTick := time.NewTicker(time.Hour * 24)
|
||||
everyMinuteTicker := time.NewTicker(time.Minute)
|
||||
defer onceEveryDayTick.Stop()
|
||||
defer everyMinuteTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-onceEveryDayTick.C:
|
||||
uss.sendUsageStats(uss.oauthProviders)
|
||||
case <-everyMinuteTicker.C:
|
||||
uss.updateTotalStats()
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package usagestats
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/metrics"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/plugins"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
var usageStatsURL = "https://stats.grafana.org/grafana-usage-report"
|
||||
|
||||
func (uss *UsageStatsService) sendUsageStats(oauthProviders map[string]bool) {
|
||||
if !setting.ReportingEnabled {
|
||||
return
|
||||
}
|
||||
|
||||
metricsLogger.Debug(fmt.Sprintf("Sending anonymous usage stats to %s", usageStatsURL))
|
||||
|
||||
version := strings.Replace(setting.BuildVersion, ".", "_", -1)
|
||||
|
||||
metrics := map[string]interface{}{}
|
||||
report := map[string]interface{}{
|
||||
"version": version,
|
||||
"metrics": metrics,
|
||||
"os": runtime.GOOS,
|
||||
"arch": runtime.GOARCH,
|
||||
"edition": getEdition(),
|
||||
"packaging": setting.Packaging,
|
||||
}
|
||||
|
||||
statsQuery := models.GetSystemStatsQuery{}
|
||||
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
|
||||
metricsLogger.Error("Failed to get system stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards
|
||||
metrics["stats.users.count"] = statsQuery.Result.Users
|
||||
metrics["stats.orgs.count"] = statsQuery.Result.Orgs
|
||||
metrics["stats.playlist.count"] = statsQuery.Result.Playlists
|
||||
metrics["stats.plugins.apps.count"] = len(plugins.Apps)
|
||||
metrics["stats.plugins.panels.count"] = len(plugins.Panels)
|
||||
metrics["stats.plugins.datasources.count"] = len(plugins.DataSources)
|
||||
metrics["stats.alerts.count"] = statsQuery.Result.Alerts
|
||||
metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
|
||||
metrics["stats.datasources.count"] = statsQuery.Result.Datasources
|
||||
metrics["stats.stars.count"] = statsQuery.Result.Stars
|
||||
metrics["stats.folders.count"] = statsQuery.Result.Folders
|
||||
metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions
|
||||
metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions
|
||||
metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards
|
||||
metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots
|
||||
metrics["stats.teams.count"] = statsQuery.Result.Teams
|
||||
|
||||
dsStats := models.GetDataSourceStatsQuery{}
|
||||
if err := uss.Bus.Dispatch(&dsStats); err != nil {
|
||||
metricsLogger.Error("Failed to get datasource stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// send counters for each data source
|
||||
// but ignore any custom data sources
|
||||
// as sending that name could be sensitive information
|
||||
dsOtherCount := 0
|
||||
for _, dsStat := range dsStats.Result {
|
||||
if models.IsKnownDataSourcePlugin(dsStat.Type) {
|
||||
metrics["stats.ds."+dsStat.Type+".count"] = dsStat.Count
|
||||
} else {
|
||||
dsOtherCount += dsStat.Count
|
||||
}
|
||||
}
|
||||
metrics["stats.ds.other.count"] = dsOtherCount
|
||||
|
||||
metrics["stats.packaging."+setting.Packaging+".count"] = 1
|
||||
|
||||
dsAccessStats := models.GetDataSourceAccessStatsQuery{}
|
||||
if err := uss.Bus.Dispatch(&dsAccessStats); err != nil {
|
||||
metricsLogger.Error("Failed to get datasource access stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// send access counters for each data source
|
||||
// but ignore any custom data sources
|
||||
// as sending that name could be sensitive information
|
||||
dsAccessOtherCount := make(map[string]int64)
|
||||
for _, dsAccessStat := range dsAccessStats.Result {
|
||||
if dsAccessStat.Access == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
access := strings.ToLower(dsAccessStat.Access)
|
||||
|
||||
if models.IsKnownDataSourcePlugin(dsAccessStat.Type) {
|
||||
metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count
|
||||
} else {
|
||||
old := dsAccessOtherCount[access]
|
||||
dsAccessOtherCount[access] = old + dsAccessStat.Count
|
||||
}
|
||||
}
|
||||
|
||||
for access, count := range dsAccessOtherCount {
|
||||
metrics["stats.ds_access.other."+access+".count"] = count
|
||||
}
|
||||
|
||||
anStats := models.GetAlertNotifierUsageStatsQuery{}
|
||||
if err := uss.Bus.Dispatch(&anStats); err != nil {
|
||||
metricsLogger.Error("Failed to get alert notification stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, stats := range anStats.Result {
|
||||
metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count
|
||||
}
|
||||
|
||||
authTypes := map[string]bool{}
|
||||
authTypes["anonymous"] = setting.AnonymousEnabled
|
||||
authTypes["basic_auth"] = setting.BasicAuthEnabled
|
||||
authTypes["ldap"] = setting.LdapEnabled
|
||||
authTypes["auth_proxy"] = setting.AuthProxyEnabled
|
||||
|
||||
for provider, enabled := range oauthProviders {
|
||||
authTypes["oauth_"+provider] = enabled
|
||||
}
|
||||
|
||||
for authType, enabled := range authTypes {
|
||||
enabledValue := 0
|
||||
if enabled {
|
||||
enabledValue = 1
|
||||
}
|
||||
metrics["stats.auth_enabled."+authType+".count"] = enabledValue
|
||||
}
|
||||
|
||||
out, _ := json.MarshalIndent(report, "", " ")
|
||||
data := bytes.NewBuffer(out)
|
||||
|
||||
client := http.Client{Timeout: 5 * time.Second}
|
||||
go client.Post(usageStatsURL, "application/json", data)
|
||||
}
|
||||
|
||||
func (uss *UsageStatsService) updateTotalStats() {
|
||||
statsQuery := models.GetSystemStatsQuery{}
|
||||
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
|
||||
metricsLogger.Error("Failed to get system stats", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
metrics.M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards))
|
||||
metrics.M_StatTotal_Users.Set(float64(statsQuery.Result.Users))
|
||||
metrics.M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers))
|
||||
metrics.M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists))
|
||||
metrics.M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs))
|
||||
}
|
||||
|
||||
func getEdition() string {
|
||||
if setting.IsEnterprise {
|
||||
return "enterprise"
|
||||
} else {
|
||||
return "oss"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package usagestats
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/plugins"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
)
|
||||
|
||||
func TestMetrics(t *testing.T) {
|
||||
Convey("Test send usage stats", t, func() {
|
||||
uss := &UsageStatsService{
|
||||
Bus: bus.New(),
|
||||
}
|
||||
|
||||
var getSystemStatsQuery *models.GetSystemStatsQuery
|
||||
uss.Bus.AddHandler(func(query *models.GetSystemStatsQuery) error {
|
||||
|
||||
query.Result = &models.SystemStats{
|
||||
Dashboards: 1,
|
||||
Datasources: 2,
|
||||
Users: 3,
|
||||
ActiveUsers: 4,
|
||||
Orgs: 5,
|
||||
Playlists: 6,
|
||||
Alerts: 7,
|
||||
Stars: 8,
|
||||
Folders: 9,
|
||||
DashboardPermissions: 10,
|
||||
FolderPermissions: 11,
|
||||
ProvisionedDashboards: 12,
|
||||
Snapshots: 13,
|
||||
Teams: 14,
|
||||
}
|
||||
getSystemStatsQuery = query
|
||||
return nil
|
||||
})
|
||||
|
||||
var getDataSourceStatsQuery *models.GetDataSourceStatsQuery
|
||||
uss.Bus.AddHandler(func(query *models.GetDataSourceStatsQuery) error {
|
||||
query.Result = []*models.DataSourceStats{
|
||||
{
|
||||
Type: models.DS_ES,
|
||||
Count: 9,
|
||||
},
|
||||
{
|
||||
Type: models.DS_PROMETHEUS,
|
||||
Count: 10,
|
||||
},
|
||||
{
|
||||
Type: "unknown_ds",
|
||||
Count: 11,
|
||||
},
|
||||
{
|
||||
Type: "unknown_ds2",
|
||||
Count: 12,
|
||||
},
|
||||
}
|
||||
getDataSourceStatsQuery = query
|
||||
return nil
|
||||
})
|
||||
|
||||
var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery
|
||||
uss.Bus.AddHandler(func(query *models.GetDataSourceAccessStatsQuery) error {
|
||||
query.Result = []*models.DataSourceAccessStats{
|
||||
{
|
||||
Type: models.DS_ES,
|
||||
Access: "direct",
|
||||
Count: 1,
|
||||
},
|
||||
{
|
||||
Type: models.DS_ES,
|
||||
Access: "proxy",
|
||||
Count: 2,
|
||||
},
|
||||
{
|
||||
Type: models.DS_PROMETHEUS,
|
||||
Access: "proxy",
|
||||
Count: 3,
|
||||
},
|
||||
{
|
||||
Type: "unknown_ds",
|
||||
Access: "proxy",
|
||||
Count: 4,
|
||||
},
|
||||
{
|
||||
Type: "unknown_ds2",
|
||||
Access: "",
|
||||
Count: 5,
|
||||
},
|
||||
{
|
||||
Type: "unknown_ds3",
|
||||
Access: "direct",
|
||||
Count: 6,
|
||||
},
|
||||
{
|
||||
Type: "unknown_ds4",
|
||||
Access: "direct",
|
||||
Count: 7,
|
||||
},
|
||||
{
|
||||
Type: "unknown_ds5",
|
||||
Access: "proxy",
|
||||
Count: 8,
|
||||
},
|
||||
}
|
||||
getDataSourceAccessStatsQuery = query
|
||||
return nil
|
||||
})
|
||||
|
||||
var getAlertNotifierUsageStatsQuery *models.GetAlertNotifierUsageStatsQuery
|
||||
uss.Bus.AddHandler(func(query *models.GetAlertNotifierUsageStatsQuery) error {
|
||||
query.Result = []*models.NotifierUsageStats{
|
||||
{
|
||||
Type: "slack",
|
||||
Count: 1,
|
||||
},
|
||||
{
|
||||
Type: "webhook",
|
||||
Count: 2,
|
||||
},
|
||||
}
|
||||
|
||||
getAlertNotifierUsageStatsQuery = query
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var responseBuffer *bytes.Buffer
|
||||
var req *http.Request
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
||||
req = r
|
||||
buf, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read response body, err=%v", err)
|
||||
}
|
||||
responseBuffer = bytes.NewBuffer(buf)
|
||||
wg.Done()
|
||||
}))
|
||||
usageStatsURL = ts.URL
|
||||
|
||||
oauthProviders := map[string]bool{
|
||||
"github": true,
|
||||
"gitlab": true,
|
||||
"google": true,
|
||||
"generic_oauth": true,
|
||||
"grafana_com": true,
|
||||
}
|
||||
|
||||
uss.sendUsageStats(oauthProviders)
|
||||
|
||||
Convey("Given reporting not enabled and sending usage stats", func() {
|
||||
setting.ReportingEnabled = false
|
||||
uss.sendUsageStats(oauthProviders)
|
||||
|
||||
Convey("Should not gather stats or call http endpoint", func() {
|
||||
So(getSystemStatsQuery, ShouldBeNil)
|
||||
So(getDataSourceStatsQuery, ShouldBeNil)
|
||||
So(getDataSourceAccessStatsQuery, ShouldBeNil)
|
||||
So(req, ShouldBeNil)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("Given reporting enabled and sending usage stats", func() {
|
||||
setting.ReportingEnabled = true
|
||||
setting.BuildVersion = "5.0.0"
|
||||
setting.AnonymousEnabled = true
|
||||
setting.BasicAuthEnabled = true
|
||||
setting.LdapEnabled = true
|
||||
setting.AuthProxyEnabled = true
|
||||
setting.Packaging = "deb"
|
||||
|
||||
wg.Add(1)
|
||||
uss.sendUsageStats(oauthProviders)
|
||||
|
||||
Convey("Should gather stats and call http endpoint", func() {
|
||||
if waitTimeout(&wg, 2*time.Second) {
|
||||
t.Fatalf("Timed out waiting for http request")
|
||||
}
|
||||
|
||||
So(getSystemStatsQuery, ShouldNotBeNil)
|
||||
So(getDataSourceStatsQuery, ShouldNotBeNil)
|
||||
So(getDataSourceAccessStatsQuery, ShouldNotBeNil)
|
||||
So(getAlertNotifierUsageStatsQuery, ShouldNotBeNil)
|
||||
So(req, ShouldNotBeNil)
|
||||
So(req.Method, ShouldEqual, http.MethodPost)
|
||||
So(req.Header.Get("Content-Type"), ShouldEqual, "application/json")
|
||||
|
||||
So(responseBuffer, ShouldNotBeNil)
|
||||
|
||||
j, err := simplejson.NewFromReader(responseBuffer)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(j.Get("version").MustString(), ShouldEqual, "5_0_0")
|
||||
So(j.Get("os").MustString(), ShouldEqual, runtime.GOOS)
|
||||
So(j.Get("arch").MustString(), ShouldEqual, runtime.GOARCH)
|
||||
|
||||
metrics := j.Get("metrics")
|
||||
So(metrics.Get("stats.dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Dashboards)
|
||||
So(metrics.Get("stats.users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Users)
|
||||
So(metrics.Get("stats.orgs.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Orgs)
|
||||
So(metrics.Get("stats.playlist.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Playlists)
|
||||
So(metrics.Get("stats.plugins.apps.count").MustInt(), ShouldEqual, len(plugins.Apps))
|
||||
So(metrics.Get("stats.plugins.panels.count").MustInt(), ShouldEqual, len(plugins.Panels))
|
||||
So(metrics.Get("stats.plugins.datasources.count").MustInt(), ShouldEqual, len(plugins.DataSources))
|
||||
So(metrics.Get("stats.alerts.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Alerts)
|
||||
So(metrics.Get("stats.active_users.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ActiveUsers)
|
||||
So(metrics.Get("stats.datasources.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Datasources)
|
||||
So(metrics.Get("stats.stars.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Stars)
|
||||
So(metrics.Get("stats.folders.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Folders)
|
||||
So(metrics.Get("stats.dashboard_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.DashboardPermissions)
|
||||
So(metrics.Get("stats.folder_permissions.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.FolderPermissions)
|
||||
So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards)
|
||||
So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots)
|
||||
So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams)
|
||||
|
||||
So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9)
|
||||
So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10)
|
||||
So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12)
|
||||
|
||||
So(metrics.Get("stats.ds_access."+models.DS_ES+".direct.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.ds_access."+models.DS_ES+".proxy.count").MustInt(), ShouldEqual, 2)
|
||||
So(metrics.Get("stats.ds_access."+models.DS_PROMETHEUS+".proxy.count").MustInt(), ShouldEqual, 3)
|
||||
So(metrics.Get("stats.ds_access.other.direct.count").MustInt(), ShouldEqual, 6+7)
|
||||
So(metrics.Get("stats.ds_access.other.proxy.count").MustInt(), ShouldEqual, 4+8)
|
||||
|
||||
So(metrics.Get("stats.alert_notifiers.slack.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.alert_notifiers.webhook.count").MustInt(), ShouldEqual, 2)
|
||||
|
||||
So(metrics.Get("stats.auth_enabled.anonymous.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.basic_auth.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.ldap.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.auth_proxy.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.oauth_github.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.oauth_gitlab.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.oauth_google.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.oauth_generic_oauth.count").MustInt(), ShouldEqual, 1)
|
||||
So(metrics.Get("stats.auth_enabled.oauth_grafana_com.count").MustInt(), ShouldEqual, 1)
|
||||
|
||||
So(metrics.Get("stats.packaging.deb.count").MustInt(), ShouldEqual, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Reset(func() {
|
||||
ts.Close()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
|
||||
c := make(chan struct{})
|
||||
go func() {
|
||||
defer close(c)
|
||||
wg.Wait()
|
||||
}()
|
||||
select {
|
||||
case <-c:
|
||||
return false // completed normally
|
||||
case <-time.After(timeout):
|
||||
return true // timed out
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user