public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stats: Stop counting the same user multiple times (#26777)

Browse Source 
* Stats: Cache based stats implementation

* Stats: Correct logic and add larger scale test

* Stats: linter

* Stats: SQL implementation

* Stats: cleanup SQL

* Stats: Tab -> Spaces

* Update pkg/services/sqlstore/stats.go

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

* Stats: Quote 'user' table with dialect.Quote

* Stats: Ensure test is run as integration test

* Stats: Use boolean value

...because if (v) { true } else { false } is unnecessary at best.

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>
This commit is contained in:
Emil Tullstedt
2020-08-24 11:23:14 +02:00
committed by GitHub
co-authored by Sofia Papagiannaki
parent 09a1af3f91
commit 954a2811b3
4 changed files with 244 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/stats.go
+109 -37
View File
@@ -2,6 +2,7 @@ package sqlstore
import (
"context"
"strconv"
"time"
"github.com/grafana/grafana/pkg/bus"
@@ -13,12 +14,12 @@ func init() {
bus.AddHandler("sql", GetDataSourceStats)
bus.AddHandler("sql", GetDataSourceAccessStats)
bus.AddHandler("sql", GetAdminStats)
bus.AddHandler("sql", GetActiveUserStats)
bus.AddHandler("sql", GetUserStats)
bus.AddHandlerCtx("sql", GetAlertNotifiersUsageStats)
bus.AddHandlerCtx("sql", GetSystemUserCountStats)
}
var activeUserTimeLimit = time.Hour * 24 * 30
const activeUserTimeLimit = time.Hour * 24 * 30
func GetAlertNotifiersUsageStats(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error {
var rawSql = `SELECT COUNT(*) AS count, type FROM ` + dialect.Quote("alert_notification") + ` GROUP BY type`
@@ -80,9 +81,7 @@ func GetSystemStats(query *models.GetSystemStatsQuery) error {
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("user_auth_token") + `) AS auth_tokens,`)
sb.Write(roleCounterSQL("Viewer", "viewers", false)+`,`, activeUserDeadlineDate)
sb.Write(roleCounterSQL("Editor", "editors", false)+`,`, activeUserDeadlineDate)
sb.Write(roleCounterSQL("Admin", "admins", false)+``, activeUserDeadlineDate)
sb.Write(roleCounterSQL())
var stats models.SystemStats
_, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats)
@@ -95,22 +94,15 @@ func GetSystemStats(query *models.GetSystemStatsQuery) error {
return nil
}
func roleCounterSQL(role string, alias string, onlyActive bool) string {
var sqlQuery = `
(
SELECT COUNT(DISTINCT u.id)
FROM ` + dialect.Quote("user") + ` AS u, org_user
WHERE u.last_seen_at > ? AND ( org_user.user_id=u.id AND org_user.role='` + role + `' )
) AS active_` + alias
if !onlyActive {
sqlQuery += `,
(
SELECT COUNT(DISTINCT u.id)
FROM ` + dialect.Quote("user") + ` AS u, org_user
WHERE ( org_user.user_id=u.id AND org_user.role='` + role + `' )
) AS ` + alias
}
func roleCounterSQL() string {
_ = updateUserRoleCountsIfNecessary(false)
sqlQuery :=
strconv.FormatInt(userStatsCache.total.Admins, 10) + ` AS admins, ` +
strconv.FormatInt(userStatsCache.total.Editors, 10) + ` AS editors, ` +
strconv.FormatInt(userStatsCache.total.Viewers, 10) + ` AS viewers, ` +
strconv.FormatInt(userStatsCache.active.Admins, 10) + ` AS active_admins, ` +
strconv.FormatInt(userStatsCache.active.Editors, 10) + ` AS active_editors, ` +
strconv.FormatInt(userStatsCache.active.Viewers, 10) + ` AS active_viewers`
return sqlQuery
}
@@ -159,9 +151,7 @@ func GetAdminStats(query *models.GetAdminStatsQuery) error {
SELECT COUNT(*)
FROM ` + dialect.Quote("user") + ` WHERE last_seen_at > ?
) AS active_users,
` + roleCounterSQL("Admin", "admins", false) + `,
` + roleCounterSQL("Editor", "editors", false) + `,
` + roleCounterSQL("Viewer", "viewers", false) + `,
` + roleCounterSQL() + `,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("user_auth_token") + ` WHERE rotated_at > ?
@@ -192,23 +182,105 @@ func GetSystemUserCountStats(ctx context.Context, query *models.GetSystemUserCou
})
}
func GetActiveUserStats(query *models.GetActiveUserStatsQuery) error {
activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit)
sb := &SqlBuilder{}
sb.Write(`SELECT `)
sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` WHERE last_seen_at > ?) AS active_users,`, activeUserDeadlineDate)
sb.Write(roleCounterSQL("Viewer", "viewers", true)+`,`, activeUserDeadlineDate)
sb.Write(roleCounterSQL("Editor", "editors", true)+`,`, activeUserDeadlineDate)
sb.Write(roleCounterSQL("Admin", "admins", true)+``, activeUserDeadlineDate)
var stats models.ActiveUserStats
_, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats)
func GetUserStats(query *models.GetUserStatsQuery) error {
err := updateUserRoleCountsIfNecessary(query.MustUpdate)
if err != nil {
return err
}
query.Result = &stats
if query.Active {
query.Result = userStatsCache.active
} else {
query.Result = userStatsCache.total
}
return nil
}
func updateUserRoleCountsIfNecessary(forced bool) error {
memoizationPeriod := time.Now().Add(-userStatsCacheLimetime)
if forced || userStatsCache.memoized.Before(memoizationPeriod) {
err := updateUserRoleCounts()
if err != nil {
return err
}
}
return nil
}
type memoUserStats struct {
active models.UserStats
total models.UserStats
memoized time.Time
}
var (
userStatsCache = memoUserStats{}
userStatsCacheLimetime = 5 * time.Minute
)
func updateUserRoleCounts() error {
query := `
SELECT role AS bitrole, active, COUNT(role) AS count FROM
(SELECT active, SUM(role) AS role
FROM (SELECT
u.id,
CASE org_user.role
WHEN 'Admin' THEN 4
WHEN 'Editor' THEN 2
ELSE 1
END AS role,
u.last_seen_at>? AS active
FROM ` + dialect.Quote("user") + ` AS u LEFT JOIN org_user ON org_user.user_id = u.id
GROUP BY u.id, u.last_seen_at, org_user.role) AS t2
GROUP BY active, id) AS t1
GROUP BY active, role;`
activeUserDeadline := time.Now().Add(-activeUserTimeLimit)
type rolebitmap struct {
Active bool
Bitrole int64
Count int64
}
bitmap := []rolebitmap{}
err := x.SQL(query, activeUserDeadline).Find(&bitmap)
if err != nil {
return err
}
memo := memoUserStats{memoized: time.Now()}
for _, role := range bitmap {
roletype := models.ROLE_VIEWER
if role.Bitrole&0b100 != 0 {
roletype = models.ROLE_ADMIN
} else if role.Bitrole&0b10 != 0 {
roletype = models.ROLE_EDITOR
}
memo.total = addToStats(memo.total, roletype, role.Count)
if role.Active {
memo.active = addToStats(memo.active, roletype, role.Count)
}
}
userStatsCache = memo
return nil
}
func addToStats(base models.UserStats, role models.RoleType, count int64) models.UserStats {
base.Users += count
switch role {
case models.ROLE_ADMIN:
base.Admins += count
case models.ROLE_EDITOR:
base.Editors += count
default:
base.Viewers += count
}
return base
}