public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Chore: Move stats service into a standalone packge from sqlstore (#59574)

Browse Source 
* move original stats service into a separate package

* add stats service to wire

* move GetAdminStats

* switch to using stats.Service

* add missing package

* fix api tests
This commit is contained in:
Serge Zaitsev
2022-11-30 18:11:07 +01:00
committed by GitHub
parent a343defe64
commit 9cdb6b07c7
14 changed files with 200 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/stats_integration_test.go
-20
View File
@@ -1,20 +0,0 @@
package sqlstore
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/require"
)
func TestIntegration_GetAdminStats(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
sqlStore := InitTestDB(t)
query := models.GetAdminStatsQuery{}
err := sqlStore.GetAdminStats(context.Background(), &query)
require.NoError(t, err)
}
pkg/services/sqlstore/store.go
-5
View File
@@ -12,13 +12,8 @@ import (
)
type Store interface {
GetAdminStats(ctx context.Context, query *models.GetAdminStatsQuery) error
GetAlertNotifiersUsageStats(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error
GetDataSourceStats(ctx context.Context, query *models.GetDataSourceStatsQuery) error
GetDataSourceAccessStats(ctx context.Context, query *models.GetDataSourceAccessStatsQuery) error
GetDialect() migrator.Dialect
GetDBType() core.DbType
GetSystemStats(ctx context.Context, query *models.GetSystemStatsQuery) error
CreateUser(ctx context.Context, cmd user.CreateUserCommand) (*user.User, error)
WithDbSession(ctx context.Context, callback DBTransactionFunc) error
WithNewDbSession(ctx context.Context, callback DBTransactionFunc) error
pkg/services/stats/stats.go
+16
View File
@@ -0,0 +1,16 @@
package stats
import (
"context"
"github.com/grafana/grafana/pkg/models"
)
type Service interface {
GetAdminStats(ctx context.Context, query *models.GetAdminStatsQuery) error
GetAlertNotifiersUsageStats(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error
GetDataSourceStats(ctx context.Context, query *models.GetDataSourceStatsQuery) error
GetDataSourceAccessStats(ctx context.Context, query *models.GetDataSourceAccessStatsQuery) error
GetSystemStats(ctx context.Context, query *models.GetSystemStatsQuery) error
GetSystemUserCountStats(ctx context.Context, query *models.GetSystemUserCountStatsQuery) error
}
pkg/services/sqlstore/stats.go → pkg/services/stats/statsimpl/stats.go
+41 -29
View File
@@ -1,39 +1,48 @@
package sqlstore
package statsimpl
import (
"context"
"strconv"
"time"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/services/stats"
)
const activeUserTimeLimit = time.Hour * 24 * 30
const dailyActiveUserTimeLimit = time.Hour * 24
func (ss *SQLStore) GetAlertNotifiersUsageStats(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error {
return ss.WithDbSession(ctx, func(dbSession *DBSession) error {
var rawSQL = `SELECT COUNT(*) AS count, type FROM ` + dialect.Quote("alert_notification") + ` GROUP BY type`
func ProvideService(db db.DB) stats.Service {
return &sqlStatsService{db: db}
}
type sqlStatsService struct{ db db.DB }
func (ss *sqlStatsService) GetAlertNotifiersUsageStats(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error {
return ss.db.WithDbSession(ctx, func(dbSession *db.Session) error {
var rawSQL = `SELECT COUNT(*) AS count, type FROM ` + ss.db.GetDialect().Quote("alert_notification") + ` GROUP BY type`
query.Result = make([]*models.NotifierUsageStats, 0)
err := dbSession.SQL(rawSQL).Find(&query.Result)
return err
})
}
func (ss *SQLStore) GetDataSourceStats(ctx context.Context, query *models.GetDataSourceStatsQuery) error {
return ss.WithDbSession(ctx, func(dbSession *DBSession) error {
var rawSQL = `SELECT COUNT(*) AS count, type FROM ` + dialect.Quote("data_source") + ` GROUP BY type`
func (ss *sqlStatsService) GetDataSourceStats(ctx context.Context, query *models.GetDataSourceStatsQuery) error {
return ss.db.WithDbSession(ctx, func(dbSession *db.Session) error {
var rawSQL = `SELECT COUNT(*) AS count, type FROM ` + ss.db.GetDialect().Quote("data_source") + ` GROUP BY type`
query.Result = make([]*models.DataSourceStats, 0)
err := dbSession.SQL(rawSQL).Find(&query.Result)
return err
})
}
func (ss *SQLStore) GetDataSourceAccessStats(ctx context.Context, query *models.GetDataSourceAccessStatsQuery) error {
return ss.WithDbSession(ctx, func(dbSession *DBSession) error {
var rawSQL = `SELECT COUNT(*) AS count, type, access FROM ` + dialect.Quote("data_source") + ` GROUP BY type, access`
func (ss *sqlStatsService) GetDataSourceAccessStats(ctx context.Context, query *models.GetDataSourceAccessStatsQuery) error {
return ss.db.WithDbSession(ctx, func(dbSession *db.Session) error {
var rawSQL = `SELECT COUNT(*) AS count, type, access FROM ` + ss.db.GetDialect().Quote("data_source") + ` GROUP BY type, access`
query.Result = make([]*models.DataSourceAccessStats, 0)
err := dbSession.SQL(rawSQL).Find(&query.Result)
return err
@@ -45,10 +54,11 @@ func notServiceAccount(dialect migrator.Dialect) string {
dialect.BooleanStr(false)
}
func (ss *SQLStore) GetSystemStats(ctx context.Context, query *models.GetSystemStatsQuery) error {
return ss.WithDbSession(ctx, func(dbSession *DBSession) error {
sb := &SQLBuilder{}
func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *models.GetSystemStatsQuery) error {
return ss.db.WithDbSession(ctx, func(dbSession *db.Session) error {
sb := &sqlstore.SQLBuilder{}
sb.Write("SELECT ")
dialect := ss.db.GetDialect()
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` WHERE ` + notServiceAccount(dialect) + `) AS users,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`)
@@ -88,10 +98,10 @@ func (ss *SQLStore) GetSystemStats(ctx context.Context, query *models.GetSystemS
WHERE d.is_folder = ?
) AS folder_permissions,`, dialect.BooleanStr(true))
sb.Write(viewersPermissionsCounterSQL("dashboards_viewers_can_edit", false, models.PERMISSION_EDIT))
sb.Write(viewersPermissionsCounterSQL("dashboards_viewers_can_admin", false, models.PERMISSION_ADMIN))
sb.Write(viewersPermissionsCounterSQL("folders_viewers_can_edit", true, models.PERMISSION_EDIT))
sb.Write(viewersPermissionsCounterSQL("folders_viewers_can_admin", true, models.PERMISSION_ADMIN))
sb.Write(viewersPermissionsCounterSQL(ss.db, "dashboards_viewers_can_edit", false, models.PERMISSION_EDIT))
sb.Write(viewersPermissionsCounterSQL(ss.db, "dashboards_viewers_can_admin", false, models.PERMISSION_ADMIN))
sb.Write(viewersPermissionsCounterSQL(ss.db, "folders_viewers_can_edit", true, models.PERMISSION_EDIT))
sb.Write(viewersPermissionsCounterSQL(ss.db, "folders_viewers_can_admin", true, models.PERMISSION_ADMIN))
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`)
@@ -112,7 +122,7 @@ func (ss *SQLStore) GetSystemStats(ctx context.Context, query *models.GetSystemS
sb.Write(ss.roleCounterSQL(ctx))
var stats models.SystemStats
_, err := dbSession.SQL(sb.GetSQLString(), sb.params...).Get(&stats)
_, err := dbSession.SQL(sb.GetSQLString(), sb.GetParams()...).Get(&stats)
if err != nil {
return err
}
@@ -123,7 +133,7 @@ func (ss *SQLStore) GetSystemStats(ctx context.Context, query *models.GetSystemS
})
}
func (ss *SQLStore) roleCounterSQL(ctx context.Context) string {
func (ss *sqlStatsService) roleCounterSQL(ctx context.Context) string {
const roleCounterTimeout = 20 * time.Second
ctx, cancel := context.WithTimeout(ctx, roleCounterTimeout)
defer cancel()
@@ -142,7 +152,8 @@ func (ss *SQLStore) roleCounterSQL(ctx context.Context) string {
return sqlQuery
}
func viewersPermissionsCounterSQL(statName string, isFolder bool, permission models.PermissionType) string {
func viewersPermissionsCounterSQL(db db.DB, statName string, isFolder bool, permission models.PermissionType) string {
dialect := db.GetDialect()
return `(
SELECT COUNT(*)
FROM ` + dialect.Quote("dashboard_acl") + ` AS acl
@@ -154,8 +165,9 @@ func viewersPermissionsCounterSQL(statName string, isFolder bool, permission mod
) AS ` + statName + `, `
}
func (ss *SQLStore) GetAdminStats(ctx context.Context, query *models.GetAdminStatsQuery) error {
return ss.WithDbSession(ctx, func(dbSession *DBSession) error {
func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *models.GetAdminStatsQuery) error {
return ss.db.WithDbSession(ctx, func(dbSession *db.Session) error {
dialect := ss.db.GetDialect()
now := time.Now()
activeEndDate := now.Add(-activeUserTimeLimit)
dailyActiveEndDate := now.Add(-dailyActiveUserTimeLimit)
@@ -231,9 +243,9 @@ func (ss *SQLStore) GetAdminStats(ctx context.Context, query *models.GetAdminSta
})
}
func (ss *SQLStore) GetSystemUserCountStats(ctx context.Context, query *models.GetSystemUserCountStatsQuery) error {
return ss.WithDbSession(ctx, func(sess *DBSession) error {
var rawSQL = `SELECT COUNT(id) AS Count FROM ` + dialect.Quote("user")
func (ss *sqlStatsService) GetSystemUserCountStats(ctx context.Context, query *models.GetSystemUserCountStatsQuery) error {
return ss.db.WithDbSession(ctx, func(sess *db.Session) error {
var rawSQL = `SELECT COUNT(id) AS Count FROM ` + ss.db.GetDialect().Quote("user")
var stats models.SystemUserCountStats
_, err := sess.SQL(rawSQL).Get(&stats)
if err != nil {
@@ -246,7 +258,7 @@ func (ss *SQLStore) GetSystemUserCountStats(ctx context.Context, query *models.G
})
}
func (ss *SQLStore) updateUserRoleCountsIfNecessary(ctx context.Context, forced bool) error {
func (ss *sqlStatsService) updateUserRoleCountsIfNecessary(ctx context.Context, forced bool) error {
memoizationPeriod := time.Now().Add(-userStatsCacheLimetime)
if forced || userStatsCache.memoized.Before(memoizationPeriod) {
err := ss.updateUserRoleCounts(ctx)
@@ -270,8 +282,8 @@ var (
userStatsCacheLimetime = 5 * time.Minute
)
func (ss *SQLStore) updateUserRoleCounts(ctx context.Context) error {
return ss.WithDbSession(ctx, func(dbSession *DBSession) error {
func (ss *sqlStatsService) updateUserRoleCounts(ctx context.Context) error {
return ss.db.WithDbSession(ctx, func(dbSession *db.Session) error {
query := `
SELECT role AS bitrole, active, COUNT(role) AS count FROM
(SELECT last_seen_at>? AS active, last_seen_at>? AS daily_active, SUM(role) AS role
@@ -283,7 +295,7 @@ SELECT role AS bitrole, active, COUNT(role) AS count FROM
ELSE 1
END AS role,
u.last_seen_at
FROM ` + dialect.Quote("user") + ` AS u INNER JOIN org_user ON org_user.user_id = u.id
FROM ` + ss.db.GetDialect().Quote("user") + ` AS u INNER JOIN org_user ON org_user.user_id = u.id
GROUP BY u.id, u.last_seen_at, org_user.role) AS t2
GROUP BY id, last_seen_at) AS t1
GROUP BY active, daily_active, role;`
pkg/services/sqlstore/stats_test.go → pkg/services/stats/statsimpl/stats_test.go
+40 -27
View File
@@ -1,28 +1,31 @@
package sqlstore
package statsimpl
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/org/orgimpl"
"github.com/grafana/grafana/pkg/services/quota/quotatest"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/user"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestIntegrationStatsDataAccess(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
sqlStore := InitTestDB(t)
populateDB(t, sqlStore)
db := sqlstore.InitTestDB(t)
statsService := &sqlStatsService{db: db}
populateDB(t, db)
t.Run("Get system stats should not results in error", func(t *testing.T) {
query := models.GetSystemStatsQuery{}
err := sqlStore.GetSystemStats(context.Background(), &query)
err := statsService.GetSystemStats(context.Background(), &query)
require.NoError(t, err)
assert.Equal(t, int64(3), query.Result.Users)
assert.Equal(t, int64(0), query.Result.Editors)
@@ -35,38 +38,40 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
t.Run("Get system user count stats should not results in error", func(t *testing.T) {
query := models.GetSystemUserCountStatsQuery{}
err := sqlStore.GetSystemUserCountStats(context.Background(), &query)
err := statsService.GetSystemUserCountStats(context.Background(), &query)
assert.NoError(t, err)
})
t.Run("Get datasource stats should not results in error", func(t *testing.T) {
query := models.GetDataSourceStatsQuery{}
err := sqlStore.GetDataSourceStats(context.Background(), &query)
err := statsService.GetDataSourceStats(context.Background(), &query)
assert.NoError(t, err)
})
t.Run("Get datasource access stats should not results in error", func(t *testing.T) {
query := models.GetDataSourceAccessStatsQuery{}
err := sqlStore.GetDataSourceAccessStats(context.Background(), &query)
err := statsService.GetDataSourceAccessStats(context.Background(), &query)
assert.NoError(t, err)
})
t.Run("Get alert notifier stats should not results in error", func(t *testing.T) {
query := models.GetAlertNotifierUsageStatsQuery{}
err := sqlStore.GetAlertNotifiersUsageStats(context.Background(), &query)
err := statsService.GetAlertNotifiersUsageStats(context.Background(), &query)
assert.NoError(t, err)
})
t.Run("Get admin stats should not result in error", func(t *testing.T) {
query := models.GetAdminStatsQuery{}
err := sqlStore.GetAdminStats(context.Background(), &query)
err := statsService.GetAdminStats(context.Background(), &query)
assert.NoError(t, err)
})
}
func populateDB(t *testing.T, sqlStore *SQLStore) {
func populateDB(t *testing.T, sqlStore *sqlstore.SQLStore) {
t.Helper()
orgService, _ := orgimpl.ProvideService(sqlStore, sqlStore.Cfg, quotatest.New(false, nil))
users := make([]user.User, 3)
for i := range users {
cmd := user.CreateUserCommand{
@@ -81,33 +86,41 @@ func populateDB(t *testing.T, sqlStore *SQLStore) {
}
// add 2nd user as editor
cmd := &models.AddOrgUserCommand{
OrgId: users[0].OrgID,
UserId: users[1].ID,
cmd := &org.AddOrgUserCommand{
OrgID: users[0].OrgID,
UserID: users[1].ID,
Role: org.RoleEditor,
}
err := sqlStore.addOrgUser(context.Background(), cmd)
err := orgService.AddOrgUser(context.Background(), cmd)
require.NoError(t, err)
// add 3rd user as viewer
cmd = &models.AddOrgUserCommand{
OrgId: users[0].OrgID,
UserId: users[2].ID,
cmd = &org.AddOrgUserCommand{
OrgID: users[0].OrgID,
UserID: users[2].ID,
Role: org.RoleViewer,
}
err = sqlStore.addOrgUser(context.Background(), cmd)
err = orgService.AddOrgUser(context.Background(), cmd)
require.NoError(t, err)
// add 1st user as admin
cmd = &models.AddOrgUserCommand{
OrgId: users[1].OrgID,
UserId: users[0].ID,
cmd = &org.AddOrgUserCommand{
OrgID: users[1].OrgID,
UserID: users[0].ID,
Role: org.RoleAdmin,
}
err = sqlStore.addOrgUser(context.Background(), cmd)
err = orgService.AddOrgUser(context.Background(), cmd)
require.NoError(t, err)
}
// force renewal of user stats
err = sqlStore.updateUserRoleCountsIfNecessary(context.Background(), true)
func TestIntegration_GetAdminStats(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
db := sqlstore.InitTestDB(t)
statsService := ProvideService(db)
query := models.GetAdminStatsQuery{}
err := statsService.GetAdminStats(context.Background(), &query)
require.NoError(t, err)
}
pkg/services/stats/statstest/stats.go
+48
View File
@@ -0,0 +1,48 @@
package statstest
import (
"context"
"github.com/grafana/grafana/pkg/models"
)
type FakeService struct {
ExpectedSystemStats *models.SystemStats
ExpectedDataSourceStats []*models.DataSourceStats
ExpectedDataSourcesAccessStats []*models.DataSourceAccessStats
ExpectedNotifierUsageStats []*models.NotifierUsageStats
ExpectedError error
}
func NewFakeService() *FakeService {
return &FakeService{}
}
func (s *FakeService) GetAdminStats(ctx context.Context, query *models.GetAdminStatsQuery) error {
return s.ExpectedError
}
func (s *FakeService) GetAlertNotifiersUsageStats(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error {
query.Result = s.ExpectedNotifierUsageStats
return s.ExpectedError
}
func (s *FakeService) GetDataSourceStats(ctx context.Context, query *models.GetDataSourceStatsQuery) error {
query.Result = s.ExpectedDataSourceStats
return s.ExpectedError
}
func (s *FakeService) GetDataSourceAccessStats(ctx context.Context, query *models.GetDataSourceAccessStatsQuery) error {
query.Result = s.ExpectedDataSourcesAccessStats
return s.ExpectedError
}
func (s *FakeService) GetSystemStats(ctx context.Context, query *models.GetSystemStatsQuery) error {
query.Result = s.ExpectedSystemStats
return s.ExpectedError
}
func (s *FakeService) GetSystemUserCountStats(ctx context.Context, query *models.GetSystemUserCountStatsQuery) error {
return s.ExpectedError
}