public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stats: use dashboard stats rather than list (#99130)

Browse Source
This commit is contained in:
Stephanie Hingtgen
2025-01-17 08:19:56 +03:00
committed by GitHub
parent 5a930e0ec6
commit e019e34eb5
11 changed files with 226 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/stats/statsimpl/stats.go
+19 -40
View File
@@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/libraryelements/model"
"github.com/grafana/grafana/pkg/services/org"
@@ -21,13 +22,14 @@ import (
const activeUserTimeLimit = time.Hour * 24 * 30
const dailyActiveUserTimeLimit = time.Hour * 24
func ProvideService(cfg *setting.Cfg, db db.DB, dashSvc dashboards.DashboardService, folderSvc folder.Service, orgSvc org.Service) stats.Service {
func ProvideService(cfg *setting.Cfg, db db.DB, dashSvc dashboards.DashboardService, folderSvc folder.Service, orgSvc org.Service, features featuremgmt.FeatureToggles) stats.Service {
return &sqlStatsService{
cfg: cfg,
db: db,
folderSvc: folderSvc,
dashSvc: dashSvc,
orgSvc: orgSvc,
features: features,
}
}
@@ -35,48 +37,23 @@ type sqlStatsService struct {
db db.DB
cfg *setting.Cfg
dashSvc dashboards.DashboardService
features featuremgmt.FeatureToggles
folderSvc folder.Service
orgSvc org.Service
}
type dashboardStats struct {
count int
bytesTotal int
bytesMax int
}
func (ss *sqlStatsService) collectDashboardStats(ctx context.Context, orgs []*org.OrgDTO, calculateByteSize bool) (dashboardStats, error) {
stats := dashboardStats{
count: 0,
bytesTotal: 0,
bytesMax: 0,
}
func (ss *sqlStatsService) getDashboardCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) {
count := int64(0)
for _, org := range orgs {
ctx = identity.WithRequester(ctx, getStatsRequester(org.ID))
dashs, err := ss.dashSvc.GetAllDashboardsByOrgId(ctx, org.ID)
dashsCount, err := ss.dashSvc.CountDashboardsInOrg(ctx, org.ID)
if err != nil {
return stats, err
}
stats.count += len(dashs)
// only calculate bytes if needed
if calculateByteSize {
for _, dash := range dashs {
b, err := dash.Data.ToDB()
if err != nil {
return stats, err
}
stats.bytesTotal += len(b)
if len(b) > stats.bytesMax {
stats.bytesMax = len(b)
}
}
return 0, err
}
count += dashsCount
}
return stats, nil
return count, nil
}
func (ss *sqlStatsService) getTagCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) {
@@ -172,7 +149,6 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
monthlyActiveUserDeadlineDate := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` WHERE `+
notServiceAccount(dialect)+` AND last_seen_at > ?) AS monthly_active_users,`, monthlyActiveUserDeadlineDate)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_version") + `) AS dashboard_versions,`)
@@ -190,6 +166,11 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
if ss.IsUnifiedAlertingEnabled() {
sb.Write(`(SELECT COUNT(DISTINCT (` + dialect.Quote("rule_group") + `)) FROM ` + dialect.Quote("alert_rule") + `) AS rule_groups,`)
}
// currently not supported when dashboards are in unified storage
if !ss.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
sb.Write(`(SELECT SUM(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_total,`, dialect.BooleanStr(false))
sb.Write(`(SELECT MAX(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_max,`, dialect.BooleanStr(false))
}
sb.Write(ss.roleCounterSQL(ctx))
@@ -216,13 +197,11 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
result.Orgs = int64(len(orgs))
// for services in unified storage, get the stats through the service rather than the db directly
dashStats, err := ss.collectDashboardStats(ctx, orgs, true)
dashCount, err := ss.getDashboardCount(ctx, orgs)
if err != nil {
return result, err
}
result.DashboardBytesMax = int64(dashStats.bytesMax)
result.DashboardBytesTotal = int64(dashStats.bytesTotal)
result.Dashboards = int64(dashStats.count)
result.Dashboards = dashCount
folderCount, err := ss.getFolderCount(ctx, orgs)
if err != nil {
@@ -329,11 +308,11 @@ func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *stats.GetAd
result.Orgs = int64(len(orgs))
// for services in unified storage, get the stats through the service rather than the db directly
dashStats, err := ss.collectDashboardStats(ctx, orgs, false)
dashCount, err := ss.getDashboardCount(ctx, orgs)
if err != nil {
return result, err
}
result.Dashboards = int64(dashStats.count)
result.Dashboards = dashCount
tagCount, err := ss.getTagCount(ctx, orgs)
if err != nil {
pkg/services/stats/statsimpl/stats_test.go
+7 -14
View File
@@ -10,12 +10,12 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/correlations"
"github.com/grafana/grafana/pkg/services/correlations/correlationstest"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/folder/foldertest"
"github.com/grafana/grafana/pkg/services/org"
@@ -40,15 +40,9 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
db, cfg := db.InitTestDBWithCfg(t)
orgSvc := populateDB(t, db, cfg)
dashSvc := &dashboards.FakeDashboardService{}
emptyJson := simplejson.New()
emptyJsonBytes, err := emptyJson.ToDB()
require.NoError(t, err)
largerJson := simplejson.NewFromAny(map[string]string{"key": "value"})
largerJsonBytes, err := largerJson.ToDB()
require.NoError(t, err)
dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(1)).Return([]*dashboards.Dashboard{{Data: largerJson}, {Data: emptyJson}}, nil)
dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(2)).Return([]*dashboards.Dashboard{}, nil)
dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(3)).Return([]*dashboards.Dashboard{}, nil)
dashSvc.On("CountDashboardsInOrg", mock.Anything, int64(1)).Return(int64(2), nil)
dashSvc.On("CountDashboardsInOrg", mock.Anything, int64(2)).Return(int64(1), nil)
dashSvc.On("CountDashboardsInOrg", mock.Anything, int64(3)).Return(int64(0), nil)
dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 1}).Return([]*dashboards.DashboardTagCloudItem{{Term: "test"}}, nil)
dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 2}).Return([]*dashboards.DashboardTagCloudItem{}, nil)
dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 3}).Return([]*dashboards.DashboardTagCloudItem{}, nil)
@@ -61,6 +55,7 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
dashSvc: dashSvc,
orgSvc: orgSvc,
folderSvc: folderService,
features: featuremgmt.WithFeatures(),
}
t.Run("Get system stats should not results in error", func(t *testing.T) {
@@ -76,10 +71,8 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
assert.Equal(t, int64(0), result.APIKeys)
assert.Equal(t, int64(2), result.Correlations)
assert.Equal(t, int64(3), result.Orgs)
assert.Equal(t, int64(2), result.Dashboards)
assert.Equal(t, int64(3), result.Dashboards)
assert.Equal(t, int64(9), result.Folders) // will return 3 folders for each org
assert.Equal(t, int64(len(largerJsonBytes)+len(emptyJsonBytes)), result.DashboardBytesTotal)
assert.Equal(t, int64(len(largerJsonBytes)), result.DashboardBytesMax)
assert.NotNil(t, result.DatabaseCreatedTime)
assert.Equal(t, db.GetDialect().DriverName(), result.DatabaseDriver)
})
@@ -113,7 +106,7 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
stats, err := statsService.GetAdminStats(context.Background(), &query)
assert.NoError(t, err)
assert.Equal(t, int64(1), stats.Tags)
assert.Equal(t, int64(2), stats.Dashboards)
assert.Equal(t, int64(3), stats.Dashboards)
assert.Equal(t, int64(3), stats.Orgs)
})
}