From 92e591d2e10c038c41a12ec1f8aa8555c1cfc347 Mon Sep 17 00:00:00 2001 From: juanicabanas Date: Tue, 11 Apr 2023 14:36:50 -0300 Subject: [PATCH] PublicDashboards: metrics collected in background service (#65836) --- .../backgroundsvcs/background_services.go | 3 + pkg/server/wire.go | 2 + .../publicdashboards/database/database.go | 14 +++ .../database/database_test.go | 90 ++++++++++++++++--- .../publicdashboards/metric/metric.go | 74 +++++++++++++++ .../publicdashboards/metric/metrics.go | 23 +++++ .../publicdashboards/models/metrics.go | 11 +++ .../public_dashboard_service_mock.go | 2 +- .../public_dashboard_service_wrapper_mock.go | 2 +- .../public_dashboard_store_mock.go | 26 +++++- .../publicdashboards/publicdashboard.go | 1 + pkg/services/stats/statsimpl/stats.go | 4 +- 12 files changed, 232 insertions(+), 20 deletions(-) create mode 100644 pkg/services/publicdashboards/metric/metric.go create mode 100644 pkg/services/publicdashboards/metric/metrics.go create mode 100644 pkg/services/publicdashboards/models/metrics.go diff --git a/pkg/server/backgroundsvcs/background_services.go b/pkg/server/backgroundsvcs/background_services.go index 08c6460f492..b26ddbbbdd0 100644 --- a/pkg/server/backgroundsvcs/background_services.go +++ b/pkg/server/backgroundsvcs/background_services.go @@ -24,6 +24,7 @@ import ( "github.com/grafana/grafana/pkg/services/notifications" plugindashboardsservice "github.com/grafana/grafana/pkg/services/plugindashboards/service" "github.com/grafana/grafana/pkg/services/provisioning" + publicdashboardsmetric "github.com/grafana/grafana/pkg/services/publicdashboards/metric" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/services/searchV2" secretsMigrations "github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations" @@ -50,6 +51,7 @@ func ProvideBackgroundServiceRegistry( saService *samanager.ServiceAccountsService, authInfoService *authinfoservice.Implementation, grpcServerProvider grpcserver.Provider, secretMigrationProvider secretsMigrations.SecretMigrationProvider, loginAttemptService *loginattemptimpl.Service, bundleService *supportbundlesimpl.Service, + publicDashboardsMetric *publicdashboardsmetric.Service, // Need to make sure these are initialized, is there a better place to put them? _ dashboardsnapshots.Service, _ *alerting.AlertNotificationService, _ serviceaccounts.Service, _ *guardian.Provider, @@ -86,6 +88,7 @@ func ProvideBackgroundServiceRegistry( secretMigrationProvider, loginAttemptService, bundleService, + publicDashboardsMetric, ) } diff --git a/pkg/server/wire.go b/pkg/server/wire.go index c906340f946..2bf368349a3 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -96,6 +96,7 @@ import ( "github.com/grafana/grafana/pkg/services/publicdashboards" publicdashboardsApi "github.com/grafana/grafana/pkg/services/publicdashboards/api" publicdashboardsStore "github.com/grafana/grafana/pkg/services/publicdashboards/database" + publicdashboardsmetric "github.com/grafana/grafana/pkg/services/publicdashboards/metric" publicdashboardsService "github.com/grafana/grafana/pkg/services/publicdashboards/service" "github.com/grafana/grafana/pkg/services/query" "github.com/grafana/grafana/pkg/services/queryhistory" @@ -327,6 +328,7 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(publicdashboards.Service), new(*publicdashboardsService.PublicDashboardServiceImpl)), publicdashboardsStore.ProvideStore, wire.Bind(new(publicdashboards.Store), new(*publicdashboardsStore.PublicDashboardStoreImpl)), + publicdashboardsmetric.ProvideService, publicdashboardsApi.ProvideApi, starApi.ProvideApi, userimpl.ProvideService, diff --git a/pkg/services/publicdashboards/database/database.go b/pkg/services/publicdashboards/database/database.go index 669f6c1d679..bdee4da07ea 100644 --- a/pkg/services/publicdashboards/database/database.go +++ b/pkg/services/publicdashboards/database/database.go @@ -285,3 +285,17 @@ func (d *PublicDashboardStoreImpl) FindByDashboardFolder(ctx context.Context, da return pubdashes, nil } + +func (d *PublicDashboardStoreImpl) GetMetrics(ctx context.Context) (*Metrics, error) { + metrics := &Metrics{ + TotalPublicDashboards: []*TotalPublicDashboard{}, + } + err := d.sqlStore.WithDbSession(ctx, func(sess *db.Session) error { + return sess.SQL("SELECT COUNT(*) as total_count, is_enabled, share as share_type FROM dashboard_public GROUP BY is_enabled, share").Find(&metrics.TotalPublicDashboards) + }) + if err != nil { + return nil, err + } + + return metrics, nil +} diff --git a/pkg/services/publicdashboards/database/database_test.go b/pkg/services/publicdashboards/database/database_test.go index e67da63d104..f918570ceae 100644 --- a/pkg/services/publicdashboards/database/database_test.go +++ b/pkg/services/publicdashboards/database/database_test.go @@ -47,9 +47,9 @@ func TestIntegrationListPublicDashboard(t *testing.T) { cDash := insertTestDashboard(t, dashboardStore, "c", orgId, 0, true) // these are in order of how they should be returned from ListPUblicDashboards - a := insertPublicDashboard(t, publicdashboardStore, bDash.UID, orgId, true) - b := insertPublicDashboard(t, publicdashboardStore, cDash.UID, orgId, true) - c := insertPublicDashboard(t, publicdashboardStore, aDash.UID, orgId, false) + a := insertPublicDashboard(t, publicdashboardStore, bDash.UID, orgId, true, PublicShareType) + b := insertPublicDashboard(t, publicdashboardStore, cDash.UID, orgId, true, PublicShareType) + c := insertPublicDashboard(t, publicdashboardStore, aDash.UID, orgId, false, PublicShareType) // this is case that can happen as of now, however, postgres and mysql sort // null in the exact opposite fashion and there is no shared syntax to sort @@ -57,7 +57,7 @@ func TestIntegrationListPublicDashboard(t *testing.T) { //d := insertPublicDashboard(t, publicdashboardStore, "missing", orgId, false) // should not be included in response - _ = insertPublicDashboard(t, publicdashboardStore, "wrongOrgId", 777, false) + _ = insertPublicDashboard(t, publicdashboardStore, "wrongOrgId", 777, false, PublicShareType) resp, err := publicdashboardStore.FindAll(context.Background(), orgId) require.NoError(t, err) @@ -258,7 +258,7 @@ func TestIntegrationFindByDashboardUid(t *testing.T) { t.Run("returns public dashboard by dashboardUid", func(t *testing.T) { setup() - savedPubdash := insertPublicDashboard(t, publicdashboardStore, savedDashboard.UID, savedDashboard.OrgID, false) + savedPubdash := insertPublicDashboard(t, publicdashboardStore, savedDashboard.UID, savedDashboard.OrgID, false, PublicShareType) pubdash, err := publicdashboardStore.FindByDashboardUid(context.Background(), savedDashboard.OrgID, savedDashboard.UID) require.NoError(t, err) assert.Equal(t, savedPubdash, pubdash) @@ -325,7 +325,7 @@ func TestIntegrationFindByAccessToken(t *testing.T) { t.Run("returns public dashboard by accessToken", func(t *testing.T) { setup() - savedPubdash := insertPublicDashboard(t, publicdashboardStore, savedDashboard.UID, savedDashboard.OrgID, false) + savedPubdash := insertPublicDashboard(t, publicdashboardStore, savedDashboard.UID, savedDashboard.OrgID, false, PublicShareType) pubdash, err := publicdashboardStore.FindByAccessToken(context.Background(), savedPubdash.AccessToken) require.NoError(t, err) assert.Equal(t, savedPubdash, pubdash) @@ -392,7 +392,7 @@ func TestIntegrationCreatePublicDashboard(t *testing.T) { publicdashboardStore = ProvideStore(sqlStore) savedDashboard = insertTestDashboard(t, dashboardStore, "testDashie", 1, 0, true) savedDashboard2 = insertTestDashboard(t, dashboardStore, "testDashie2", 1, 0, true) - insertPublicDashboard(t, publicdashboardStore, savedDashboard2.UID, savedDashboard2.OrgID, false) + insertPublicDashboard(t, publicdashboardStore, savedDashboard2.UID, savedDashboard2.OrgID, false, PublicShareType) } t.Run("saves new public dashboard", func(t *testing.T) { @@ -646,7 +646,7 @@ func TestIntegrationDelete(t *testing.T) { require.NoError(t, err) publicdashboardStore = ProvideStore(sqlStore) savedDashboard = insertTestDashboard(t, dashboardStore, "testDashie", 1, 0, true) - savedPublicDashboard = insertPublicDashboard(t, publicdashboardStore, savedDashboard.UID, savedDashboard.OrgID, true) + savedPublicDashboard = insertPublicDashboard(t, publicdashboardStore, savedDashboard.UID, savedDashboard.OrgID, true, PublicShareType) } t.Run("Delete success", func(t *testing.T) { @@ -697,10 +697,10 @@ func TestGetDashboardByFolder(t *testing.T) { dashboardStore, err := dashboardsDB.ProvideDashboardStore(sqlStore, cfg, featuremgmt.WithFeatures(), tagimpl.ProvideService(sqlStore, cfg), quotaService) require.NoError(t, err) pubdashStore := ProvideStore(sqlStore) - dashboard := insertTestDashboard(t, dashboardStore, "title", 1, 1, true) - pubdash := insertPublicDashboard(t, pubdashStore, dashboard.UID, dashboard.OrgID, true) - dashboard2 := insertTestDashboard(t, dashboardStore, "title", 1, 2, true) - _ = insertPublicDashboard(t, pubdashStore, dashboard2.UID, dashboard2.OrgID, true) + dashboard := insertTestDashboard(t, dashboardStore, "title", 1, 1, true, PublicShareType) + pubdash := insertPublicDashboard(t, pubdashStore, dashboard.UID, dashboard.OrgID, true, PublicShareType) + dashboard2 := insertTestDashboard(t, dashboardStore, "title", 1, 2, true, PublicShareType) + _ = insertPublicDashboard(t, pubdashStore, dashboard2.UID, dashboard2.OrgID, true, PublicShareType) pubdashes, err := pubdashStore.FindByDashboardFolder(context.Background(), dashboard) @@ -710,6 +710,69 @@ func TestGetDashboardByFolder(t *testing.T) { }) } +func TestGetMetrics(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + var sqlStore db.DB + var cfg *setting.Cfg + var dashboardStore dashboards.Store + var publicdashboardStore *PublicDashboardStoreImpl + var savedDashboard *dashboards.Dashboard + var savedDashboard2 *dashboards.Dashboard + var savedDashboard3 *dashboards.Dashboard + var savedDashboard4 *dashboards.Dashboard + + setup := func() { + sqlStore, cfg = db.InitTestDBwithCfg(t, db.InitTestDBOpt{FeatureFlags: []string{featuremgmt.FlagPublicDashboards}}) + quotaService := quotatest.New(false, nil) + store, err := dashboardsDB.ProvideDashboardStore(sqlStore, cfg, featuremgmt.WithFeatures(), tagimpl.ProvideService(sqlStore, cfg), quotaService) + require.NoError(t, err) + dashboardStore = store + publicdashboardStore = ProvideStore(sqlStore) + savedDashboard = insertTestDashboard(t, dashboardStore, "testDashie", 1, 0, false) + savedDashboard2 = insertTestDashboard(t, dashboardStore, "testDashie2", 1, 0, false) + savedDashboard3 = insertTestDashboard(t, dashboardStore, "testDashie3", 2, 0, false) + savedDashboard4 = insertTestDashboard(t, dashboardStore, "testDashie4", 2, 0, false) + insertPublicDashboard(t, publicdashboardStore, savedDashboard.UID, savedDashboard.OrgID, true, PublicShareType) + insertPublicDashboard(t, publicdashboardStore, savedDashboard2.UID, savedDashboard2.OrgID, true, PublicShareType) + insertPublicDashboard(t, publicdashboardStore, savedDashboard3.UID, savedDashboard3.OrgID, true, EmailShareType) + insertPublicDashboard(t, publicdashboardStore, savedDashboard4.UID, savedDashboard4.OrgID, false, EmailShareType) + } + + t.Run("returns correct list of metrics", func(t *testing.T) { + setup() + + metrics, err := publicdashboardStore.GetMetrics(context.Background()) + require.NoError(t, err) + + assert.Equal(t, 3, len(metrics.TotalPublicDashboards)) + enabledAndPublicCount := -1 + enabledAndEmailCount := -1 + disabledAndEmailCount := -1 + disabledAndPublicCount := -1 + for _, metric := range metrics.TotalPublicDashboards { + if metric.ShareType == string(PublicShareType) && metric.IsEnabled { + enabledAndPublicCount = int(metric.TotalCount) + } + if metric.ShareType == string(PublicShareType) && !metric.IsEnabled { + disabledAndPublicCount = int(metric.TotalCount) + } + if metric.ShareType == string(EmailShareType) && metric.IsEnabled { + enabledAndEmailCount = int(metric.TotalCount) + } + if metric.ShareType == string(EmailShareType) && !metric.IsEnabled { + disabledAndEmailCount = int(metric.TotalCount) + } + } + + assert.Equal(t, 2, enabledAndPublicCount) + assert.Equal(t, 1, enabledAndEmailCount) + assert.Equal(t, 1, disabledAndEmailCount) + assert.Equal(t, -1, disabledAndPublicCount) + }) +} + // helper function to insert a dashboard func insertTestDashboard(t *testing.T, dashboardStore dashboards.Store, title string, orgId int64, folderId int64, isFolder bool, tags ...interface{}) *dashboards.Dashboard { @@ -733,7 +796,7 @@ func insertTestDashboard(t *testing.T, dashboardStore dashboards.Store, title st } // helper function to insert a public dashboard -func insertPublicDashboard(t *testing.T, publicdashboardStore *PublicDashboardStoreImpl, dashboardUid string, orgId int64, isEnabled bool) *PublicDashboard { +func insertPublicDashboard(t *testing.T, publicdashboardStore *PublicDashboardStoreImpl, dashboardUid string, orgId int64, isEnabled bool, shareType ShareType) *PublicDashboard { ctx := context.Background() uid := util.GenerateShortUID() @@ -751,6 +814,7 @@ func insertPublicDashboard(t *testing.T, publicdashboardStore *PublicDashboardSt CreatedBy: 1, CreatedAt: time.Now(), AccessToken: accessToken, + Share: shareType, }, } diff --git a/pkg/services/publicdashboards/metric/metric.go b/pkg/services/publicdashboards/metric/metric.go new file mode 100644 index 00000000000..faa68a867e9 --- /dev/null +++ b/pkg/services/publicdashboards/metric/metric.go @@ -0,0 +1,74 @@ +package metric + +import ( + "context" + "errors" + "strconv" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/publicdashboards" + "github.com/prometheus/client_golang/prometheus" +) + +type Service struct { + store publicdashboards.Store + Metrics *Metrics + log log.Logger +} + +func ProvideService( + store publicdashboards.Store, + prom prometheus.Registerer, +) (*Service, error) { + s := &Service{ + store: store, + Metrics: newMetrics(), + log: log.New("publicdashboards.metric"), + } + + if err := s.registerMetrics(prom); err != nil { + return nil, err + } + + return s, nil +} + +func (s *Service) registerMetrics(prom prometheus.Registerer) error { + err := prom.Register(s.Metrics.PublicDashboardsAmount) + var alreadyRegisterErr prometheus.AlreadyRegisteredError + if errors.As(err, &alreadyRegisterErr) { + if alreadyRegisterErr.ExistingCollector == alreadyRegisterErr.NewCollector { + err = nil + } + } + + return err +} + +func (s *Service) Run(ctx context.Context) error { + s.recordMetrics(ctx) + + ticker := time.NewTicker(12 * time.Hour) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + s.recordMetrics(ctx) + } + } +} + +func (s *Service) recordMetrics(ctx context.Context) { + records, err := s.store.GetMetrics(ctx) + if err != nil { + s.log.Error("error collecting background metrics", "err", err) + return + } + + s.Metrics.PublicDashboardsAmount.Reset() + for _, r := range records.TotalPublicDashboards { + s.Metrics.PublicDashboardsAmount.WithLabelValues(strconv.FormatBool(r.IsEnabled), r.ShareType).Set(r.TotalCount) + } +} diff --git a/pkg/services/publicdashboards/metric/metrics.go b/pkg/services/publicdashboards/metric/metrics.go new file mode 100644 index 00000000000..ad4d19b9616 --- /dev/null +++ b/pkg/services/publicdashboards/metric/metrics.go @@ -0,0 +1,23 @@ +package metric + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + namespace = "grafana" +) + +type Metrics struct { + PublicDashboardsAmount *prometheus.GaugeVec +} + +func newMetrics() *Metrics { + return &Metrics{ + PublicDashboardsAmount: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "public_dashboards_amount", + Help: "Total amount of public dashboards", + }, []string{"is_enabled", "share_type"}), + } +} diff --git a/pkg/services/publicdashboards/models/metrics.go b/pkg/services/publicdashboards/models/metrics.go new file mode 100644 index 00000000000..a2e458464d0 --- /dev/null +++ b/pkg/services/publicdashboards/models/metrics.go @@ -0,0 +1,11 @@ +package models + +type TotalPublicDashboard struct { + TotalCount float64 + IsEnabled bool + ShareType string +} + +type Metrics struct { + TotalPublicDashboards []*TotalPublicDashboard +} diff --git a/pkg/services/publicdashboards/public_dashboard_service_mock.go b/pkg/services/publicdashboards/public_dashboard_service_mock.go index 0aa2eb1d568..886cd340ec4 100644 --- a/pkg/services/publicdashboards/public_dashboard_service_mock.go +++ b/pkg/services/publicdashboards/public_dashboard_service_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.16.0. DO NOT EDIT. +// Code generated by mockery v2.14.0. DO NOT EDIT. package publicdashboards diff --git a/pkg/services/publicdashboards/public_dashboard_service_wrapper_mock.go b/pkg/services/publicdashboards/public_dashboard_service_wrapper_mock.go index 5b96a227f48..69ee15d2b9a 100644 --- a/pkg/services/publicdashboards/public_dashboard_service_wrapper_mock.go +++ b/pkg/services/publicdashboards/public_dashboard_service_wrapper_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.16.0. DO NOT EDIT. +// Code generated by mockery v2.14.0. DO NOT EDIT. package publicdashboards diff --git a/pkg/services/publicdashboards/public_dashboard_store_mock.go b/pkg/services/publicdashboards/public_dashboard_store_mock.go index 07e5f97cfdc..47fccf70101 100644 --- a/pkg/services/publicdashboards/public_dashboard_store_mock.go +++ b/pkg/services/publicdashboards/public_dashboard_store_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.16.0. DO NOT EDIT. +// Code generated by mockery v2.14.0. DO NOT EDIT. package publicdashboards @@ -7,7 +7,6 @@ import ( dashboards "github.com/grafana/grafana/pkg/services/dashboards" mock "github.com/stretchr/testify/mock" - models "github.com/grafana/grafana/pkg/services/publicdashboards/models" ) @@ -238,6 +237,29 @@ func (_m *FakePublicDashboardStore) FindDashboard(ctx context.Context, orgId int return r0, r1 } +// GetMetrics provides a mock function with given fields: ctx +func (_m *FakePublicDashboardStore) GetMetrics(ctx context.Context) (*models.Metrics, error) { + ret := _m.Called(ctx) + + var r0 *models.Metrics + if rf, ok := ret.Get(0).(func(context.Context) *models.Metrics); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.Metrics) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetOrgIdByAccessToken provides a mock function with given fields: ctx, accessToken func (_m *FakePublicDashboardStore) GetOrgIdByAccessToken(ctx context.Context, accessToken string) (int64, error) { ret := _m.Called(ctx, accessToken) diff --git a/pkg/services/publicdashboards/publicdashboard.go b/pkg/services/publicdashboards/publicdashboard.go index a1b69547dfa..2b37aa94384 100644 --- a/pkg/services/publicdashboards/publicdashboard.go +++ b/pkg/services/publicdashboards/publicdashboard.go @@ -61,4 +61,5 @@ type Store interface { FindByDashboardFolder(ctx context.Context, dashboard *dashboards.Dashboard) ([]*PublicDashboard, error) ExistsEnabledByAccessToken(ctx context.Context, accessToken string) (bool, error) ExistsEnabledByDashboardUid(ctx context.Context, dashboardUid string) (bool, error) + GetMetrics(ctx context.Context) (*Metrics, error) } diff --git a/pkg/services/stats/statsimpl/stats.go b/pkg/services/stats/statsimpl/stats.go index 122f6093707..66ad8a2d07c 100644 --- a/pkg/services/stats/statsimpl/stats.go +++ b/pkg/services/stats/statsimpl/stats.go @@ -124,9 +124,7 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("library_element")+` WHERE kind = ?) AS library_variables,`, model.VariableElement) sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_keys") + `) AS data_keys,`) sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_keys") + `WHERE active = true) AS active_data_keys,`) - - // TODO: table name will change and filter should check only for is_enabled = true - sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("dashboard_public") + `WHERE is_enabled = true) AS public_dashboards,`) + sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("dashboard_public") + `) AS public_dashboards,`) sb.Write(ss.roleCounterSQL(ctx))