diff --git a/pkg/services/annotations/annotationsimpl/annotations.go b/pkg/services/annotations/annotationsimpl/annotations.go index b73ed32bd36..c1929c875a3 100644 --- a/pkg/services/annotations/annotationsimpl/annotations.go +++ b/pkg/services/annotations/annotationsimpl/annotations.go @@ -40,7 +40,7 @@ func ProvideService( l := log.New("annotations") l.Debug("Initializing annotations service") - xormStore := NewXormStore(cfg, log.New("annotations.sql"), db, tagService) + xormStore := NewXormStore(cfg, log.New("annotations.sql"), db, tagService, reg) write := xormStore var read readStore diff --git a/pkg/services/annotations/annotationsimpl/cleanup.go b/pkg/services/annotations/annotationsimpl/cleanup.go index b81597122f0..59fd31dfee5 100644 --- a/pkg/services/annotations/annotationsimpl/cleanup.go +++ b/pkg/services/annotations/annotationsimpl/cleanup.go @@ -15,7 +15,7 @@ type CleanupServiceImpl struct { func ProvideCleanupService(db db.DB, cfg *setting.Cfg) *CleanupServiceImpl { return &CleanupServiceImpl{ - store: NewXormStore(cfg, log.New("annotations"), db, nil), + store: NewXormStore(cfg, log.New("annotations"), db, nil, nil), } } diff --git a/pkg/services/annotations/annotationsimpl/cleanup_test.go b/pkg/services/annotations/annotationsimpl/cleanup_test.go index 5d3902e6363..9146be6c1ed 100644 --- a/pkg/services/annotations/annotationsimpl/cleanup_test.go +++ b/pkg/services/annotations/annotationsimpl/cleanup_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -185,7 +186,7 @@ func TestIntegrationOldAnnotationsAreDeletedFirst(t *testing.T) { // run the clean up task to keep one annotation. cfg := setting.NewCfg() cfg.AnnotationCleanupJobBatchSize = 1 - cleaner := NewXormStore(cfg, log.New("annotation.test"), fakeSQL, nil) + cleaner := NewXormStore(cfg, log.New("annotation.test"), fakeSQL, nil, prometheus.NewRegistry()) _, err = cleaner.CleanAnnotations(context.Background(), setting.AnnotationCleanupSettings{MaxCount: 1}, alertAnnotationType) require.NoError(t, err) diff --git a/pkg/services/annotations/annotationsimpl/xorm_store.go b/pkg/services/annotations/annotationsimpl/xorm_store.go index 8945c02075a..c59dc43f43f 100644 --- a/pkg/services/annotations/annotationsimpl/xorm_store.go +++ b/pkg/services/annotations/annotationsimpl/xorm_store.go @@ -8,12 +8,15 @@ import ( "strings" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/grafana/grafana/pkg/services/annotations/accesscontrol" "github.com/grafana/grafana/pkg/services/sqlstore/migrations" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/metrics/metricutil" "github.com/grafana/grafana/pkg/services/annotations" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/tag" @@ -38,26 +41,59 @@ func validateTimeRange(item *annotations.Item) error { } type xormRepositoryImpl struct { - cfg *setting.Cfg - db db.DB - log log.Logger - tagService tag.Service + cfg *setting.Cfg + db db.DB + log log.Logger + tagService tag.Service + queryRangeStart *prometheus.HistogramVec + queryRangeDuration *prometheus.HistogramVec + queryResultsCount *prometheus.HistogramVec } -func NewXormStore(cfg *setting.Cfg, l log.Logger, db db.DB, tagService tag.Service) *xormRepositoryImpl { - // populate dashboard_uid at startup, to ensure safe downgrades & upgrades after - // the initial migration occurs +func NewXormStore(cfg *setting.Cfg, l log.Logger, db db.DB, tagService tag.Service, reg prometheus.Registerer) *xormRepositoryImpl { err := migrations.RunDashboardUIDMigrations(db.GetEngine().NewSession(), db.GetEngine().DriverName()) if err != nil { l.Error("failed to populate dashboard_uid for annotations", "error", err) } - return &xormRepositoryImpl{ + repo := &xormRepositoryImpl{ cfg: cfg, db: db, log: l, tagService: tagService, } + + if reg != nil { + repo.queryRangeStart = metricutil.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "grafana", + Subsystem: "annotations", + Name: "query_range_start_hours", + Help: "How far back in time (hours from now) annotation queries request", + }, + []string{"query_type"}, + ) + repo.queryRangeDuration = metricutil.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "grafana", + Subsystem: "annotations", + Name: "query_range_duration_hours", + Help: "Time range duration (hours) of annotation queries", + }, + []string{"query_type"}, + ) + repo.queryResultsCount = metricutil.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "grafana", + Subsystem: "annotations", + Name: "query_results_count", + Help: "Number of annotation results returned per query", + }, + []string{"query_type"}, + ) + reg.MustRegister(repo.queryRangeStart, repo.queryRangeDuration, repo.queryResultsCount) + } + return repo } func (r *xormRepositoryImpl) Type() string { @@ -166,13 +202,16 @@ func (r *xormRepositoryImpl) update(ctx context.Context, item *annotations.Item) existing.Text = item.Text if item.Epoch != 0 { + r.log.Info("updating epoch for annotation", "id", item.ID, "orgId", item.OrgID, "oldEpoch", existing.Epoch, "newEpoch", item.Epoch) existing.Epoch = item.Epoch } if item.EpochEnd != 0 { + r.log.Info("updating epoch_end for annotation", "id", item.ID, "orgId", item.OrgID, "oldEpochEnd", existing.EpochEnd, "newEpochEnd", item.EpochEnd) existing.EpochEnd = item.EpochEnd } if item.Data != nil { + r.log.Info("updating data for annotation", "id", item.ID, "orgId", item.OrgID) existing.Data = item.Data } @@ -383,9 +422,46 @@ func (r *xormRepositoryImpl) Get(ctx context.Context, query annotations.ItemQuer }, ) + if err == nil { + r.recordQueryMetrics(query, len(items)) + } + return items, err } +func (r *xormRepositoryImpl) recordQueryMetrics(query annotations.ItemQuery, resultCount int) { + if r.queryRangeStart == nil || query.From == 0 && query.To == 0 { + return + } + + queryType := "all" + if query.AnnotationID != 0 { + queryType = "by_id" + } else if query.AlertID != 0 || query.AlertUID != "" { + queryType = "by_alert" + } else if query.DashboardUID != "" || query.DashboardID != 0 { // nolint: staticcheck + queryType = "by_dashboard" + } else if len(query.Tags) > 0 { + queryType = "by_tags" + } + + if query.From > 0 && query.To > 0 { + now := time.Now().UnixMilli() + startOffsetHours := float64(now-query.To) / (1000.0 * 3600.0) + if startOffsetHours < 0 { + startOffsetHours = 0 + } + r.queryRangeStart.WithLabelValues(queryType).Observe(startOffsetHours) + + durationHours := float64(query.To-query.From) / (1000.0 * 3600.0) + if durationHours < 0 { + durationHours = 0 + } + r.queryRangeDuration.WithLabelValues(queryType).Observe(durationHours) + } + r.queryResultsCount.WithLabelValues(queryType).Observe(float64(resultCount)) +} + func (r *xormRepositoryImpl) getAccessControlFilter(accessResources *accesscontrol.AccessResources, dashboardUID string) (string, []any) { if accessResources.SkipAccessControlFilter { return "", nil diff --git a/pkg/services/annotations/annotationsimpl/xorm_store_test.go b/pkg/services/annotations/annotationsimpl/xorm_store_test.go index d8591099d21..43fff3be327 100644 --- a/pkg/services/annotations/annotationsimpl/xorm_store_test.go +++ b/pkg/services/annotations/annotationsimpl/xorm_store_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -39,7 +40,7 @@ func TestIntegrationAnnotations(t *testing.T) { cfg := setting.NewCfg() cfg.AnnotationMaximumTagsLength = 60 - store := NewXormStore(cfg, log.New("annotation.test"), sql, tagimpl.ProvideService(sql)) + store := NewXormStore(cfg, log.New("annotation.test"), sql, tagimpl.ProvideService(sql), prometheus.NewRegistry()) testUser := &user.SignedInUser{ OrgID: 1,