From 1d0a86252ea6b76727487e96081560984070cbf1 Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Tue, 18 Mar 2025 09:53:50 +0100 Subject: [PATCH] Alerting: Add target datasource support to Prometheus conversion (#102307) --- .../ngalert/api/api_convert_prometheus.go | 26 ++++++++-- .../api/api_convert_prometheus_test.go | 49 ++++++++++++++++++- pkg/services/ngalert/prom/convert.go | 23 +++++++-- pkg/services/ngalert/prom/convert_test.go | 27 +++++++++- 4 files changed, 114 insertions(+), 11 deletions(-) diff --git a/pkg/services/ngalert/api/api_convert_prometheus.go b/pkg/services/ngalert/api/api_convert_prometheus.go index 57d312768cb..d9f28509622 100644 --- a/pkg/services/ngalert/api/api_convert_prometheus.go +++ b/pkg/services/ngalert/api/api_convert_prometheus.go @@ -31,6 +31,8 @@ import ( const ( // datasourceUIDHeader is the name of the header that specifies the UID of the datasource to be used for the rules. datasourceUIDHeader = "X-Grafana-Alerting-Datasource-UID" + // targetDatasourceUIDHeader is the name of the header that specifies the UID of the target datasource to be used for recording rules. + targetDatasourceUIDHeader = "X-Grafana-Alerting-Target-Datasource-UID" // If the folderUIDHeader is present, namespaces and rule groups will be created in the specified folder. // If not, the root folder will be used as the default. @@ -351,7 +353,18 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroup(c *contextm ds, err := srv.datasourceCache.GetDatasourceByUID(c.Req.Context(), datasourceUID, c.SignedInUser, c.SkipDSCache) if err != nil { logger.Error("Failed to get datasource", "datasource_uid", datasourceUID, "error", err) - return errorToResponse(err) + return errorToResponse(fmt.Errorf("failed to get datasource: %w", err)) + } + + // By default the target datasource is the same as the query datasource, + // but if the header "X-Grafana-Alerting-Target-Datasource-UID" is present, we use that instead. + tds := ds + if uid := strings.TrimSpace(c.Req.Header.Get(targetDatasourceUIDHeader)); uid != "" { + tds, err = srv.datasourceCache.GetDatasourceByUID(c.Req.Context(), uid, c.SignedInUser, c.SkipDSCache) + if err != nil { + logger.Error("Failed to get target datasource for recording rules", "datasource_uid", uid, "error", err) + return errorToResponse(fmt.Errorf("failed to get recording rules target datasource: %w", err)) + } } provenance := getProvenance(c) @@ -362,7 +375,7 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroup(c *contextm // to ensure we can return them in this API in Prometheus format. keepOriginalRuleDefinition := provenance == models.ProvenanceConvertedPrometheus - group, err := srv.convertToGrafanaRuleGroup(c, ds, ns.UID, promGroup, keepOriginalRuleDefinition, logger) + group, err := srv.convertToGrafanaRuleGroup(c, ds, tds, ns.UID, promGroup, keepOriginalRuleDefinition, logger) if err != nil { logger.Error("Failed to convert Prometheus rules to Grafana rules", "error", err) return errorToResponse(err) @@ -400,6 +413,7 @@ func (srv *ConvertPrometheusSrv) getOrCreateNamespace(c *contextmodel.ReqContext func (srv *ConvertPrometheusSrv) convertToGrafanaRuleGroup( c *contextmodel.ReqContext, ds *datasources.DataSource, + tds *datasources.DataSource, namespaceUID string, promGroup apimodels.PrometheusRuleGroup, keepOriginalRuleDefinition bool, @@ -437,9 +451,11 @@ func (srv *ConvertPrometheusSrv) convertToGrafanaRuleGroup( converter, err := prom.NewConverter( prom.Config{ - DatasourceUID: ds.UID, - DatasourceType: ds.Type, - DefaultInterval: srv.cfg.DefaultRuleEvaluationInterval, + DatasourceUID: ds.UID, + DatasourceType: ds.Type, + TargetDatasourceUID: tds.UID, + TargetDatasourceType: tds.Type, + DefaultInterval: srv.cfg.DefaultRuleEvaluationInterval, RecordingRules: prom.RulesConfig{ IsPaused: pauseRecordingRules, }, diff --git a/pkg/services/ngalert/api/api_convert_prometheus_test.go b/pkg/services/ngalert/api/api_convert_prometheus_test.go index fda8be8cecd..437ec3ffef3 100644 --- a/pkg/services/ngalert/api/api_convert_prometheus_test.go +++ b/pkg/services/ngalert/api/api_convert_prometheus_test.go @@ -381,6 +381,53 @@ func TestRouteConvertPrometheusPostRuleGroup(t *testing.T) { require.Nil(t, r.Metadata.PrometheusStyleRule) } }) + + t.Run("returns error when target datasource does not exist", func(t *testing.T) { + srv, _, _, _ := createConvertPrometheusSrv(t) + rc := createRequestCtx() + rc.Req.Header.Set(targetDatasourceUIDHeader, "some-data-source") + + response := srv.RouteConvertPrometheusPostRuleGroup(rc, "test", simpleGroup) + require.Equal(t, http.StatusNotFound, response.Status()) + require.Contains(t, string(response.Body()), "failed to get recording rules target datasource") + }) + + t.Run("uses target datasource for recording rules", func(t *testing.T) { + srv, dsCache, ruleStore, _ := createConvertPrometheusSrv(t) + rc := createRequestCtx() + targetDSUID := util.GenerateShortUID() + ds := &datasources.DataSource{ + UID: targetDSUID, + Type: datasources.DS_PROMETHEUS, + } + dsCache.DataSources = append(dsCache.DataSources, ds) + rc.Req.Header.Set(targetDatasourceUIDHeader, targetDSUID) + + simpleGroup := apimodels.PrometheusRuleGroup{ + Name: "Test Group", + Interval: prommodel.Duration(1 * time.Minute), + Rules: []apimodels.PrometheusRule{ + { + Record: "recorded-metric", + Expr: "vector(1)", + Labels: map[string]string{ + "severity": "warning", + }, + }, + }, + } + + response := srv.RouteConvertPrometheusPostRuleGroup(rc, "test", simpleGroup) + require.Equal(t, http.StatusAccepted, response.Status()) + + remaining, err := ruleStore.ListAlertRules(context.Background(), &models.ListAlertRulesQuery{ + OrgID: 1, + }) + require.NoError(t, err) + require.Len(t, remaining, 1) + require.NotNil(t, remaining[0].Record) + require.Equal(t, targetDSUID, remaining[0].Record.TargetDatasourceUID) + }) } func TestRouteConvertPrometheusGetRuleGroup(t *testing.T) { @@ -976,7 +1023,7 @@ func withFeatureToggles(toggles featuremgmt.FeatureToggles) convertPrometheusSrv } } -func createConvertPrometheusSrv(t *testing.T, opts ...convertPrometheusSrvOptionsFunc) (*ConvertPrometheusSrv, datasources.CacheService, *fakes.RuleStore, *foldertest.FakeService) { +func createConvertPrometheusSrv(t *testing.T, opts ...convertPrometheusSrvOptionsFunc) (*ConvertPrometheusSrv, *dsfakes.FakeCacheService, *fakes.RuleStore, *foldertest.FakeService) { t.Helper() // By default the quota checker will allow the operation diff --git a/pkg/services/ngalert/prom/convert.go b/pkg/services/ngalert/prom/convert.go index f64b17bfc5e..483ec67929e 100644 --- a/pkg/services/ngalert/prom/convert.go +++ b/pkg/services/ngalert/prom/convert.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "gopkg.in/yaml.v3" + "github.com/grafana/grafana/pkg/apimachinery/errutil" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/util" @@ -18,18 +19,25 @@ const ( // alert rule when converting it to a Grafana alert rule. If this label is not present, // a stable UID will be generated automatically based on the rule's data. ruleUIDLabel = "__grafana_alert_rule_uid__" -) -const ( queryRefID = "query" prometheusMathRefID = "prometheus_math" thresholdRefID = "threshold" ) +var ( + ErrInvalidDatasourceType = errutil.ValidationFailed("alerting.invalidDatasourceType") +) + // Config defines the configuration options for the Prometheus to Grafana rules converter. type Config struct { + // DataSourceUID is the UID of the datasource the rules are querying. DatasourceUID string DatasourceType string + // TargetDatasourceUID is the UID of the datasource the recording rules are writing to. + // If not set, it defaults to DataSourceUID. + TargetDatasourceUID string + TargetDatasourceType string // DefaultInterval is the default interval for rules in the groups that // don't have Interval set. DefaultInterval time.Duration @@ -74,6 +82,10 @@ func NewConverter(cfg Config) (*Converter, error) { if cfg.DatasourceUID == "" { return nil, fmt.Errorf("datasource UID is required") } + if cfg.TargetDatasourceUID == "" { + cfg.TargetDatasourceUID = cfg.DatasourceUID + cfg.TargetDatasourceType = cfg.DatasourceType + } if cfg.DatasourceType == "" { return nil, fmt.Errorf("datasource type is required") } @@ -96,7 +108,10 @@ func NewConverter(cfg Config) (*Converter, error) { cfg.KeepOriginalRuleDefinition = defaultConfig.KeepOriginalRuleDefinition } if cfg.DatasourceType != datasources.DS_PROMETHEUS && cfg.DatasourceType != datasources.DS_LOKI { - return nil, fmt.Errorf("invalid datasource type: %s", cfg.DatasourceType) + return nil, ErrInvalidDatasourceType.Errorf("invalid datasource type: %s, must be prometheus or loki", cfg.DatasourceType) + } + if cfg.TargetDatasourceType != datasources.DS_PROMETHEUS { + return nil, ErrInvalidDatasourceType.Errorf("invalid target datasource type: %s, must be prometheus", cfg.TargetDatasourceType) } return &Converter{ @@ -200,7 +215,7 @@ func (p *Converter) convertRule(orgID int64, namespaceUID string, promGroup Prom record = &models.Record{ From: queryRefID, Metric: rule.Record, - TargetDatasourceUID: p.cfg.DatasourceUID, + TargetDatasourceUID: p.cfg.TargetDatasourceUID, } isPaused = p.cfg.RecordingRules.IsPaused diff --git a/pkg/services/ngalert/prom/convert_test.go b/pkg/services/ngalert/prom/convert_test.go index 2281a9da1c0..5f039e69831 100644 --- a/pkg/services/ngalert/prom/convert_test.go +++ b/pkg/services/ngalert/prom/convert_test.go @@ -103,6 +103,26 @@ func TestPrometheusRulesToGrafana(t *testing.T) { }, expectError: false, }, + { + name: "recording rule with target datasource", + orgID: 1, + namespace: "namespaceUID", + promGroup: PrometheusRuleGroup{ + Name: "test-group-1", + Interval: prommodel.Duration(10 * time.Second), + Rules: []PrometheusRule{ + { + Record: "some_metric", + Expr: "sum(rate(http_requests_total[5m]))", + }, + }, + }, + config: Config{ + TargetDatasourceUID: "target-datasource-uid", + TargetDatasourceType: datasources.DS_PROMETHEUS, + }, + expectError: false, + }, { name: "rule group with query_offset is not supported", orgID: 1, @@ -198,7 +218,12 @@ func TestPrometheusRulesToGrafana(t *testing.T) { require.NotNil(t, grafanaRule.Record) require.Equal(t, grafanaRule.Record.From, queryRefID) require.Equal(t, promRule.Record, grafanaRule.Record.Metric) - require.Equal(t, tc.config.DatasourceUID, grafanaRule.Record.TargetDatasourceUID) + + targetDatasourceUID := tc.config.TargetDatasourceUID + if targetDatasourceUID == "" { + targetDatasourceUID = tc.config.DatasourceUID + } + require.Equal(t, targetDatasourceUID, grafanaRule.Record.TargetDatasourceUID) } else { require.Equal(t, fmt.Sprintf("[%s] %s", tc.promGroup.Name, promRule.Alert), grafanaRule.Title) }