Alerting: Allow disabling provenance in the Prometheus conversion API (#101573)

When creating Grafana-managed alerts from Prometheus rule definitions with mimirtool or cortextool, the rules are marked as "provisioned" and are not editable in the Grafana UI. This PR allows changing this by providing an extra header: --extra-header="X-Disable-Provenance=true".

When provenance is disabled, we do not keep the original rule definition in YAML, so it is impossible to read it back using the Prometheus conversion API (mimirtool/cortextool). This is intentional because if we did keep it and the rule was later changed in the UI, its Prometheus YAML definition would no longer reflect the latest version of the alert rule, as it would be unchanged.

pkg/services/ngalert/api/api_convert_prometheus.go

@@ -188,11 +188,12 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusDeleteNamespace(c *contex
 	}
 	logger.Info("Deleting all Prometheus-imported rule groups", "folder_uid", namespace.UID, "folder_title", namespaceTitle)
 
+	provenance := getProvenance(c)
 	filterOpts := &provisioning.FilterOptions{
 		NamespaceUIDs:          []string{namespace.UID},
 		ImportedPrometheusRule: util.Pointer(true),
 	}
-	err = srv.alertRuleService.DeleteRuleGroups(c.Req.Context(), c.SignedInUser, models.ProvenanceConvertedPrometheus, filterOpts)
+	err = srv.alertRuleService.DeleteRuleGroups(c.Req.Context(), c.SignedInUser, provenance, filterOpts)
 	if errors.Is(err, models.ErrAlertRuleGroupNotFound) {
 		return response.Empty(http.StatusNotFound)
 	}
@@ -218,7 +219,8 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusDeleteRuleGroup(c *contex
 	}
 	logger.Info("Deleting Prometheus-imported rule group", "folder_uid", folder.UID, "folder_title", namespaceTitle, "group", group)
 
-	err = srv.alertRuleService.DeleteRuleGroup(c.Req.Context(), c.SignedInUser, folder.UID, group, models.ProvenanceConvertedPrometheus)
+	provenance := getProvenance(c)
+	err = srv.alertRuleService.DeleteRuleGroup(c.Req.Context(), c.SignedInUser, folder.UID, group, provenance)
 	if errors.Is(err, models.ErrAlertRuleGroupNotFound) {
 		return response.Empty(http.StatusNotFound)
 	}
@@ -352,13 +354,21 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroup(c *contextm
 		return errorToResponse(err)
 	}
 
-	group, err := srv.convertToGrafanaRuleGroup(c, ds, ns.UID, promGroup, logger)
+	provenance := getProvenance(c)
+
+	// If the provenance is not ConvertedPrometheus, we don't keep the original rule definition.
+	// This is because the rules can be modified through the UI, which may break compatibility
+	// with the Prometheus format. We only preserve the original rule definition
+	// to ensure we can return them in this API in Prometheus format.
+	keepOriginalRuleDefinition := provenance == models.ProvenanceConvertedPrometheus
+
+	group, err := srv.convertToGrafanaRuleGroup(c, ds, ns.UID, promGroup, keepOriginalRuleDefinition, logger)
 	if err != nil {
 		logger.Error("Failed to convert Prometheus rules to Grafana rules", "error", err)
 		return errorToResponse(err)
 	}
 
-	err = srv.alertRuleService.ReplaceRuleGroup(c.Req.Context(), c.SignedInUser, *group, models.ProvenanceConvertedPrometheus)
+	err = srv.alertRuleService.ReplaceRuleGroup(c.Req.Context(), c.SignedInUser, *group, provenance)
 	if err != nil {
 		logger.Error("Failed to replace rule group", "error", err)
 		return errorToResponse(err)
@@ -387,7 +397,14 @@ func (srv *ConvertPrometheusSrv) getOrCreateNamespace(c *contextmodel.ReqContext
 	return ns, nil
 }
 
-func (srv *ConvertPrometheusSrv) convertToGrafanaRuleGroup(c *contextmodel.ReqContext, ds *datasources.DataSource, namespaceUID string, promGroup apimodels.PrometheusRuleGroup, logger log.Logger) (*models.AlertRuleGroup, error) {
+func (srv *ConvertPrometheusSrv) convertToGrafanaRuleGroup(
+	c *contextmodel.ReqContext,
+	ds *datasources.DataSource,
+	namespaceUID string,
+	promGroup apimodels.PrometheusRuleGroup,
+	keepOriginalRuleDefinition bool,
+	logger log.Logger,
+) (*models.AlertRuleGroup, error) {
 	logger.Info("Converting Prometheus rules to Grafana rules", "rules", len(promGroup.Rules), "folder_uid", namespaceUID, "datasource_uid", ds.UID, "datasource_type", ds.Type)
 
 	rules := make([]prom.PrometheusRule, len(promGroup.Rules))
@@ -429,6 +446,7 @@ func (srv *ConvertPrometheusSrv) convertToGrafanaRuleGroup(c *contextmodel.ReqCo
 			AlertRules: prom.RulesConfig{
 				IsPaused: pauseAlertRules,
 			},
+			KeepOriginalRuleDefinition: util.Pointer(keepOriginalRuleDefinition),
 		},
 	)
 	if err != nil {
@@ -537,3 +555,13 @@ func promGroupHasRecordingRules(promGroup apimodels.PrometheusRuleGroup) bool {
 	}
 	return false
 }
+
+// getProvenance determines the provenance value to use for rules created via the Prometheus conversion API.
+// If the X-Disable-Provenance header is present in the request, returns ProvenanceNone,
+// otherwise returns ProvenanceConvertedPrometheus.
+func getProvenance(ctx *contextmodel.ReqContext) models.Provenance {
+	if _, disabled := ctx.Req.Header[disableProvenanceHeaderName]; disabled {
+		return models.ProvenanceNone
+	}
+	return models.ProvenanceConvertedPrometheus
+}

pkg/services/ngalert/api/api_convert_prometheus_test.go

@@ -144,6 +144,11 @@ func TestRouteConvertPrometheusPostRuleGroup(t *testing.T) {
 			promDefinition, err := r.PrometheusRuleDefinition()
 			require.NoError(t, err)
 			require.Equal(t, expectedDef, promDefinition)
+
+			// Verify provenance was set to ProvenanceConvertedPrometheus
+			prov, err := provenanceStore.GetProvenance(context.Background(), r, 1)
+			require.NoError(t, err)
+			require.Equal(t, models.ProvenanceConvertedPrometheus, prov)
 		}
 	})
 
@@ -341,6 +346,41 @@ func TestRouteConvertPrometheusPostRuleGroup(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("with disable provenance header should use ProvenanceNone", func(t *testing.T) {
+		provenanceStore := fakes.NewFakeProvisioningStore()
+		srv, _, ruleStore, folderService := createConvertPrometheusSrv(t, withProvenanceStore(provenanceStore))
+
+		// Create a folder in the root
+		fldr := randFolder()
+		fldr.ParentUID = ""
+		folderService.ExpectedFolder = fldr
+		folderService.ExpectedFolders = []*folder.Folder{fldr}
+		ruleStore.Folders[1] = append(ruleStore.Folders[1], fldr)
+
+		// Create request with the X-Disable-Provenance header
+		rc := createRequestCtx()
+		rc.Req.Header.Set("X-Disable-Provenance", "true")
+
+		response := srv.RouteConvertPrometheusPostRuleGroup(rc, fldr.Title, simpleGroup)
+		require.Equal(t, http.StatusAccepted, response.Status())
+
+		// Get the created rules
+		rules, err := ruleStore.ListAlertRules(context.Background(), &models.ListAlertRulesQuery{
+			OrgID: 1,
+		})
+		require.NoError(t, err)
+		require.Len(t, rules, 2)
+
+		// Verify provenance was set to ProvenanceNone
+		for _, r := range rules {
+			prov, err := provenanceStore.GetProvenance(context.Background(), r, 1)
+			require.NoError(t, err)
+			require.Equal(t, models.ProvenanceNone, prov, "Provenance should be ProvenanceNone when X-Disable-Provenance header is set")
+			// Prometheus rule definition should not be saved when provenance is disabled
+			require.Nil(t, r.Metadata.PrometheusStyleRule)
+		}
+	})
 }
 
 func TestRouteConvertPrometheusGetRuleGroup(t *testing.T) {
@@ -743,6 +783,29 @@ func TestRouteConvertPrometheusDeleteNamespace(t *testing.T) {
 			require.NoError(t, err)
 			require.NotNil(t, remaining)
 		})
+
+		t.Run("with disable provenance header should still be able to delete rules", func(t *testing.T) {
+			provenanceStore := fakes.NewFakeProvisioningStore()
+			srv, ruleStore, fldr, rule := initNamespace("prometheus definition", withProvenanceStore(provenanceStore))
+
+			// Mark the rule as provisioned with API provenance
+			err := provenanceStore.SetProvenance(context.Background(), rule, 1, models.ProvenanceConvertedPrometheus)
+			require.NoError(t, err)
+
+			rc := createRequestCtx()
+			rc.Req.Header.Set("X-Disable-Provenance", "true")
+
+			response := srv.RouteConvertPrometheusDeleteNamespace(rc, fldr.Title)
+			require.Equal(t, http.StatusAccepted, response.Status())
+
+			// Verify the rule was deleted
+			remaining, err := ruleStore.GetAlertRuleByUID(context.Background(), &models.GetAlertRuleByUIDQuery{
+				UID:   rule.UID,
+				OrgID: rule.OrgID,
+			})
+			require.Error(t, err)
+			require.Nil(t, remaining)
+		})
 	})
 }
 
@@ -854,6 +917,29 @@ func TestRouteConvertPrometheusDeleteRuleGroup(t *testing.T) {
 			require.NoError(t, err)
 			require.NotNil(t, remaining)
 		})
+
+		t.Run("with disable provenance header should still be able to delete rules", func(t *testing.T) {
+			provenanceStore := fakes.NewFakeProvisioningStore()
+			srv, ruleStore, fldr, rule := initGroup("", groupName, withProvenanceStore(provenanceStore))
+
+			// Mark the rule as provisioned with API provenance
+			err := provenanceStore.SetProvenance(context.Background(), rule, 1, models.ProvenanceConvertedPrometheus)
+			require.NoError(t, err)
+
+			rc := createRequestCtx()
+			rc.Req.Header.Set("X-Disable-Provenance", "true")
+
+			response := srv.RouteConvertPrometheusDeleteRuleGroup(rc, fldr.Title, groupName)
+			require.Equal(t, http.StatusAccepted, response.Status())
+
+			// Verify the rule was deleted
+			remaining, err := ruleStore.GetAlertRuleByUID(context.Background(), &models.GetAlertRuleByUIDQuery{
+				UID:   rule.UID,
+				OrgID: rule.OrgID,
+			})
+			require.Error(t, err)
+			require.Nil(t, remaining)
+		})
 	})
 }
 
@@ -995,3 +1081,32 @@ func TestGetWorkingFolderUID(t *testing.T) {
 		require.Equal(t, specifiedFolderUID, folderUID)
 	})
 }
+
+func TestGetProvenance(t *testing.T) {
+	t.Run("should return ProvenanceConvertedPrometheus when header is not present", func(t *testing.T) {
+		rc := createRequestCtx()
+		// Ensure the header is not present
+		rc.Req.Header.Del(disableProvenanceHeaderName)
+
+		provenance := getProvenance(rc)
+		require.Equal(t, models.ProvenanceConvertedPrometheus, provenance)
+	})
+
+	t.Run("should return ProvenanceNone when header is present", func(t *testing.T) {
+		rc := createRequestCtx()
+		// Set the disable provenance header
+		rc.Req.Header.Set(disableProvenanceHeaderName, "true")
+
+		provenance := getProvenance(rc)
+		require.Equal(t, models.ProvenanceNone, provenance)
+	})
+
+	t.Run("should return ProvenanceNone when header is present with any value", func(t *testing.T) {
+		rc := createRequestCtx()
+		// Set the disable provenance header with an empty value
+		rc.Req.Header.Set(disableProvenanceHeaderName, "")
+
+		provenance := getProvenance(rc)
+		require.Equal(t, models.ProvenanceNone, provenance)
+	})
+}

pkg/services/ngalert/prom/convert.go

@@ -37,8 +37,12 @@ type Config struct {
 	EvaluationOffset *time.Duration
 	ExecErrState     models.ExecutionErrorState
 	NoDataState      models.NoDataState
-	RecordingRules   RulesConfig
-	AlertRules       RulesConfig
+	// KeepOriginalRuleDefinition indicates whether the original Prometheus rule definition
+	// if saved to the alert rule metadata. If not, then it will not be possible to convert
+	// the alert rule back to Prometheus format.
+	KeepOriginalRuleDefinition *bool
+	RecordingRules             RulesConfig
+	AlertRules                 RulesConfig
 }
 
 // RulesConfig contains configuration that applies to either recording or alerting rules.
@@ -51,10 +55,11 @@ var (
 	defaultEvaluationOffset = 0 * time.Minute
 
 	defaultConfig = Config{
-		FromTimeRange:    &defaultTimeRange,
-		EvaluationOffset: &defaultEvaluationOffset,
-		ExecErrState:     models.ErrorErrState,
-		NoDataState:      models.OK,
+		FromTimeRange:              &defaultTimeRange,
+		EvaluationOffset:           &defaultEvaluationOffset,
+		ExecErrState:               models.ErrorErrState,
+		NoDataState:                models.OK,
+		KeepOriginalRuleDefinition: util.Pointer(true),
 	}
 )
 
@@ -87,7 +92,9 @@ func NewConverter(cfg Config) (*Converter, error) {
 	if cfg.NoDataState == "" {
 		cfg.NoDataState = defaultConfig.NoDataState
 	}
-
+	if cfg.KeepOriginalRuleDefinition == nil {
+		cfg.KeepOriginalRuleDefinition = defaultConfig.KeepOriginalRuleDefinition
+	}
 	if cfg.DatasourceType != datasources.DS_PROMETHEUS && cfg.DatasourceType != datasources.DS_LOKI {
 		return nil, fmt.Errorf("invalid datasource type: %s", cfg.DatasourceType)
 	}
@@ -233,11 +240,12 @@ func (p *Converter) convertRule(orgID int64, namespaceUID string, promGroup Prom
 		RuleGroup:    promGroup.Name,
 		IsPaused:     isPaused,
 		Record:       record,
-		Metadata: models.AlertRuleMetadata{
-			PrometheusStyleRule: &models.PrometheusStyleRule{
-				OriginalRuleDefinition: string(originalRuleDefinition),
-			},
-		},
+	}
+
+	if p.cfg.KeepOriginalRuleDefinition != nil && *p.cfg.KeepOriginalRuleDefinition {
+		result.Metadata.PrometheusStyleRule = &models.PrometheusStyleRule{
+			OriginalRuleDefinition: string(originalRuleDefinition),
+		}
 	}
 
 	return result, nil

pkg/services/ngalert/prom/convert_test.go

@@ -618,3 +618,71 @@ func TestPrometheusRulesToGrafana_UID(t *testing.T) {
 		})
 	})
 }
+
+func TestPrometheusRulesToGrafana_KeepOriginalRuleDefinition(t *testing.T) {
+	orgID := int64(1)
+	namespace := "namespace"
+
+	promGroup := PrometheusRuleGroup{
+		Name: "test-group",
+		Rules: []PrometheusRule{
+			{
+				Alert: "test-alert",
+				Expr:  "up == 0",
+			},
+		},
+	}
+
+	testCases := []struct {
+		name                       string
+		keepOriginalRuleDefinition *bool
+		expectDefinition           bool
+	}{
+		{
+			name:                       "keep original rule definition is true",
+			keepOriginalRuleDefinition: util.Pointer(true),
+			expectDefinition:           true,
+		},
+		{
+			name:                       "keep original rule definition is false",
+			keepOriginalRuleDefinition: util.Pointer(false),
+			expectDefinition:           false,
+		},
+		{
+			name:                       "keep original rule definition is nil (should use default)",
+			keepOriginalRuleDefinition: nil,
+			expectDefinition:           true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := Config{
+				DatasourceUID:              "datasource-uid",
+				DatasourceType:             datasources.DS_PROMETHEUS,
+				DefaultInterval:            1 * time.Minute,
+				KeepOriginalRuleDefinition: tc.keepOriginalRuleDefinition,
+			}
+
+			converter, err := NewConverter(cfg)
+			require.NoError(t, err)
+
+			// Convert the Prometheus rule to Grafana
+			grafanaGroup, err := converter.PrometheusRulesToGrafana(orgID, namespace, promGroup)
+			require.NoError(t, err)
+			require.Len(t, grafanaGroup.Rules, 1)
+
+			if tc.expectDefinition {
+				originalRuleDefinition, err := yaml.Marshal(promGroup.Rules[0])
+				require.NoError(t, err)
+				require.Equal(
+					t,
+					string(originalRuleDefinition),
+					grafanaGroup.Rules[0].Metadata.PrometheusStyleRule.OriginalRuleDefinition,
+				)
+			} else {
+				require.Nil(t, grafanaGroup.Rules[0].Metadata.PrometheusStyleRule)
+			}
+		})
+	}
+}

pkg/services/ngalert/provisioning/alert_rules_test.go

@@ -674,6 +674,42 @@ func TestAlertRuleService(t *testing.T) {
 				to:     models.ProvenanceNone,
 				errNil: false,
 			},
+			{
+				name:   "should be able to update from provenance none to 'converted prometheus'",
+				from:   models.ProvenanceNone,
+				to:     models.ProvenanceConvertedPrometheus,
+				errNil: true,
+			},
+			{
+				name:   "should be able to update from provenance 'converted prometheus' to none",
+				from:   models.ProvenanceConvertedPrometheus,
+				to:     models.ProvenanceNone,
+				errNil: true,
+			},
+			{
+				name:   "should not be able to update from provenance 'converted prometheus' to api",
+				from:   models.ProvenanceConvertedPrometheus,
+				to:     models.ProvenanceAPI,
+				errNil: false,
+			},
+			{
+				name:   "should not be able to update from provenance 'converted prometheus' to file",
+				from:   models.ProvenanceConvertedPrometheus,
+				to:     models.ProvenanceFile,
+				errNil: false,
+			},
+			{
+				name:   "should not be able to update from provenance api to 'converted prometheus'",
+				from:   models.ProvenanceAPI,
+				to:     models.ProvenanceConvertedPrometheus,
+				errNil: false,
+			},
+			{
+				name:   "should not be able to update from provenance file to 'converted prometheus'",
+				from:   models.ProvenanceFile,
+				to:     models.ProvenanceConvertedPrometheus,
+				errNil: false,
+			},
 		}
 		for _, test := range tests {
 			t.Run(test.name, func(t *testing.T) {

pkg/services/ngalert/provisioning/validation/provenance.go

@@ -7,9 +7,23 @@ import (
 // CanUpdateProvenanceInRuleGroup checks if a provenance can be updated for a rule group and its alerts.
 // ReplaceRuleGroup function intends to replace an entire rule group: inserting, updating, and removing rules.
 func CanUpdateProvenanceInRuleGroup(storedProvenance, provenance models.Provenance) bool {
-	return storedProvenance == provenance ||
-		storedProvenance == models.ProvenanceNone ||
-		(storedProvenance == models.ProvenanceAPI && provenance == models.ProvenanceNone)
+	// Same provenance is always allowed
+	if storedProvenance == provenance {
+		return true
+	}
+
+	// Can always update stored ProvenanceNone
+	if storedProvenance == models.ProvenanceNone {
+		return true
+	}
+
+	// Can reset to ProvenanceNone from specific provenances
+	if provenance == models.ProvenanceNone {
+		return storedProvenance == models.ProvenanceAPI ||
+			storedProvenance == models.ProvenanceConvertedPrometheus
+	}
+
+	return false
 }
 
 type ProvenanceStatusTransitionValidator = func(from, to models.Provenance) error

pkg/services/ngalert/provisioning/validation/provenance_test.go

@@ -15,6 +15,7 @@ func TestValidateProvenanceRelaxed(t *testing.T) {
 		models.ProvenanceNone,
 		models.ProvenanceAPI,
 		models.ProvenanceFile,
+		models.ProvenanceConvertedPrometheus,
 		models.Provenance(fmt.Sprintf("random-%s", util.GenerateShortUID())),
 	}
 	t.Run("all transitions from 'none' are allowed", func(t *testing.T) {
@@ -49,3 +50,62 @@ func TestValidateProvenanceRelaxed(t *testing.T) {
 		}
 	})
 }
+
+func TestCanUpdateProvenanceInRuleGroup(t *testing.T) {
+	all := []models.Provenance{
+		models.ProvenanceNone,
+		models.ProvenanceAPI,
+		models.ProvenanceFile,
+		models.ProvenanceConvertedPrometheus,
+		models.Provenance(fmt.Sprintf("random-%s", util.GenerateShortUID())),
+	}
+
+	t.Run("same provenance transitions are allowed", func(t *testing.T) {
+		for _, provenance := range all {
+			assert.True(t, CanUpdateProvenanceInRuleGroup(provenance, provenance))
+		}
+	})
+
+	t.Run("all transitions from 'none' are allowed", func(t *testing.T) {
+		for _, provenance := range all {
+			assert.True(t, CanUpdateProvenanceInRuleGroup(models.ProvenanceNone, provenance))
+		}
+	})
+
+	t.Run("only specific provenances can transition to 'none'", func(t *testing.T) {
+		allowed := []models.Provenance{
+			models.ProvenanceAPI,
+			models.ProvenanceConvertedPrometheus,
+		}
+
+		for _, from := range allowed {
+			assert.True(t, CanUpdateProvenanceInRuleGroup(from, models.ProvenanceNone),
+				"transition %s -> 'none' should be allowed", from)
+		}
+
+		notAllowed := []models.Provenance{
+			models.ProvenanceFile,
+			models.Provenance(fmt.Sprintf("random-%s", util.GenerateShortUID())),
+		}
+
+		for _, from := range notAllowed {
+			assert.False(t, CanUpdateProvenanceInRuleGroup(from, models.ProvenanceNone),
+				"transition %s -> 'none' should not be allowed", from)
+		}
+	})
+
+	t.Run("transitions between different provenances are not allowed", func(t *testing.T) {
+		for _, from := range all {
+			if from == models.ProvenanceNone {
+				continue // always allowed
+			}
+			for _, to := range all {
+				if from == to || to == models.ProvenanceNone {
+					continue // always allowed
+				}
+				assert.False(t, CanUpdateProvenanceInRuleGroup(from, to),
+					"transition %s -> '%s' should not be allowed", from, to)
+			}
+		}
+	})
+}