public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Allow more characters in label names so notifications are sent (#38629)

Browse Source 
Remove validation for labels to be accepted in the Alertmanager, This helps with datasources that produce non-compatible labels.

Adds an "object_matchers" to alert manager routers so we can support labels names with extended characters beyond prometheus/openmetrics. It only does this for the internal Grafana managed Alert Manager.

This requires a change to alert manager, so for now we use grafana/alertmanager which is a slight fork, with the intention of going back to upstream.

The frontend handles the migration of "matchers" -> "object_matchers" when the route is edited and saved. Once this is done, downgrades will not work old versions will not recognize the "object_matchers".

Co-authored-by: Kyle Brandt <kyle@grafana.com>
Co-authored-by: Nathan Rodman <nathanrodman@gmail.com>
This commit is contained in:
gotjosh
2021-10-04 15:06:40 +02:00
committed by GitHub
co-authored by Kyle Brandt Nathan Rodman
parent 706a665240
commit 6572017ec7
16 changed files with 740 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/api/tooling/definitions/alertmanager.go
+207 -4
View File
@@ -5,12 +5,15 @@ import (
"encoding/json"
"fmt"
"reflect"
"sort"
"time"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
amv2 "github.com/prometheus/alertmanager/api/v2/models"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/common/model"
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -214,7 +217,7 @@ func (s *GettableStatus) UnmarshalJSON(b []byte) error {
s.Cluster = amStatus.Cluster
s.Config = &PostableApiAlertingConfig{Config: Config{
Global: c.Global,
Route: c.Route,
Route: AsGrafanaRoute(c.Route),
InhibitRules: c.InhibitRules,
Templates: c.Templates,
}}
@@ -556,7 +559,7 @@ func (c *GettableApiAlertingConfig) validate() error {
return fmt.Errorf("cannot mix Alertmanager & Grafana receiver types")
}
for _, receiver := range AllReceivers(c.Route) {
for _, receiver := range AllReceivers(c.Route.AsAMRoute()) {
_, ok := receivers[receiver]
if !ok {
return fmt.Errorf("unexpected receiver (%s) is undefined", receiver)
@@ -569,11 +572,124 @@ func (c *GettableApiAlertingConfig) validate() error {
// Config is the top-level configuration for Alertmanager's config files.
type Config struct {
Global *config.GlobalConfig `yaml:"global,omitempty" json:"global,omitempty"`
Route *config.Route `yaml:"route,omitempty" json:"route,omitempty"`
Route *Route `yaml:"route,omitempty" json:"route,omitempty"`
InhibitRules []*config.InhibitRule `yaml:"inhibit_rules,omitempty" json:"inhibit_rules,omitempty"`
Templates []string `yaml:"templates" json:"templates"`
}
// A Route is a node that contains definitions of how to handle alerts. This is modified
// from the upstream alertmanager in that it adds the ObjectMatchers property.
type Route struct {
Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"`
GroupByStr []string `yaml:"group_by,omitempty" json:"group_by,omitempty"`
GroupBy []model.LabelName `yaml:"-" json:"-"`
GroupByAll bool `yaml:"-" json:"-"`
// Deprecated. Remove before v1.0 release.
Match map[string]string `yaml:"match,omitempty" json:"match,omitempty"`
// Deprecated. Remove before v1.0 release.
MatchRE config.MatchRegexps `yaml:"match_re,omitempty" json:"match_re,omitempty"`
Matchers config.Matchers `yaml:"matchers,omitempty" json:"matchers,omitempty"`
ObjectMatchers ObjectMatchers `yaml:"object_matchers,omitempty" json:"object_matchers,omitempty"`
MuteTimeIntervals []string `yaml:"mute_time_intervals,omitempty" json:"mute_time_intervals,omitempty"`
Continue bool `yaml:"continue" json:"continue,omitempty"`
Routes []*Route `yaml:"routes,omitempty" json:"routes,omitempty"`
GroupWait *model.Duration `yaml:"group_wait,omitempty" json:"group_wait,omitempty"`
GroupInterval *model.Duration `yaml:"group_interval,omitempty" json:"group_interval,omitempty"`
RepeatInterval *model.Duration `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface for Route. This is a copy of alertmanager's upstream except it removes validation on the label key.
func (r *Route) UnmarshalYAML(unmarshal func(interface{}) error) error {
type plain Route
if err := unmarshal((*plain)(r)); err != nil {
return err
}
for _, l := range r.GroupByStr {
if l == "..." {
r.GroupByAll = true
} else {
r.GroupBy = append(r.GroupBy, model.LabelName(l))
}
}
if len(r.GroupBy) > 0 && r.GroupByAll {
return fmt.Errorf("cannot have wildcard group_by (`...`) and other other labels at the same time")
}
groupBy := map[model.LabelName]struct{}{}
for _, ln := range r.GroupBy {
if _, ok := groupBy[ln]; ok {
return fmt.Errorf("duplicated label %q in group_by", ln)
}
groupBy[ln] = struct{}{}
}
if r.GroupInterval != nil && time.Duration(*r.GroupInterval) == time.Duration(0) {
return fmt.Errorf("group_interval cannot be zero")
}
if r.RepeatInterval != nil && time.Duration(*r.RepeatInterval) == time.Duration(0) {
return fmt.Errorf("repeat_interval cannot be zero")
}
return nil
}
// Return an alertmanager route from a Grafana route. The ObjectMatchers are converted to Matchers.
func (r *Route) AsAMRoute() *config.Route {
amRoute := &config.Route{
Receiver: r.Receiver,
GroupByStr: r.GroupByStr,
GroupBy: r.GroupBy,
GroupByAll: r.GroupByAll,
Match: r.Match,
MatchRE: r.MatchRE,
Matchers: append(r.Matchers, r.ObjectMatchers...),
MuteTimeIntervals: r.MuteTimeIntervals,
Continue: r.Continue,
GroupWait: r.GroupWait,
GroupInterval: r.GroupInterval,
RepeatInterval: r.RepeatInterval,
Routes: make([]*config.Route, 0, len(r.Routes)),
}
for _, rt := range r.Routes {
amRoute.Routes = append(amRoute.Routes, rt.AsAMRoute())
}
return amRoute
}
// Return a Grafana route from an alertmanager route. The Matchers are converted to ObjectMatchers.
func AsGrafanaRoute(r *config.Route) *Route {
gRoute := &Route{
Receiver: r.Receiver,
GroupByStr: r.GroupByStr,
GroupBy: r.GroupBy,
GroupByAll: r.GroupByAll,
Match: r.Match,
MatchRE: r.MatchRE,
ObjectMatchers: ObjectMatchers(r.Matchers),
MuteTimeIntervals: r.MuteTimeIntervals,
Continue: r.Continue,
GroupWait: r.GroupWait,
GroupInterval: r.GroupInterval,
RepeatInterval: r.RepeatInterval,
Routes: make([]*Route, 0, len(r.Routes)),
}
for _, rt := range r.Routes {
gRoute.Routes = append(gRoute.Routes, AsGrafanaRoute(rt))
}
return gRoute
}
// Config is the entrypoint for the embedded Alertmanager config with the exception of receivers.
// Prometheus historically uses yaml files as the method of configuration and thus some
// post-validation is included in the UnmarshalYAML method. Here we simply run this with
@@ -686,7 +802,7 @@ func (c *PostableApiAlertingConfig) validate() error {
}
}
for _, receiver := range AllReceivers(c.Route) {
for _, receiver := range AllReceivers(c.Route.AsAMRoute()) {
_, ok := receivers[receiver]
if !ok {
return fmt.Errorf("unexpected receiver (%s) is undefined", receiver)
@@ -972,3 +1088,90 @@ func processReceiverConfigs(c []*PostableApiReceiver) error {
}
return nil
}
// ObjectMatchers is Matchers with a different Unmarshal and Marshal methods that accept matchers as objects
// that have already been parsed.
type ObjectMatchers labels.Matchers
// UnmarshalYAML implements the yaml.Unmarshaler interface for Matchers.
func (m *ObjectMatchers) UnmarshalYAML(unmarshal func(interface{}) error) error {
var rawMatchers [][3]string
if err := unmarshal(&rawMatchers); err != nil {
return err
}
for _, rawMatcher := range rawMatchers {
var matchType labels.MatchType
switch rawMatcher[1] {
case "=":
matchType = labels.MatchEqual
case "!=":
matchType = labels.MatchNotEqual
case "=~":
matchType = labels.MatchRegexp
case "!~":
matchType = labels.MatchNotRegexp
default:
return fmt.Errorf("unsupported match type %q in matcher", rawMatcher[1])
}
matcher, err := labels.NewMatcher(matchType, rawMatcher[0], rawMatcher[2])
if err != nil {
return err
}
*m = append(*m, matcher)
}
sort.Sort(labels.Matchers(*m))
return nil
}
// UnmarshalJSON implements the json.Unmarshaler interface for Matchers.
func (m *ObjectMatchers) UnmarshalJSON(data []byte) error {
var rawMatchers [][3]string
if err := json.Unmarshal(data, &rawMatchers); err != nil {
return err
}
for _, rawMatcher := range rawMatchers {
var matchType labels.MatchType
switch rawMatcher[1] {
case "=":
matchType = labels.MatchEqual
case "!=":
matchType = labels.MatchNotEqual
case "=~":
matchType = labels.MatchRegexp
case "!~":
matchType = labels.MatchNotRegexp
default:
return fmt.Errorf("unsupported match type %q in matcher", rawMatcher[1])
}
matcher, err := labels.NewMatcher(matchType, rawMatcher[0], rawMatcher[2])
if err != nil {
return err
}
*m = append(*m, matcher)
}
sort.Sort(labels.Matchers(*m))
return nil
}
// MarshalYAML implements the yaml.Marshaler interface for Matchers.
func (m ObjectMatchers) MarshalYAML() (interface{}, error) {
result := make([][3]string, len(m))
for i, matcher := range m {
result[i] = [3]string{matcher.Name, matcher.Type.String(), matcher.Value}
}
return result, nil
}
// MarshalJSON implements the json.Marshaler interface for Matchers.
func (m ObjectMatchers) MarshalJSON() ([]byte, error) {
if len(m) == 0 {
return nil, nil
}
result := make([][3]string, len(m))
for i, matcher := range m {
result[i] = [3]string{matcher.Name, matcher.Type.String(), matcher.Value}
}
return json.Marshal(result)
}
pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go
+22 -21
View File
@@ -115,12 +115,12 @@ func Test_APIReceiverType(t *testing.T) {
}
func Test_AllReceivers(t *testing.T) {
input := &config.Route{
input := &Route{
Receiver: "foo",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "bar",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "bazz",
},
@@ -132,11 +132,12 @@ func Test_AllReceivers(t *testing.T) {
},
}
require.Equal(t, []string{"foo", "bar", "bazz", "buzz"}, AllReceivers(input))
require.Equal(t, []string{"foo", "bar", "bazz", "buzz"}, AllReceivers(input.AsAMRoute()))
// test empty
var empty []string
require.Equal(t, empty, AllReceivers(&config.Route{}))
emptyRoute := &Route{}
require.Equal(t, empty, AllReceivers(emptyRoute.AsAMRoute()))
}
func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
@@ -149,9 +150,9 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
desc: "success am",
input: PostableApiAlertingConfig{
Config: Config{
Route: &config.Route{
Route: &Route{
Receiver: "am",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "am",
},
@@ -172,9 +173,9 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
desc: "success graf",
input: PostableApiAlertingConfig{
Config: Config{
Route: &config.Route{
Route: &Route{
Receiver: "graf",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "graf",
},
@@ -197,9 +198,9 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
desc: "failure undefined am receiver",
input: PostableApiAlertingConfig{
Config: Config{
Route: &config.Route{
Route: &Route{
Receiver: "am",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "unmentioned",
},
@@ -221,9 +222,9 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
desc: "failure undefined graf receiver",
input: PostableApiAlertingConfig{
Config: Config{
Route: &config.Route{
Route: &Route{
Receiver: "graf",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "unmentioned",
},
@@ -263,8 +264,8 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
desc: "failure graf no default receiver",
input: PostableApiAlertingConfig{
Config: Config{
Route: &config.Route{
Routes: []*config.Route{
Route: &Route{
Routes: []*Route{
{
Receiver: "graf",
},
@@ -288,9 +289,9 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
desc: "failure graf root route with matchers",
input: PostableApiAlertingConfig{
Config: Config{
Route: &config.Route{
Route: &Route{
Receiver: "graf",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "graf",
},
@@ -315,9 +316,9 @@ func Test_ApiAlertingConfig_Marshaling(t *testing.T) {
desc: "failure graf nested route duplicate group by labels",
input: PostableApiAlertingConfig{
Config: Config{
Route: &config.Route{
Route: &Route{
Receiver: "graf",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "graf",
GroupByStr: []string{"foo", "bar", "foo"},
@@ -481,9 +482,9 @@ alertmanager_config: |
AlertmanagerConfig: GettableApiAlertingConfig{
Config: Config{
Templates: []string{},
Route: &config.Route{
Route: &Route{
Receiver: "am",
Routes: []*config.Route{
Routes: []*Route{
{
Receiver: "am",
},
pkg/services/ngalert/api/tooling/definitions/prom.go
+6 -6
View File
@@ -88,7 +88,7 @@ type AlertingRule struct {
Query string `json:"query,omitempty"`
Duration float64 `json:"duration,omitempty"`
// required: true
Annotations labels `json:"annotations,omitempty"`
Annotations overrideLabels `json:"annotations,omitempty"`
// required: true
Alerts []*Alert `json:"alerts,omitempty"`
Rule
@@ -100,8 +100,8 @@ type Rule struct {
// required: true
Name string `json:"name"`
// required: true
Query string `json:"query"`
Labels labels `json:"labels"`
Query string `json:"query"`
Labels overrideLabels `json:"labels"`
// required: true
Health string `json:"health"`
LastError string `json:"lastError"`
@@ -115,9 +115,9 @@ type Rule struct {
// swagger:model
type Alert struct {
// required: true
Labels labels `json:"labels"`
Labels overrideLabels `json:"labels"`
// required: true
Annotations labels `json:"annotations"`
Annotations overrideLabels `json:"annotations"`
// required: true
State string `json:"state"`
ActiveAt *time.Time `json:"activeAt"`
@@ -127,4 +127,4 @@ type Alert struct {
// override the labels type with a map for generation.
// The custom marshaling for labels.Labels ends up doing this anyways.
type labels map[string]string
type overrideLabels map[string]string