diff --git a/pkg/services/ngalert/state/template/funcs.go b/pkg/services/ngalert/state/template/funcs.go index ad300969d2d..34866461010 100644 --- a/pkg/services/ngalert/state/template/funcs.go +++ b/pkg/services/ngalert/state/template/funcs.go @@ -5,7 +5,10 @@ import ( "fmt" "net/url" "regexp" + "strings" "text/template" + + "golang.org/x/exp/slices" ) type query struct { @@ -13,14 +16,25 @@ type query struct { Expr string `json:"expr"` } +const ( + FilterLabelFuncName = "filterLabels" + FilterLabelReFuncName = "filterLabelsRe" + GraphLinkFuncName = "graphLink" + RemoveLabelsFuncName = "removeLabels" + RemoveLabelsReFuncName = "removeLabelsRe" + TableLinkFuncName = "tableLink" + MergeLabelValuesFuncName = "mergeLabelValues" +) + var ( defaultFuncs = template.FuncMap{ - "filterLabels": filterLabelsFunc, - "filterLabelsRe": filterLabelsReFunc, - "graphLink": graphLinkFunc, - "removeLabels": removeLabelsFunc, - "removeLabelslRe": removeLabelsReFunc, - "tableLink": tableLinkFunc, + FilterLabelFuncName: filterLabelsFunc, + FilterLabelReFuncName: filterLabelsReFunc, + GraphLinkFuncName: graphLinkFunc, + RemoveLabelsFuncName: removeLabelsFunc, + RemoveLabelsReFuncName: removeLabelsReFunc, + TableLinkFuncName: tableLinkFunc, + MergeLabelValuesFuncName: mergeLabelValuesFunc, } ) @@ -89,3 +103,32 @@ func tableLinkFunc(data string) string { expr := url.QueryEscape(q.Expr) return fmt.Sprintf(`/explore?left={"datasource":%[1]q,"queries":[{"datasource":%[1]q,"expr":%q,"instant":true,"range":false,"refId":"A"}],"range":{"from":"now-1h","to":"now"}}`, datasource, expr) } + +// mergeLabelValuesFunc returns a map of label keys to deduplicated and comma separated values. +func mergeLabelValuesFunc(values map[string]Value) Labels { + type uniqueLabelVals map[string]struct{} + + labels := make(map[string]uniqueLabelVals) + for _, value := range values { + for k, v := range value.Labels { + var ul uniqueLabelVals + var ok bool + if ul, ok = labels[k]; !ok { + ul = uniqueLabelVals{} + labels[k] = ul + } + ul[v] = struct{}{} + } + } + + res := make(Labels) + for label, vals := range labels { + keys := make([]string, 0, len(vals)) + for val := range vals { + keys = append(keys, val) + } + slices.Sort(keys) + res[label] = strings.Join(keys, ", ") + } + return res +} diff --git a/pkg/services/ngalert/state/template/funcs_test.go b/pkg/services/ngalert/state/template/funcs_test.go index 4164208561f..e002c810ec1 100644 --- a/pkg/services/ngalert/state/template/funcs_test.go +++ b/pkg/services/ngalert/state/template/funcs_test.go @@ -25,3 +25,27 @@ func TestRemoveLabelsReFunc(t *testing.T) { l := Labels{"foo": "bar", "bar": "baz"} assert.Equal(t, Labels{"bar": "baz"}, removeLabelsReFunc(l, "f.*")) } + +func TestDeduplicateLabelsFunc(t *testing.T) { + v := map[string]Value{ + "v1": {Labels: Labels{"foo": "bar", "bar": "foo"}, Value: 1}, + "v2": {Labels: Labels{"foo": "bar", "bar": "baz", "baz": "bat"}, Value: 2}, + } + assert.Equal(t, Labels{"foo": "bar", "bar": "baz, foo", "baz": "bat"}, mergeLabelValuesFunc(v)) +} + +func TestDeduplicateLabelsFuncAllSameVal(t *testing.T) { + v := map[string]Value{ + "v1": {Labels: Labels{"foo": "bar", "bar": "baz"}, Value: 1}, + "v2": {Labels: Labels{"foo": "bar", "bar": "baz"}, Value: 2}, + } + assert.Equal(t, Labels{"foo": "bar", "bar": "baz"}, mergeLabelValuesFunc(v)) +} + +func TestDeduplicateLabelsFuncNoDuplicates(t *testing.T) { + v := map[string]Value{ + "v1": {Labels: Labels{"foo": "bar"}, Value: 1}, + "v2": {Labels: Labels{"bar": "baz"}, Value: 2}, + } + assert.Equal(t, Labels{"foo": "bar", "bar": "baz"}, mergeLabelValuesFunc(v)) +} diff --git a/pkg/services/sqlstore/migrations/ualert/alert_rule.go b/pkg/services/sqlstore/migrations/ualert/alert_rule.go index 8b769494fb6..2039c7e1922 100644 --- a/pkg/services/sqlstore/migrations/ualert/alert_rule.go +++ b/pkg/services/sqlstore/migrations/ualert/alert_rule.go @@ -108,8 +108,9 @@ func addMigrationInfo(da *dashAlert) (map[string]string, map[string]string) { func (m *migration) makeAlertRule(l log.Logger, cond condition, da dashAlert, folderUID string) (*alertRule, error) { lbls, annotations := addMigrationInfo(&da) - annotations["message"] = da.Message - var err error + + message := MigrateTmpl(l.New("field", "message"), da.Message) + annotations["message"] = message data, err := migrateAlertRuleQueries(l, cond.Data) if err != nil { diff --git a/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go b/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go index 1dabf19f2cf..c8f2aff7f4a 100644 --- a/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go +++ b/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go @@ -195,6 +195,20 @@ func TestMakeAlertRule(t *testing.T) { require.Nil(t, err) require.Equal(t, string(models.ErrorErrState), ar.ExecErrState) }) + + t.Run("migrate message template", func(t *testing.T) { + m := newTestMigration(t) + da := createTestDashAlert() + da.Message = "Instance ${instance} is down" + cnd := createTestDashAlertCondition() + + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + require.Nil(t, err) + expected := + "{{- $mergedLabels := mergeLabelValues $values -}}\n" + + "Instance {{$mergedLabels.instance}} is down" + require.Equal(t, expected, ar.Annotations["message"]) + }) } func createTestDashAlert() dashAlert { diff --git a/pkg/services/sqlstore/migrations/ualert/template.go b/pkg/services/sqlstore/migrations/ualert/template.go new file mode 100644 index 00000000000..08eb3dfbc43 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/template.go @@ -0,0 +1,208 @@ +// This file contains code that parses templates from old alerting into a sequence +// of tokens. Each token can be either a string literal or a variable. + +package ualert + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + "unicode" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/ngalert/state/template" +) + +// Token contains either a string literal or a variable. +type Token struct { + Literal string + Variable string +} + +func (t Token) IsLiteral() bool { + return t.Literal != "" +} + +func (t Token) IsVariable() bool { + return t.Variable != "" +} + +func (t Token) String() string { + if t.IsLiteral() { + return t.Literal + } else if t.IsVariable() { + return t.Variable + } else { + panic("empty token") + } +} + +func MigrateTmpl(l log.Logger, oldTmpl string) string { + var newTmpl string + + tokens := tokenizeTmpl(l, oldTmpl) + tokens = escapeLiterals(tokens) + + if anyVariableToken(tokens) { + tokens = variablesToMapLookups(tokens, "mergedLabels") + newTmpl += fmt.Sprintf("{{- $mergedLabels := %s $values -}}\n", template.MergeLabelValuesFuncName) + } + + newTmpl += tokensToTmpl(tokens) + return newTmpl +} + +func tokenizeTmpl(logger log.Logger, tmpl string) []Token { + var ( + tokens []Token + l int + r int + err error + ) + + in := []rune(tmpl) + for r < len(in) { + if !startVariable(in[r:]) { + r++ + continue + } + + token, offset, tokenErr := tokenizeVariable(in[r:]) + if tokenErr != nil { + err = errors.Join(err, tokenErr) + r += offset + continue + } + + // we've found a variable, so everything from l -> r is the literal before the variable + // ex: "foo ${bar}" -> Literal: "foo ", Variable: "bar" + if r > l { + tokens = append(tokens, Token{Literal: string(in[l:r])}) + } + tokens = append(tokens, token) + + // seek l and r past the variable + r += offset + l = r + } + + // any remaining runes will be a final literal + if r > l { + tokens = append(tokens, Token{Literal: string(in[l:r])}) + } + + if err != nil { + logger.Warn("Encountered malformed template", "template", tmpl, "err", err) + } + + return tokens +} + +func tokenizeVariable(in []rune) (Token, int, error) { + var ( + pos int + r rune + runes []rune + ) + + if !startVariable(in) { + panic("tokenizeVariable called with input that doesn't start with delimiter") + } + pos += 2 // seek past opening delimiter + + // consume valid runes until we hit a closing brace + // non-space whitespace and the opening delimiter are invalid + for pos < len(in) { + r = in[pos] + + if unicode.IsSpace(r) && r != ' ' { + return Token{}, pos, fmt.Errorf("unexpected whitespace") + } + + if startVariable(in[pos:]) { + return Token{}, pos, fmt.Errorf("ambiguous delimiter") + } + + if r == '}' { + pos++ + break + } + + runes = append(runes, r) + pos++ + } + + // variable must end with '}' delimiter + if r != '}' { + return Token{}, pos, fmt.Errorf("expected '}', got '%c'", r) + } + + return Token{Variable: string(runes)}, pos, nil +} + +func startVariable(in []rune) bool { + return len(in) >= 2 && in[0] == '$' && in[1] == '{' +} + +func anyVariableToken(tokens []Token) bool { + for _, token := range tokens { + if token.IsVariable() { + return true + } + } + return false +} + +// tokensToTmpl returns the tokens as a Go template +func tokensToTmpl(tokens []Token) string { + buf := bytes.Buffer{} + for _, token := range tokens { + if token.IsVariable() { + buf.WriteString("{{") + buf.WriteString(token.String()) + buf.WriteString("}}") + } else { + buf.WriteString(token.String()) + } + } + return buf.String() +} + +// escapeLiterals escapes any token literals with substrings that would be interpreted as Go template syntax +func escapeLiterals(tokens []Token) []Token { + result := make([]Token, 0, len(tokens)) + for _, token := range tokens { + if token.IsLiteral() && shouldEscape(token.Literal) { + token.Literal = fmt.Sprintf("{{`%s`}}", token.Literal) + } + result = append(result, token) + } + return result +} + +func shouldEscape(literal string) bool { + return strings.Contains(literal, "{{") || literal[len(literal)-1] == '{' +} + +// variablesToMapLookups converts any variables in a slice of tokens to Go template map lookups +func variablesToMapLookups(tokens []Token, mapName string) []Token { + result := make([]Token, 0, len(tokens)) + for _, token := range tokens { + if token.IsVariable() { + token.Variable = mapLookupString(token.Variable, mapName) + } + result = append(result, token) + } + return result +} + +func mapLookupString(v string, mapName string) string { + for _, r := range v { + if !(unicode.IsDigit(r) || unicode.IsLetter(r) || r == '_') { + return fmt.Sprintf(`index $%s %s`, mapName, strconv.Quote(v)) // quote v to escape any special characters + } + } + return fmt.Sprintf(`$%s.%s`, mapName, v) +} diff --git a/pkg/services/sqlstore/migrations/ualert/template_test.go b/pkg/services/sqlstore/migrations/ualert/template_test.go new file mode 100644 index 00000000000..f95fbc757f3 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/template_test.go @@ -0,0 +1,321 @@ +package ualert + +import ( + "fmt" + "testing" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/stretchr/testify/assert" +) + +func TestTokenString(t *testing.T) { + t1 := Token{Literal: "this is a literal"} + assert.Equal(t, "this is a literal", t1.String()) + t2 := Token{Variable: "this is a variable"} + assert.Equal(t, "this is a variable", t2.String()) +} + +func TestTokenizeVariable(t *testing.T) { + tests := []struct { + name string + text string + token Token + offset int + err string + }{{ + name: "variable with no trailing text", + text: "${instance}", + token: Token{Variable: "instance"}, + offset: 11, + }, { + name: "variable with trailing text", + text: "${instance} is down", + token: Token{Variable: "instance"}, + offset: 11, + }, { + name: "varaiable with numbers", + text: "${instance1} is down", + token: Token{Variable: "instance1"}, + offset: 12, + }, { + name: "variable with underscores", + text: "${instance_with_underscores} is down", + token: Token{Variable: "instance_with_underscores"}, + offset: 28, + }, { + name: "variable with spaces", + text: "${instance with spaces} is down", + token: Token{Variable: "instance with spaces"}, + offset: 23, + }, { + name: "variable with non-reserved special character", + text: "${@instance1} is down", + token: Token{Variable: "@instance1"}, + offset: 13, + }, { + name: "two variables without spaces", + text: "${variable1}${variable2}", + token: Token{Variable: "variable1"}, + offset: 12, + }, { + name: "variable with two closing braces stops at first brace", + text: "${instance}} is down", + token: Token{Variable: "instance"}, + offset: 11, + }, { + name: "variable with newline", + text: "${instance\n} is down", + offset: 10, + err: "unexpected whitespace", + }, { + name: "variable with ambiguous delimiter returns error", + text: "${${instance}", + offset: 2, + err: "ambiguous delimiter", + }, { + name: "variable without closing brace returns error", + text: "${instance is down", + offset: 18, + err: "expected '}', got 'n'", + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + token, offset, err := tokenizeVariable([]rune(test.text)) + if test.err != "" { + assert.EqualError(t, err, test.err) + } + assert.Equal(t, test.offset, offset) + assert.Equal(t, test.token, token) + }) + } +} + +func TestTokenizeTmpl(t *testing.T) { + tests := []struct { + name string + tmpl string + tokens []Token + }{{ + name: "simple template can be tokenized", + tmpl: "${instance} is down", + tokens: []Token{{Variable: "instance"}, {Literal: " is down"}}, + }, { + name: "complex template can be tokenized", + tmpl: "More than ${value} ${status_code} in the last 5 minutes", + tokens: []Token{ + {Literal: "More than "}, + {Variable: "value"}, + {Literal: " "}, + {Variable: "status_code"}, + {Literal: " in the last 5 minutes"}, + }, + }, { + name: "variables without spaces between can be tokenized", + tmpl: "${value}${status_code}", + tokens: []Token{{Variable: "value"}, {Variable: "status_code"}}, + }, { + name: "variables without spaces between then literal can be tokenized", + tmpl: "${value}${status_code} in the last 5 minutes", + tokens: []Token{{Variable: "value"}, {Variable: "status_code"}, {Literal: " in the last 5 minutes"}}, + }, { + name: "variables with reserved characters can be tokenized", + tmpl: "More than ${$value} ${{status_code} in the last 5 minutes", + tokens: []Token{ + {Literal: "More than "}, + {Variable: "$value"}, + {Literal: " "}, + {Variable: "{status_code"}, + {Literal: " in the last 5 minutes"}, + }, + }, { + name: "ambiguous delimiters are tokenized as literals", + tmpl: "Instance ${instance and ${instance} is down", + tokens: []Token{{Literal: "Instance ${instance and "}, {Variable: "instance"}, {Literal: " is down"}}, + }, { + name: "all '$' runes preceding a variable are included in literal", + tmpl: "Instance $${instance} is down", + tokens: []Token{{Literal: "Instance $"}, {Variable: "instance"}, {Literal: " is down"}}, + }, { + name: "sole '$' rune is included in literal", + tmpl: "Instance $instance and ${instance} is down", + tokens: []Token{{Literal: "Instance $instance and "}, {Variable: "instance"}, {Literal: " is down"}}, + }, { + name: "extra closing brace is included in literal", + tmpl: "Instance ${instance}} and ${instance} is down", + tokens: []Token{{Literal: "Instance "}, {Variable: "instance"}, {Literal: "} and "}, {Variable: "instance"}, {Literal: " is down"}}, + }, { + name: "variable with newline tokenized as literal", + tmpl: "${value}${status_code\n}${value} in the last 5 minutes", + tokens: []Token{{Variable: "value"}, {Literal: "${status_code\n}"}, {Variable: "value"}, {Literal: " in the last 5 minutes"}}, + }, { + name: "extra closing brace between variables is included in literal", + tmpl: "${value}${status_code}}${value} in the last 5 minutes", + tokens: []Token{{Variable: "value"}, {Variable: "status_code"}, {Literal: "}"}, {Variable: "value"}, {Literal: " in the last 5 minutes"}}, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tokens := tokenizeTmpl(log.NewNopLogger(), test.tmpl) + assert.Equal(t, test.tokens, tokens) + }) + } +} + +func TestTokensToTmpl(t *testing.T) { + tokens := []Token{{Variable: "instance"}, {Literal: " is down"}} + assert.Equal(t, "{{instance}} is down", tokensToTmpl(tokens)) +} + +func TestTokensToTmplNewlines(t *testing.T) { + tokens := []Token{{Variable: "instance"}, {Literal: " is down\n"}, {Variable: "job"}, {Literal: " is down"}} + assert.Equal(t, "{{instance}} is down\n{{job}} is down", tokensToTmpl(tokens)) +} + +func TestMapLookupString(t *testing.T) { + cases := []struct { + name string + input string + expected string + }{ + { + name: "when there are no spaces", + input: "instance", + expected: "$labels.instance", + }, + { + name: "when there are spaces", + input: "instance with spaces", + expected: `index $labels "instance with spaces"`, + }, + { + name: "when there are quotes", + input: `instance with "quotes"`, + expected: `index $labels "instance with \"quotes\""`, + }, + { + name: "when there are backslashes", + input: `instance with \backslashes\`, + expected: `index $labels "instance with \\backslashes\\"`, + }, + { + name: "when there are legacy delimiter characters", + input: `instance{ with $delim} characters`, + expected: `index $labels "instance{ with $delim} characters"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, mapLookupString(tc.input, "labels")) + }) + } +} + +func TestVariablesToMapLookups(t *testing.T) { + tokens := []Token{{Variable: "instance"}, {Literal: " is down"}} + expected := []Token{{Variable: "$labels.instance"}, {Literal: " is down"}} + assert.Equal(t, expected, variablesToMapLookups(tokens, "labels")) +} + +func TestVariablesToMapLookupsSpace(t *testing.T) { + tokens := []Token{{Variable: "instance with spaces"}, {Literal: " is down"}} + expected := []Token{{Variable: "index $labels \"instance with spaces\""}, {Literal: " is down"}} + assert.Equal(t, expected, variablesToMapLookups(tokens, "labels")) +} + +func TestEscapeLiterals(t *testing.T) { + cases := []struct { + name string + input []Token + expected []Token + }{ + { + name: "when there are no literals", + input: []Token{{Variable: "instance"}}, + expected: []Token{{Variable: "instance"}}, + }, + { + name: "literal with double braces: {{", + input: []Token{{Literal: "instance {{"}}, + expected: []Token{{Literal: "{{`instance {{`}}"}}, + }, + { + name: "literal that ends with closing brace: {", + input: []Token{{Literal: "instance {"}}, + expected: []Token{{Literal: "{{`instance {`}}"}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, escapeLiterals(tc.input)) + }) + } +} + +func TestMigrateTmpl(t *testing.T) { + cases := []struct { + name string + input string + expected string + vars bool + }{ + { + name: "template does not contain variables", + input: "instance is down", + expected: "instance is down", + vars: false, + }, + { + name: "template contains variable", + input: "${instance} is down", + expected: withDeduplicateMap("{{$mergedLabels.instance}} is down"), + vars: true, + }, + { + name: "template contains double braces", + input: "{{CRITICAL}} instance is down", + expected: "{{`{{CRITICAL}} instance is down`}}", + vars: false, + }, + { + name: "template contains opening brace before variable", + input: `${${instance} is down`, + expected: withDeduplicateMap("{{`${`}}{{$mergedLabels.instance}} is down"), + vars: true, + }, + { + name: "template contains newline", + input: "CRITICAL\n${instance} is down", + expected: withDeduplicateMap("CRITICAL\n{{$mergedLabels.instance}} is down"), + vars: true, + }, + { + name: "partial migration, no variables", + input: "${instance is down", + expected: "${instance is down", + }, + { + name: "partial migration, with variables", + input: "${instance} is down ${${nestedVar}}", + expected: withDeduplicateMap("{{$mergedLabels.instance}}{{` is down ${`}}{{$mergedLabels.nestedVar}}}"), + vars: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tmpl := MigrateTmpl(log.NewNopLogger(), tc.input) + + assert.Equal(t, tc.expected, tmpl) + }) + } +} + +func withDeduplicateMap(input string) string { + // hardcode function name to fail tests if it changes + funcName := "mergeLabelValues" + + return fmt.Sprintf("{{- $mergedLabels := %s $values -}}\n", funcName) + input +}