public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Replace VictorOps receiver with the one from alerting repository (#60543)

Browse Source 
* replace victorops with one from alerting

* update other usages
This commit is contained in:
Yuri Tseretyan
2022-12-20 10:55:41 +01:00
committed by GitHub
parent d7b555c405
commit 35090c376c
8 changed files with 8 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -56,7 +56,7 @@ require (
github.com/google/uuid v1.3.0
github.com/google/wire v0.5.0
github.com/gorilla/websocket v1.5.0
github.com/grafana/alerting v0.0.0-20221216210437-c818b1197cdd
github.com/grafana/alerting v0.0.0-20221219210434-60ecaff51745
github.com/grafana/cuetsy v0.1.1
github.com/grafana/grafana-aws-sdk v0.11.0
github.com/grafana/grafana-azure-sdk-go v1.3.1
go.sum
+2
View File
@@ -1373,6 +1373,8 @@ github.com/grafana/alerting v0.0.0-20221215195045-4dd9b084e84d h1:2uPWbeBhkBfS5w
github.com/grafana/alerting v0.0.0-20221215195045-4dd9b084e84d/go.mod h1:BO51roH8bMRpAqeWxvnGePyCQoqgk1TiNISYKfoyHzQ=
github.com/grafana/alerting v0.0.0-20221216210437-c818b1197cdd h1:EiSgiWT16KVktYkZxblUqXPfueLcyLQf1oF5mTDh4NY=
github.com/grafana/alerting v0.0.0-20221216210437-c818b1197cdd/go.mod h1:A+ko8Ui4Ojw9oTi1WMCPH937mFUozN8Y41cqrOfNuy8=
github.com/grafana/alerting v0.0.0-20221219210434-60ecaff51745 h1:6HIwDYa01WcVBdz7WXnidVXfGLRAzYFNKPPFFwg9OXE=
github.com/grafana/alerting v0.0.0-20221219210434-60ecaff51745/go.mod h1:A+ko8Ui4Ojw9oTi1WMCPH937mFUozN8Y41cqrOfNuy8=
github.com/grafana/codejen v0.0.3 h1:tAWxoTUuhgmEqxJPOLtJoxlPBbMULFwKFOcRsPRPXDw=
github.com/grafana/codejen v0.0.3/go.mod h1:zmwwM/DRyQB7pfuBjTWII3CWtxcXh8LTwAYGfDfpR6s=
github.com/grafana/cuetsy v0.1.1 h1:+1jaDDYCpvKlcOWJgBRbkc5+VZIClCEn5mbI+4PLZqM=
pkg/services/ngalert/api/tooling/definitions/provisioning_contactpoints.go
+2 -1
View File
@@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
ngchannels "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels"
"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels_config"
"github.com/grafana/grafana/pkg/setting"
)
// swagger:route GET /api/v1/provisioning/contact-points provisioning stable RouteGetContactpoints
@@ -119,7 +120,7 @@ func (e *EmbeddedContactPoint) Valid(decryptFunc channels.GetDecryptedValueFn) e
Type: e.Type,
}, nil, decryptFunc, nil, nil, func(ctx ...interface{}) channels.Logger {
return &channels.FakeLogger{}
})
}, setting.BuildVersion)
if _, err := factory(cfg); err != nil {
return err
}
pkg/services/ngalert/notifier/alertmanager.go
+1 -1
View File
@@ -515,7 +515,7 @@ func (am *Alertmanager) buildReceiverIntegration(r *apimodels.PostableGrafanaRec
SecureSettings: secureSettings,
}
)
factoryConfig, err := channels.NewFactoryConfig(cfg, NewNotificationSender(am.NotificationService), am.decryptFn, tmpl, newImageStore(am.Store), LoggerFactory)
factoryConfig, err := channels.NewFactoryConfig(cfg, NewNotificationSender(am.NotificationService), am.decryptFn, tmpl, newImageStore(am.Store), LoggerFactory, setting.BuildVersion)
if err != nil {
return nil, InvalidReceiverError{
Receiver: r,
pkg/services/ngalert/notifier/channels/factory.go
+1 -1
View File
@@ -22,7 +22,7 @@ var receiverFactories = map[string]func(channels.FactoryConfig) (channels.Notifi
"teams": channels.TeamsFactory,
"telegram": channels.TelegramFactory,
"threema": channels.ThreemaFactory,
"victorops": VictorOpsFactory,
"victorops": channels.VictorOpsFactory,
"webhook": channels.WebHookFactory,
"wecom": channels.WeComFactory,
"webex": channels.WebexFactory,
pkg/services/ngalert/notifier/channels/victorops.go
-182
View File
@@ -1,182 +0,0 @@
package channels
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/common/model"
"github.com/grafana/alerting/alerting/notifier/channels"
"github.com/grafana/grafana/pkg/setting"
)
// https://help.victorops.com/knowledge-base/incident-fields-glossary/ - 20480 characters.
const victorOpsMaxMessageLenRunes = 20480
const (
// victoropsAlertStateCritical - Victorops uses "CRITICAL" string to indicate "Alerting" state
victoropsAlertStateCritical = "CRITICAL"
// victoropsAlertStateRecovery - VictorOps "RECOVERY" message type
victoropsAlertStateRecovery = "RECOVERY"
)
type victorOpsSettings struct {
URL string `json:"url,omitempty" yaml:"url,omitempty"`
MessageType string `json:"messageType,omitempty" yaml:"messageType,omitempty"`
Title string `json:"title,omitempty" yaml:"title,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
}
func buildVictorOpsSettings(fc channels.FactoryConfig) (victorOpsSettings, error) {
settings := victorOpsSettings{}
err := json.Unmarshal(fc.Config.Settings, &settings)
if err != nil {
return settings, fmt.Errorf("failed to unmarshal settings: %w", err)
}
if settings.URL == "" {
return settings, errors.New("could not find victorops url property in settings")
}
if settings.MessageType == "" {
settings.MessageType = victoropsAlertStateCritical
}
if settings.Title == "" {
settings.Title = channels.DefaultMessageTitleEmbed
}
if settings.Description == "" {
settings.Description = channels.DefaultMessageEmbed
}
return settings, nil
}
func VictorOpsFactory(fc channels.FactoryConfig) (channels.NotificationChannel, error) {
notifier, err := NewVictoropsNotifier(fc)
if err != nil {
return nil, receiverInitError{
Reason: err.Error(),
Cfg: *fc.Config,
}
}
return notifier, nil
}
// NewVictoropsNotifier creates an instance of VictoropsNotifier that
// handles posting notifications to Victorops REST API
func NewVictoropsNotifier(fc channels.FactoryConfig) (*VictoropsNotifier, error) {
settings, err := buildVictorOpsSettings(fc)
if err != nil {
return nil, err
}
return &VictoropsNotifier{
Base: channels.NewBase(fc.Config),
log: fc.Logger,
images: fc.ImageStore,
ns: fc.NotificationService,
tmpl: fc.Template,
settings: settings,
}, nil
}
// VictoropsNotifier defines URL property for Victorops REST API
// and handles notification process by formatting POST body according to
// Victorops specifications (http://victorops.force.com/knowledgebase/articles/Integration/Alert-Ingestion-API-Documentation/)
type VictoropsNotifier struct {
*channels.Base
log channels.Logger
images channels.ImageStore
ns channels.WebhookSender
tmpl *template.Template
settings victorOpsSettings
}
// Notify sends notification to Victorops via POST to URL endpoint
func (vn *VictoropsNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
vn.log.Debug("sending notification", "notification", vn.Name)
var tmplErr error
tmpl, _ := channels.TmplText(ctx, vn.tmpl, as, vn.log, &tmplErr)
messageType := buildMessageType(vn.log, tmpl, vn.settings.MessageType, as...)
groupKey, err := notify.ExtractGroupKey(ctx)
if err != nil {
return false, err
}
stateMessage, truncated := channels.TruncateInRunes(tmpl(vn.settings.Description), victorOpsMaxMessageLenRunes)
if truncated {
vn.log.Warn("Truncated stateMessage", "incident", groupKey, "max_runes", victorOpsMaxMessageLenRunes)
}
bodyJSON := map[string]interface{}{
"message_type": messageType,
"entity_id": groupKey.Hash(),
"entity_display_name": tmpl(vn.settings.Title),
"timestamp": time.Now().Unix(),
"state_message": stateMessage,
"monitoring_tool": "Grafana v" + setting.BuildVersion,
}
if tmplErr != nil {
vn.log.Warn("failed to expand message template. "+
"", "error", tmplErr.Error())
tmplErr = nil
}
_ = withStoredImages(ctx, vn.log, vn.images,
func(index int, image channels.Image) error {
if image.URL != "" {
bodyJSON["image_url"] = image.URL
return channels.ErrImagesDone
}
return nil
}, as...)
ruleURL := joinUrlPath(vn.tmpl.ExternalURL.String(), "/alerting/list", vn.log)
bodyJSON["alert_url"] = ruleURL
u := tmpl(vn.settings.URL)
if tmplErr != nil {
vn.log.Info("failed to expand URL template", "error", tmplErr.Error(), "fallback", vn.settings.URL)
u = vn.settings.URL
}
b, err := json.Marshal(bodyJSON)
if err != nil {
return false, err
}
cmd := &channels.SendWebhookSettings{
URL: u,
Body: string(b),
}
if err := vn.ns.SendWebhook(ctx, cmd); err != nil {
vn.log.Error("failed to send notification", "error", err, "webhook", vn.Name)
return false, err
}
return true, nil
}
func (vn *VictoropsNotifier) SendResolved() bool {
return !vn.GetDisableResolveMessage()
}
func buildMessageType(l channels.Logger, tmpl func(string) string, msgType string, as ...*types.Alert) string {
if types.Alerts(as...).Status() == model.AlertResolved {
return victoropsAlertStateRecovery
}
if messageType := strings.ToUpper(tmpl(msgType)); messageType != "" {
return messageType
}
l.Warn("expansion of message type template resulted in an empty string. Using fallback", "fallback", victoropsAlertStateCritical, "template", msgType)
return victoropsAlertStateCritical
}
pkg/services/ngalert/notifier/channels/victorops_test.go
-246
View File
@@ -1,246 +0,0 @@
package channels
import (
"context"
"encoding/json"
"net/url"
"testing"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/types"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
"github.com/grafana/alerting/alerting/notifier/channels"
"github.com/grafana/grafana/pkg/setting"
)
func TestVictoropsNotifier(t *testing.T) {
tmpl := templateForTests(t)
images := newFakeImageStore(2)
externalURL, err := url.Parse("http://localhost")
require.NoError(t, err)
tmpl.ExternalURL = externalURL
cases := []struct {
name string
settings string
alerts []*types.Alert
expMsg map[string]interface{}
expInitError string
expMsgError error
}{
{
name: "A single alert with image",
settings: `{"url": "http://localhost"}`,
alerts: []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh", "__alertImageToken__": "test-image-1"},
},
},
},
expMsg: map[string]interface{}{
"alert_url": "http://localhost/alerting/list",
"entity_display_name": "[FIRING:1] (val1)",
"entity_id": "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
"image_url": "https://www.example.com/test-image-1.jpg",
"message_type": "CRITICAL",
"monitoring_tool": "Grafana v" + setting.BuildVersion,
"state_message": "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
},
expMsgError: nil,
}, {
name: "Multiple alerts with images",
settings: `{"url": "http://localhost"}`,
alerts: []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
Annotations: model.LabelSet{"ann1": "annv1", "__alertImageToken__": "test-image-1"},
},
}, {
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
Annotations: model.LabelSet{"ann1": "annv2", "__alertImageToken__": "test-image-2"},
},
},
},
expMsg: map[string]interface{}{
"alert_url": "http://localhost/alerting/list",
"entity_display_name": "[FIRING:2] ",
"entity_id": "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
"image_url": "https://www.example.com/test-image-1.jpg",
"message_type": "CRITICAL",
"monitoring_tool": "Grafana v" + setting.BuildVersion,
"state_message": "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val2\nAnnotations:\n - ann1 = annv2\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval2\n",
},
expMsgError: nil,
}, {
name: "Custom message",
settings: `{"url": "http://localhost", "messageType": "Alerts firing: {{ len .Alerts.Firing }}"}`,
alerts: []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
Annotations: model.LabelSet{"ann1": "annv1"},
},
}, {
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
Annotations: model.LabelSet{"ann1": "annv2"},
},
},
},
expMsg: map[string]interface{}{
"alert_url": "http://localhost/alerting/list",
"entity_display_name": "[FIRING:2] ",
"entity_id": "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
"message_type": "ALERTS FIRING: 2",
"monitoring_tool": "Grafana v" + setting.BuildVersion,
"state_message": "**Firing**\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1\n\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val2\nAnnotations:\n - ann1 = annv2\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval2\n",
},
expMsgError: nil,
}, {
name: "Custom title and description",
settings: `{"url": "http://localhost", "title": "Alerts firing: {{ len .Alerts.Firing }}", "description": "customDescription"}`,
alerts: []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
Annotations: model.LabelSet{"ann1": "annv1"},
},
}, {
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
Annotations: model.LabelSet{"ann1": "annv2"},
},
},
},
expMsg: map[string]interface{}{
"alert_url": "http://localhost/alerting/list",
"entity_display_name": "Alerts firing: 2",
"entity_id": "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
"message_type": "CRITICAL",
"monitoring_tool": "Grafana v" + setting.BuildVersion,
"state_message": "customDescription",
},
expMsgError: nil,
}, {
name: "Missing field in template",
settings: `{"url": "http://localhost", "messageType": "custom template {{ .NotAField }} bad template"}`,
alerts: []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
Annotations: model.LabelSet{"ann1": "annv1"},
},
}, {
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
Annotations: model.LabelSet{"ann1": "annv2"},
},
},
},
expMsg: map[string]interface{}{
"alert_url": "http://localhost/alerting/list",
"entity_display_name": "",
"entity_id": "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
"message_type": "CUSTOM TEMPLATE ",
"monitoring_tool": "Grafana v" + setting.BuildVersion,
"state_message": "",
},
expMsgError: nil,
}, {
name: "Invalid template",
settings: `{"url": "http://localhost", "messageType": "custom template {{ {.NotAField }} bad template"}`,
alerts: []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val1"},
Annotations: model.LabelSet{"ann1": "annv1"},
},
}, {
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val2"},
Annotations: model.LabelSet{"ann1": "annv2"},
},
},
},
expMsg: map[string]interface{}{
"alert_url": "http://localhost/alerting/list",
"entity_display_name": "",
"entity_id": "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
"message_type": "CRITICAL",
"monitoring_tool": "Grafana v" + setting.BuildVersion,
"state_message": "",
},
expMsgError: nil,
}, {
name: "Error in initing, no URL",
settings: `{}`,
expInitError: `could not find victorops url property in settings`,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
settingsJSON := json.RawMessage(c.settings)
m := &channels.NotificationChannelConfig{
Name: "victorops_testing",
Type: "victorops",
Settings: settingsJSON,
}
webhookSender := mockNotificationService()
fc := channels.FactoryConfig{
Config: m,
NotificationService: webhookSender,
ImageStore: images,
Template: tmpl,
Logger: &channels.FakeLogger{},
}
pn, err := NewVictoropsNotifier(fc)
if c.expInitError != "" {
require.Error(t, err)
require.Equal(t, c.expInitError, err.Error())
return
}
require.NoError(t, err)
ctx := notify.WithGroupKey(context.Background(), "alertname")
ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""})
ok, err := pn.Notify(ctx, c.alerts...)
if c.expMsgError != nil {
require.False(t, ok)
require.Error(t, err)
require.Equal(t, c.expMsgError.Error(), err.Error())
return
}
require.NoError(t, err)
require.True(t, ok)
require.NotEmpty(t, webhookSender.Webhook.URL)
// Remove the non-constant timestamp
data := make(map[string]interface{})
err = json.Unmarshal([]byte(webhookSender.Webhook.Body), &data)
require.NoError(t, err)
delete(data, "timestamp")
b, err := json.Marshal(data)
require.NoError(t, err)
body := string(b)
expJson, err := json.Marshal(c.expMsg)
require.NoError(t, err)
require.JSONEq(t, string(expJson), body)
})
}
}
pkg/services/sqlstore/migrations/ualert/ualert.go
+1 -1
View File
@@ -507,7 +507,7 @@ func (m *migration) validateAlertmanagerConfig(orgID int64, config *PostableUser
}
factoryConfig, err := channels.NewFactoryConfig(cfg, nil, decryptFunc, nil, nil, func(ctx ...interface{}) channels.Logger {
return &channels.FakeLogger{}
})
}, setting.BuildVersion)
if err != nil {
return err
}