Instrumentation: Add success rate metrics for email notifications (#33359) (#33409)

Signed-off-by: bergquist <carl.bergquist@gmail.com>
(cherry picked from commit 58a325a4e4)

pkg/services/notifications/mailer.go

@@ -20,8 +20,29 @@ import (
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/util/errutil"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
+var (
+	emailsSentTotal  prometheus.Counter
+	emailsSentFailed prometheus.Counter
+)
+
+func init() {
+	emailsSentTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name:      "emails_sent_total",
+		Help:      "Number of emails sent by Grafana",
+		Namespace: "grafana",
+	})
+
+	emailsSentFailed = promauto.NewCounter(prometheus.CounterOpts{
+		Name:      "emails_sent_failed",
+		Help:      "Number of emails Grafana failed to send",
+		Namespace: "grafana",
+	})
+}
+
 func (ns *NotificationService) send(msg *Message) (int, error) {
 	messages := []*Message{}
 
@@ -38,10 +59,11 @@ func (ns *NotificationService) send(msg *Message) (int, error) {
 	return ns.dialAndSend(messages...)
 }
 
-func (ns *NotificationService) dialAndSend(messages ...*Message) (num int, err error) {
+func (ns *NotificationService) dialAndSend(messages ...*Message) (int, error) {
+	sentEmailsCount := 0
 	dialer, err := ns.createDialer()
 	if err != nil {
-		return
+		return sentEmailsCount, err
 	}
 
 	for _, msg := range messages {
@@ -58,15 +80,18 @@ func (ns *NotificationService) dialAndSend(messages ...*Message) (num int, err e
 
 		m.SetBody("text/html", msg.Body)
 
-		if e := dialer.DialAndSend(m); e != nil {
+		innerError := dialer.DialAndSend(m)
-			err = errutil.Wrapf(e, "Failed to send notification to email addresses: %s", strings.Join(msg.To, ";"))
+		emailsSentTotal.Inc()
+		if innerError != nil {
+			emailsSentFailed.Inc()
+			err = errutil.Wrapf(innerError, "Failed to send notification to email addresses: %s", strings.Join(msg.To, ";"))
 			continue
 		}
 
-		num++
+		sentEmailsCount++
 	}
 
-	return
+	return sentEmailsCount, err
 }
 
 // setFiles attaches files in various forms