Fix flaky tests in several notifiers (#42668) (#42734)

* Fix flaky tests in several notifiers

- Non-mocked time in sensu go tests
- Close server in Slack tests
- Use a mutex for writing responses in the fake slack server

* Remove mutex at the fake slack server

(cherry picked from commit bdab1d1f1f)

Co-authored-by: gotjosh <josue@grafana.com>

pkg/services/ngalert/notifier/channels/sensugo.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
-	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -120,7 +119,7 @@ func (sn *SensuGoNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 				},
 			},
 			"output":   tmpl(sn.Message),
-			"issued":   time.Now().Unix(),
+			"issued":   timeNow().Unix(),
 			"interval": 86400,
 			"status":   status,
 			"handlers": handlers,

pkg/services/ngalert/notifier/channels/sensugo_test.go

@@ -20,6 +20,9 @@ import (
 )
 
 func TestSensuGoNotifier(t *testing.T) {
+	constNow := time.Now()
+	defer mockTimeNow(constNow)()
+
 	tmpl := templateForTests(t)
 
 	externalURL, err := url.Parse("http://localhost")
@@ -60,7 +63,7 @@ func TestSensuGoNotifier(t *testing.T) {
 						},
 					},
 					"output":   "**Firing**\n\nValue: <no value>\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matchers=alertname%3Dalert1%2Clbl1%3Dval1\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
-					"issued":   time.Now().Unix(),
+					"issued":   timeNow().Unix(),
 					"interval": 86400,
 					"status":   2,
 					"handlers": nil,
@@ -107,7 +110,7 @@ func TestSensuGoNotifier(t *testing.T) {
 						},
 					},
 					"output":   "2 alerts are firing, 0 are resolved",
-					"issued":   time.Now().Unix(),
+					"issued":   timeNow().Unix(),
 					"interval": 86400,
 					"status":   2,
 					"handlers": []string{"myhandler"},

pkg/services/ngalert/notifier/channels/slack_test.go

@@ -303,6 +303,7 @@ func TestSendSlackRequest(t *testing.T) {
 				_, err := w.Write([]byte(test.slackResponse))
 				require.NoError(tt, err)
 			}))
+			defer server.Close()
 			req, err := http.NewRequest(http.MethodGet, server.URL, nil)
 			require.NoError(tt, err)