Alerting: Persist silence state immediately on Create/Delete (#84705)

* Alerting: Persist silence state immediately on Create/Delete

Persists the silence state to the kvstore immediately instead of waiting for the
 next maintenance run. This is used after Create/Delete to prevent silences from
 being lost when a new Alertmanager is started before the state has persisted.
 This can happen, for example, in a rolling deployment scenario.

* Fix test that requires real data

* Don't error if silence state persist fails, maintenance will correct

pkg/services/ngalert/notifier/testing.go

@@ -7,14 +7,18 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"testing"
 	"time"
 
+	"github.com/go-openapi/strfmt"
 	"github.com/matttproud/golang_protobuf_extensions/pbutil"
+	amv2 "github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/prometheus/alertmanager/nflog/nflogpb"
 	"github.com/prometheus/alertmanager/silence/silencepb"
 	"github.com/prometheus/common/model"
 
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
 
@@ -357,3 +361,24 @@ func createNotificationLog(groupKey string, receiverName string, sentAt, expires
 		ExpiresAt: expiresAt,
 	}
 }
+
+func GenSilence(createdBy string) *apimodels.PostableSilence {
+	starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
+	ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))
+	comment := "test comment"
+	isEqual := true
+	name := "test"
+	value := "test"
+	isRegex := false
+	matchers := amv2.Matchers{&amv2.Matcher{IsEqual: &isEqual, Name: &name, Value: &value, IsRegex: &isRegex}}
+
+	return &apimodels.PostableSilence{
+		Silence: amv2.Silence{
+			Comment:   &comment,
+			CreatedBy: &createdBy,
+			Matchers:  matchers,
+			StartsAt:  &starts,
+			EndsAt:    &ends,
+		},
+	}
+}