Alerting: Stop persisting silences and nflog to disk (#84706)

With this change, we no longer need to persist silence/nflog states to disk in addition to the kvstore

pkg/services/ngalert/remote/alertmanager.go

@@ -3,12 +3,18 @@ package remote
 import (
 	"context"
 	"crypto/md5"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
 
 	"github.com/go-openapi/strfmt"
+	amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
+	amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
+	amreceiver "github.com/prometheus/alertmanager/api/v2/client/receiver"
+	amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
+
 	"github.com/grafana/grafana/pkg/infra/log"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -16,14 +22,13 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/sender"
-	amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
-	amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
-	amreceiver "github.com/prometheus/alertmanager/api/v2/client/receiver"
-	amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
+
+	alertingClusterPB "github.com/grafana/alerting/cluster/clusterpb"
 )
 
 type stateStore interface {
-	GetFullState(ctx context.Context, keys ...string) (string, error)
+	GetSilences(ctx context.Context) (string, error)
+	GetNotificationLog(ctx context.Context) (string, error)
 }
 
 // DecryptFn is a function that takes in an encrypted value and returns it decrypted.
@@ -223,7 +228,7 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
 // CompareAndSendState gets the Alertmanager's internal state and compares it with the remote Alertmanager's one.
 // If the states are different, it updates the remote Alertmanager's state with that of the internal Alertmanager.
 func (am *Alertmanager) CompareAndSendState(ctx context.Context) error {
-	state, err := am.state.GetFullState(ctx, notifier.SilencesFilename, notifier.NotificationLogFilename)
+	state, err := am.getFullState(ctx)
 	if err != nil {
 		return err
 	}
@@ -399,6 +404,33 @@ func (am *Alertmanager) Ready() bool {
 // CleanUp does not have an equivalent in a "remote Alertmanager" context, we don't have files on disk, no-op.
 func (am *Alertmanager) CleanUp() {}
 
+// getFullState returns a base64-encoded protobuf message representing the Alertmanager's internal state.
+func (am *Alertmanager) getFullState(ctx context.Context) (string, error) {
+	var parts []alertingClusterPB.Part
+
+	silences, err := am.state.GetSilences(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error getting silences: %w", err)
+	}
+	parts = append(parts, alertingClusterPB.Part{Key: notifier.SilencesFilename, Data: []byte(silences)})
+
+	notificationLog, err := am.state.GetNotificationLog(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error getting notification log: %w", err)
+	}
+	parts = append(parts, alertingClusterPB.Part{Key: notifier.NotificationLogFilename, Data: []byte(notificationLog)})
+
+	fs := alertingClusterPB.FullState{
+		Parts: parts,
+	}
+	b, err := fs.Marshal()
+	if err != nil {
+		return "", fmt.Errorf("error marshaling full state: %w", err)
+	}
+
+	return base64.StdEncoding.EncodeToString(b), nil
+}
+
 // shouldSendConfig compares the remote Alertmanager configuration with our local one.
 // It returns true if the configurations are different.
 func (am *Alertmanager) shouldSendConfig(ctx context.Context, rawConfig []byte) bool {

pkg/services/ngalert/remote/alertmanager_test.go

@@ -17,6 +17,10 @@ import (
 	"time"
 
 	"github.com/go-openapi/strfmt"
+	amv2 "github.com/prometheus/alertmanager/api/v2/models"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+
 	"github.com/grafana/grafana/pkg/infra/db"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -30,10 +34,6 @@ import (
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/tests/testsuite"
 	"github.com/grafana/grafana/pkg/util"
-	amv2 "github.com/prometheus/alertmanager/api/v2/models"
-	"github.com/prometheus/alertmanager/cluster/clusterpb"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/stretchr/testify/require"
 )
 
 // Valid Grafana Alertmanager configurations.
@@ -144,7 +144,7 @@ func TestApplyConfig(t *testing.T) {
 
 	ctx := context.Background()
 	store := ngfakes.NewFakeKVStore(t)
-	fstore := notifier.NewFileStore(1, store, "")
+	fstore := notifier.NewFileStore(1, store)
 	require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.SilencesFilename, "test"))
 	require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.NotificationLogFilename, "test"))
 
@@ -196,7 +196,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 		require.NoError(t, err)
 	}))
 
-	fstore := notifier.NewFileStore(1, ngfakes.NewFakeKVStore(t), "")
+	fstore := notifier.NewFileStore(1, ngfakes.NewFakeKVStore(t))
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
 	cfg := AlertmanagerConfig{
 		OrgID:    1,
@@ -296,27 +296,20 @@ func TestIntegrationRemoteAlertmanagerApplyConfigOnlyUploadsOnce(t *testing.T) {
 	silences := []byte("test-silences")
 	nflog := []byte("test-notifications")
 	store := ngfakes.NewFakeKVStore(t)
-	fstore := notifier.NewFileStore(cfg.OrgID, store, "")
+	fstore := notifier.NewFileStore(cfg.OrgID, store)
 
 	ctx := context.Background()
 	require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.SilencesFilename, base64.StdEncoding.EncodeToString(silences)))
 	require.NoError(t, store.Set(ctx, cfg.OrgID, "alertmanager", notifier.NotificationLogFilename, base64.StdEncoding.EncodeToString(nflog)))
 
-	fs := clusterpb.FullState{
-		Parts: []clusterpb.Part{
-			{Key: "silences", Data: silences},
-			{Key: "notifications", Data: nflog},
-		},
-	}
-	fullState, err := fs.Marshal()
-	require.NoError(t, err)
-	encodedFullState := base64.StdEncoding.EncodeToString(fullState)
-
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
 	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
 	am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, m)
 	require.NoError(t, err)
 
+	encodedFullState, err := am.getFullState(ctx)
+	require.NoError(t, err)
+
 	// We should have no configuration or state at first.
 	{
 		_, err := am.mimirClient.GetGrafanaAlertmanagerConfig(ctx)