Zanzana: Add metric for last reconciliation (#115768)

pkg/server/wire_gen.go

@@ -847,7 +847,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api
 	if err != nil {
 		return nil, err
 	}
-	zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService)
+	zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService, registerer)
 	investigationsAppProvider := investigations.RegisterApp(cfg)
 	appregistryService, err := appregistry.ProvideBuilderRunners(apiserverService, eventualRestConfigProvider, featureToggles, investigationsAppProvider, cfg)
 	if err != nil {
@@ -1509,7 +1509,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac
 	if err != nil {
 		return nil, err
 	}
-	zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService)
+	zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService, registerer)
 	investigationsAppProvider := investigations.RegisterApp(cfg)
 	appregistryService, err := appregistry.ProvideBuilderRunners(apiserverService, eventualRestConfigProvider, featureToggles, investigationsAppProvider, cfg)
 	if err != nil {

pkg/services/accesscontrol/dualwrite/reconciler.go

@@ -6,6 +6,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"go.opentelemetry.io/otel"
 
 	claims "github.com/grafana/authlib/types"
@@ -34,12 +36,15 @@ type ZanzanaReconciler struct {
 	store    db.DB
 	client   zanzana.Client
 	lock     *serverlock.ServerLockService
+	metrics  struct {
+		lastSuccess prometheus.Gauge
+	}
 	// reconcilers are migrations that tries to reconcile the state of grafana db to zanzana store.
 	// These are run periodically to try to maintain a consistent state.
 	reconcilers []resourceReconciler
 }
 
-func ProvideZanzanaReconciler(cfg *setting.Cfg, features featuremgmt.FeatureToggles, client zanzana.Client, store db.DB, lock *serverlock.ServerLockService, folderService folder.Service) *ZanzanaReconciler {
+func ProvideZanzanaReconciler(cfg *setting.Cfg, features featuremgmt.FeatureToggles, client zanzana.Client, store db.DB, lock *serverlock.ServerLockService, folderService folder.Service, reg prometheus.Registerer) *ZanzanaReconciler {
 	zanzanaReconciler := &ZanzanaReconciler{
 		cfg:      cfg,
 		log:      reconcilerLogger,
@@ -93,6 +98,13 @@ func ProvideZanzanaReconciler(cfg *setting.Cfg, features featuremgmt.FeatureTogg
 		},
 	}
 
+	if reg != nil {
+		zanzanaReconciler.metrics.lastSuccess = promauto.With(reg).NewGauge(prometheus.GaugeOpts{
+			Name: "grafana_zanzana_reconcile_last_success_timestamp_seconds",
+			Help: "Unix timestamp (seconds) when the Zanzana reconciler last completed a reconciliation cycle.",
+		})
+	}
+
 	if cfg.Anonymous.Enabled {
 		zanzanaReconciler.reconcilers = append(zanzanaReconciler.reconcilers,
 			newResourceReconciler(
@@ -165,7 +177,7 @@ func (r *ZanzanaReconciler) hasBasicRolePermissions(ctx context.Context) bool {
 func (r *ZanzanaReconciler) waitForBasicRolesSeeded(ctx context.Context) {
 	// Best-effort: don't block forever. If we can't observe basic roles, proceed anyway.
 	const (
-		maxWait  = 30 * time.Second
+		maxWait  = 15 * time.Second
 		interval = 1 * time.Second
 	)
 
@@ -199,6 +211,9 @@ func (r *ZanzanaReconciler) reconcile(ctx context.Context) {
 				r.log.Warn("Failed to perform reconciliation for resource", "err", err)
 			}
 		}
+		if r.metrics.lastSuccess != nil {
+			r.metrics.lastSuccess.SetToCurrentTime()
+		}
 		r.log.Debug("Finished reconciliation", "elapsed", time.Since(now))
 	}
 

pkg/tests/apis/folder/folder_tree_test.go

@@ -102,6 +102,8 @@ func runIntegrationFolderTree(t *testing.T, opts testinfra.GrafanaOpts) {
 	helper := apis.NewK8sTestHelper(t, opts)
 	defer helper.Shutdown()
 
+	apis.AwaitZanzanaReconcileNext(t, helper)
+
 	tests := []struct {
 		Name       string
 		Definition FolderDefinition
@@ -247,6 +249,8 @@ func (f *FolderDefinition) CreateWithLegacyAPI(t *testing.T, h *apis.K8sTestHelp
 		})
 		require.NoError(t, err)
 
+		apis.AwaitZanzanaReconcileNext(t, h)
+
 		var statusCode int
 		result := client.Post().AbsPath("api", "folders").
 			Body(body).

pkg/tests/apis/zanzana_reconcile.go

@@ -0,0 +1,87 @@
+package apis
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+)
+
+const zanzanaReconcileLastSuccessMetric = "grafana_zanzana_reconcile_last_success_timestamp_seconds"
+
+// AwaitZanzanaReconcileNext waits for the next Zanzana reconciliation cycle to complete.
+// It is a no-op unless the `zanzana` feature toggle is enabled for the running test env.
+func AwaitZanzanaReconcileNext(t *testing.T, helper *K8sTestHelper) {
+	t.Helper()
+
+	enabled := false
+	if helper != nil {
+		enabled = helper.GetEnv().FeatureToggles.GetEnabled(context.Background())[featuremgmt.FlagZanzana]
+	}
+	if helper == nil || !enabled {
+		return
+	}
+
+	prev, ok := getZanzanaReconcileLastSuccessTimestampSeconds(t, helper)
+	if !ok {
+		prev = 0
+	}
+
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		ts, ok := getZanzanaReconcileLastSuccessTimestampSeconds(t, helper)
+		assert.True(c, ok, "expected to find %s in /metrics", zanzanaReconcileLastSuccessMetric)
+		if !ok {
+			return
+		}
+		assert.Greater(c, ts, prev, "expected %s (%v) > %v", zanzanaReconcileLastSuccessMetric, ts, prev)
+	}, 30*time.Second, 50*time.Millisecond)
+}
+
+func getZanzanaReconcileLastSuccessTimestampSeconds(t *testing.T, helper *K8sTestHelper) (float64, bool) {
+	t.Helper()
+
+	rsp := DoRequest(helper, RequestParams{
+		User:   helper.Org1.Admin,
+		Path:   "/metrics",
+		Accept: "text/plain",
+	}, &struct{}{})
+	if rsp.Response == nil || rsp.Response.StatusCode != http.StatusOK {
+		return 0, false
+	}
+
+	parser := expfmt.NewTextParser(model.UTF8Validation)
+	metrics, err := parser.TextToMetricFamilies(bytes.NewReader(rsp.Body))
+	if err != nil {
+		return 0, false
+	}
+
+	metric := metrics[zanzanaReconcileLastSuccessMetric]
+	if metric == nil || len(metric.Metric) == 0 {
+		return 0, false
+	}
+
+	m := metric.Metric[0]
+	switch metric.GetType() {
+	case dto.MetricType_GAUGE:
+		if m.Gauge == nil {
+			return 0, false
+		}
+		return m.Gauge.GetValue(), true
+	case dto.MetricType_UNTYPED:
+		if m.Untyped == nil {
+			return 0, false
+		}
+		return m.Untyped.GetValue(), true
+	default:
+		return 0, false
+	}
+}