diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index b958e5f7ad9..4ae1194ef28 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -847,7 +847,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api if err != nil { return nil, err } - zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService) + zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService, registerer) investigationsAppProvider := investigations.RegisterApp(cfg) appregistryService, err := appregistry.ProvideBuilderRunners(apiserverService, eventualRestConfigProvider, featureToggles, investigationsAppProvider, cfg) if err != nil { @@ -1509,7 +1509,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac if err != nil { return nil, err } - zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService) + zanzanaReconciler := dualwrite2.ProvideZanzanaReconciler(cfg, featureToggles, zanzanaClient, sqlStore, serverLockService, folderimplService, registerer) investigationsAppProvider := investigations.RegisterApp(cfg) appregistryService, err := appregistry.ProvideBuilderRunners(apiserverService, eventualRestConfigProvider, featureToggles, investigationsAppProvider, cfg) if err != nil { diff --git a/pkg/services/accesscontrol/dualwrite/reconciler.go b/pkg/services/accesscontrol/dualwrite/reconciler.go index d66039d44f2..ab27972e86e 100644 --- a/pkg/services/accesscontrol/dualwrite/reconciler.go +++ b/pkg/services/accesscontrol/dualwrite/reconciler.go @@ -6,6 +6,8 @@ import ( "strconv" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "go.opentelemetry.io/otel" claims "github.com/grafana/authlib/types" @@ -34,12 +36,15 @@ type ZanzanaReconciler struct { store db.DB client zanzana.Client lock *serverlock.ServerLockService + metrics struct { + lastSuccess prometheus.Gauge + } // reconcilers are migrations that tries to reconcile the state of grafana db to zanzana store. // These are run periodically to try to maintain a consistent state. reconcilers []resourceReconciler } -func ProvideZanzanaReconciler(cfg *setting.Cfg, features featuremgmt.FeatureToggles, client zanzana.Client, store db.DB, lock *serverlock.ServerLockService, folderService folder.Service) *ZanzanaReconciler { +func ProvideZanzanaReconciler(cfg *setting.Cfg, features featuremgmt.FeatureToggles, client zanzana.Client, store db.DB, lock *serverlock.ServerLockService, folderService folder.Service, reg prometheus.Registerer) *ZanzanaReconciler { zanzanaReconciler := &ZanzanaReconciler{ cfg: cfg, log: reconcilerLogger, @@ -93,6 +98,13 @@ func ProvideZanzanaReconciler(cfg *setting.Cfg, features featuremgmt.FeatureTogg }, } + if reg != nil { + zanzanaReconciler.metrics.lastSuccess = promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "grafana_zanzana_reconcile_last_success_timestamp_seconds", + Help: "Unix timestamp (seconds) when the Zanzana reconciler last completed a reconciliation cycle.", + }) + } + if cfg.Anonymous.Enabled { zanzanaReconciler.reconcilers = append(zanzanaReconciler.reconcilers, newResourceReconciler( @@ -165,7 +177,7 @@ func (r *ZanzanaReconciler) hasBasicRolePermissions(ctx context.Context) bool { func (r *ZanzanaReconciler) waitForBasicRolesSeeded(ctx context.Context) { // Best-effort: don't block forever. If we can't observe basic roles, proceed anyway. const ( - maxWait = 30 * time.Second + maxWait = 15 * time.Second interval = 1 * time.Second ) @@ -199,6 +211,9 @@ func (r *ZanzanaReconciler) reconcile(ctx context.Context) { r.log.Warn("Failed to perform reconciliation for resource", "err", err) } } + if r.metrics.lastSuccess != nil { + r.metrics.lastSuccess.SetToCurrentTime() + } r.log.Debug("Finished reconciliation", "elapsed", time.Since(now)) } diff --git a/pkg/tests/apis/folder/folder_tree_test.go b/pkg/tests/apis/folder/folder_tree_test.go index 26e7b5f6884..613d021b236 100644 --- a/pkg/tests/apis/folder/folder_tree_test.go +++ b/pkg/tests/apis/folder/folder_tree_test.go @@ -102,6 +102,8 @@ func runIntegrationFolderTree(t *testing.T, opts testinfra.GrafanaOpts) { helper := apis.NewK8sTestHelper(t, opts) defer helper.Shutdown() + apis.AwaitZanzanaReconcileNext(t, helper) + tests := []struct { Name string Definition FolderDefinition @@ -247,6 +249,8 @@ func (f *FolderDefinition) CreateWithLegacyAPI(t *testing.T, h *apis.K8sTestHelp }) require.NoError(t, err) + apis.AwaitZanzanaReconcileNext(t, h) + var statusCode int result := client.Post().AbsPath("api", "folders"). Body(body). diff --git a/pkg/tests/apis/zanzana_reconcile.go b/pkg/tests/apis/zanzana_reconcile.go new file mode 100644 index 00000000000..f8a5673fed7 --- /dev/null +++ b/pkg/tests/apis/zanzana_reconcile.go @@ -0,0 +1,87 @@ +package apis + +import ( + "bytes" + "context" + "net/http" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/services/featuremgmt" +) + +const zanzanaReconcileLastSuccessMetric = "grafana_zanzana_reconcile_last_success_timestamp_seconds" + +// AwaitZanzanaReconcileNext waits for the next Zanzana reconciliation cycle to complete. +// It is a no-op unless the `zanzana` feature toggle is enabled for the running test env. +func AwaitZanzanaReconcileNext(t *testing.T, helper *K8sTestHelper) { + t.Helper() + + enabled := false + if helper != nil { + enabled = helper.GetEnv().FeatureToggles.GetEnabled(context.Background())[featuremgmt.FlagZanzana] + } + if helper == nil || !enabled { + return + } + + prev, ok := getZanzanaReconcileLastSuccessTimestampSeconds(t, helper) + if !ok { + prev = 0 + } + + require.EventuallyWithT(t, func(c *assert.CollectT) { + ts, ok := getZanzanaReconcileLastSuccessTimestampSeconds(t, helper) + assert.True(c, ok, "expected to find %s in /metrics", zanzanaReconcileLastSuccessMetric) + if !ok { + return + } + assert.Greater(c, ts, prev, "expected %s (%v) > %v", zanzanaReconcileLastSuccessMetric, ts, prev) + }, 30*time.Second, 50*time.Millisecond) +} + +func getZanzanaReconcileLastSuccessTimestampSeconds(t *testing.T, helper *K8sTestHelper) (float64, bool) { + t.Helper() + + rsp := DoRequest(helper, RequestParams{ + User: helper.Org1.Admin, + Path: "/metrics", + Accept: "text/plain", + }, &struct{}{}) + if rsp.Response == nil || rsp.Response.StatusCode != http.StatusOK { + return 0, false + } + + parser := expfmt.NewTextParser(model.UTF8Validation) + metrics, err := parser.TextToMetricFamilies(bytes.NewReader(rsp.Body)) + if err != nil { + return 0, false + } + + metric := metrics[zanzanaReconcileLastSuccessMetric] + if metric == nil || len(metric.Metric) == 0 { + return 0, false + } + + m := metric.Metric[0] + switch metric.GetType() { + case dto.MetricType_GAUGE: + if m.Gauge == nil { + return 0, false + } + return m.Gauge.GetValue(), true + case dto.MetricType_UNTYPED: + if m.Untyped == nil { + return 0, false + } + return m.Untyped.GetValue(), true + default: + return 0, false + } +}