From fa4dbce7daeb5fcf1698b6d22787d2d3f854a89a Mon Sep 17 00:00:00 2001 From: Rafael Paulovic Date: Fri, 9 Jan 2026 19:18:56 +0100 Subject: [PATCH] fix(unified-storage): add per-replica timeout for SLO compliance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Phase 3 review findings: - Add 200ms per-replica timeout in searchSubIndexWithFailover() - Ensures failover happens quickly if primary replica is slow - Maintains ≤500ms search-distributor latency SLO target - With replication factor 2: worst case is 2×200ms + merge time --- .../unified/resource/search_server_distributor.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/storage/unified/resource/search_server_distributor.go b/pkg/storage/unified/resource/search_server_distributor.go index 5fd4dacf7d9..64e99b3e3b8 100644 --- a/pkg/storage/unified/resource/search_server_distributor.go +++ b/pkg/storage/unified/resource/search_server_distributor.go @@ -420,6 +420,9 @@ func (ds *distributorServer) searchSubIndexWithFailover(ctx context.Context, sub rCtx := userutils.InjectOrgID(metadata.NewOutgoingContext(ctx, md), subIndex.Namespace) // Try each replica in order until success + // Per-replica timeout ensures SLO compliance (≤500ms for distributed search) + // With replication factor 2: worst case is 2 attempts × 200ms = 400ms, leaving room for merge + const replicaTimeout = 200 * time.Millisecond var lastErr error for replicaIdx, replica := range replicas { client, err := ds.clientPool.GetClientForInstance(replica) @@ -433,7 +436,11 @@ func (ds *distributorServer) searchSubIndexWithFailover(ctx context.Context, sub continue } - resp, err := client.(*RingClient).Client.Search(rCtx, r) + // Apply per-replica timeout to ensure failover happens quickly + replicaCtx, cancel := context.WithTimeout(rCtx, replicaTimeout) + resp, err := client.(*RingClient).Client.Search(replicaCtx, r) + cancel() // Always cancel to release resources + if err == nil && resp.Error == nil { // Success result.response = resp