diff --git a/pkg/server/search_server_distributor_test.go b/pkg/server/search_server_distributor_test.go index 520443bc15f..83c32fced37 100644 --- a/pkg/server/search_server_distributor_test.go +++ b/pkg/server/search_server_distributor_test.go @@ -314,6 +314,11 @@ func createStorageServerApi(t *testing.T, instanceId int, dbType, dbConnStr stri cfg.IndexPath = t.TempDir() + cfg.InstanceID cfg.IndexFileThreshold = testIndexFileThreshold cfg.Target = []string{modules.StorageServer} + // make sure the resource server has enough time to join the ring + // before the tests start sending traffic + // otherwise the tests will be flaky, + // also, tests are going to timeout after 300 seconds anyway + cfg.ResourceServerJoinRingTimeout = 300 * time.Second return initModuleServerForTest(t, cfg, Options{}, api.ServerOptions{}) } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index ed661529221..5cad5c1ba9b 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -601,6 +601,7 @@ type Cfg struct { SprinklesApiServerPageLimit int CACertPath string HttpsSkipVerify bool + ResourceServerJoinRingTimeout time.Duration // Secrets Management SecretsManagement SecretsManagerSettings diff --git a/pkg/setting/setting_unified_storage.go b/pkg/setting/setting_unified_storage.go index 759d076e51c..a3702dc0a3b 100644 --- a/pkg/setting/setting_unified_storage.go +++ b/pkg/setting/setting_unified_storage.go @@ -77,6 +77,7 @@ func (cfg *Cfg) setUnifiedStorageConfig() { cfg.SprinklesApiServerPageLimit = section.Key("sprinkles_api_server_page_limit").MustInt(10000) cfg.CACertPath = section.Key("ca_cert_path").String() cfg.HttpsSkipVerify = section.Key("https_skip_verify").MustBool(false) + cfg.ResourceServerJoinRingTimeout = section.Key("resource_server_join_ring_timeout").MustDuration(10 * time.Second) cfg.MaxFileIndexAge = section.Key("max_file_index_age").MustDuration(0) cfg.MinFileIndexBuildVersion = section.Key("min_file_index_build_version").MustString("") diff --git a/pkg/storage/unified/sql/service.go b/pkg/storage/unified/sql/service.go index 1e0c228cd1c..00e2ae73750 100644 --- a/pkg/storage/unified/sql/service.go +++ b/pkg/storage/unified/sql/service.go @@ -310,7 +310,7 @@ func (s *service) starting(ctx context.Context) error { if s.cfg.EnableSharding { s.log.Info("waiting until resource server is JOINING in the ring") - lfcCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) + lfcCtx, cancel := context.WithTimeout(context.Background(), s.cfg.ResourceServerJoinRingTimeout) defer cancel() if err := ring.WaitInstanceState(lfcCtx, s.searchRing, s.ringLifecycler.GetInstanceID(), ring.JOINING); err != nil { return fmt.Errorf("error switching to JOINING in the ring: %s", err)