diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 6eca21d1d50..c284c003ded 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -604,7 +604,6 @@ type Cfg struct { MinFileIndexBuildVersion string // Minimum version of Grafana that built the file-based index. If index was built with older Grafana, it will be rebuilt asynchronously. EnableSharding bool SubIndexesPerNamespace int // Number of sub-indexes per (namespace, group, resource) for sharding. 0 = disabled. - LargeFolderThreshold int // Folders with more resources than this threshold get sub-sharded. 0 = disabled. QOSEnabled bool QOSNumberWorker int QOSMaxSizePerTenant int diff --git a/pkg/setting/setting_unified_storage.go b/pkg/setting/setting_unified_storage.go index e0c64676397..91c886897bc 100644 --- a/pkg/setting/setting_unified_storage.go +++ b/pkg/setting/setting_unified_storage.go @@ -102,21 +102,33 @@ func (cfg *Cfg) setUnifiedStorageConfig() { } cfg.EnableSearch = section.Key("enable_search").MustBool(false) cfg.MaxPageSizeBytes = section.Key("max_page_size_bytes").MustInt(0) + // Index storage path. For Kubernetes Deployments without PVCs, use emptyDir: + // index_path = /var/lib/grafana/unified-search/bleve + // Indexes are derived data and will be rebuilt from SQL on pod restart. cfg.IndexPath = section.Key("index_path").String() cfg.IndexWorkers = section.Key("index_workers").MustInt(10) cfg.IndexRebuildWorkers = section.Key("index_rebuild_workers").MustInt(5) + // Sharding configuration for large-scale deployments (200k+ dashboards) + // When enable_sharding=true, indexes are distributed across pods using a ring. + // Each pod owns a subset of sub-indexes and rebuilds them from SQL on startup. + // This enables horizontal scaling without requiring PVCs (use emptyDir volumes). cfg.EnableSharding = section.Key("enable_sharding").MustBool(false) - cfg.SubIndexesPerNamespace = section.Key("sub_indexes_per_namespace").MustInt(0) // 0 = disabled, recommended: 64 for large scale - cfg.LargeFolderThreshold = section.Key("large_folder_threshold").MustInt(0) // 0 = disabled, recommended: 10000 + cfg.SubIndexesPerNamespace = section.Key("sub_indexes_per_namespace").MustInt(0) // 0 = disabled, recommended: 64 for 1M scale cfg.QOSEnabled = section.Key("qos_enabled").MustBool(false) cfg.QOSNumberWorker = section.Key("qos_num_worker").MustInt(16) cfg.QOSMaxSizePerTenant = section.Key("qos_max_size_per_tenant").MustInt(1000) + // Memberlist ring configuration for distributed search + // For Kubernetes Deployments, use DNS-based discovery with headless services: + // memberlist_join_member = dnssrv+grafana-memberlist.namespace.svc:7946 + // The dnssrv+ prefix triggers SRV record lookup for pod IPs. cfg.MemberlistBindAddr = section.Key("memberlist_bind_addr").String() cfg.MemberlistAdvertiseAddr = section.Key("memberlist_advertise_addr").String() cfg.MemberlistAdvertisePort = section.Key("memberlist_advertise_port").MustInt(7946) cfg.MemberlistJoinMember = section.Key("memberlist_join_member").String() cfg.MemberlistClusterLabel = section.Key("memberlist_cluster_label").String() cfg.MemberlistClusterLabelVerificationDisabled = section.Key("memberlist_cluster_label_verification_disabled").MustBool(false) + // SearchRingReplicationFactor configures replication factor of indexes across multiple instances. + // Recommended: 2 for production deployments using emptyDir volumes to provides availability during pod restarts/rebuilds. cfg.SearchRingReplicationFactor = section.Key("search_ring_replication_factor").MustInt(1) cfg.InstanceID = section.Key("instance_id").String() cfg.IndexFileThreshold = section.Key("index_file_threshold").MustInt(10) diff --git a/pkg/storage/unified/resource/search_server_distributor.go b/pkg/storage/unified/resource/search_server_distributor.go index 64e99b3e3b8..0fe2b41c181 100644 --- a/pkg/storage/unified/resource/search_server_distributor.go +++ b/pkg/storage/unified/resource/search_server_distributor.go @@ -44,7 +44,6 @@ func ProvideSearchDistributorServer(cfg *setting.Cfg, features featuremgmt.Featu clientPool: ringClientPool, tracing: tracer, subIndexesPerNamespace: cfg.SubIndexesPerNamespace, - replicationFactor: cfg.SearchRingReplicationFactor, } healthService, err := ProvideHealthService(distributorServer) @@ -94,7 +93,6 @@ type distributorServer struct { log log.Logger tracing trace.Tracer subIndexesPerNamespace int // Number of sub-indexes per namespace (0 = disabled) - replicationFactor int // Ring replication factor for replica failover } var ( @@ -271,7 +269,7 @@ func (ds *distributorServer) getClientToDistributeRequest(ctx context.Context, n return ctx, nil, err } - rs, err := ds.ring.GetWithOptions(ringHasher.Sum32(), searchRingRead, ring.WithReplicationFactor(ds.ring.ReplicationFactor())) + rs, err := ds.ring.GetWithOptions(ringHasher.Sum32(), searchRingRead) if err != nil { ds.log.Debug("error getting replication set from ring", "err", err, "namespace", namespace) return ctx, nil, err @@ -343,13 +341,7 @@ func (ds *distributorServer) getReplicasForSubIndex(subIndex SubIndexKey) ([]rin if err != nil { return nil, fmt.Errorf("error hashing sub-index key: %w", err) } - - replicationFactor := ds.replicationFactor - if replicationFactor <= 0 { - replicationFactor = ds.ring.ReplicationFactor() - } - - rs, err := ds.ring.GetWithOptions(ringHasher.Sum32(), searchRingRead, ring.WithReplicationFactor(replicationFactor)) + rs, err := ds.ring.GetWithOptions(ringHasher.Sum32(), searchRingRead) if err != nil { return nil, fmt.Errorf("error getting replication set from ring for sub-index %s: %w", subIndex.String(), err) } diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index d2826ce4bd2..b3b6ee23f02 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -93,11 +93,6 @@ type BleveOptions struct { // This enables horizontal scaling for large namespaces (1M+ documents). // Recommended: 64 for large scale deployments. SubIndexCount int - - // LargeFolderThreshold is the resource count above which folders get sub-sharded. - // When a folder has more than this many resources, additional sub-sharding is applied. - // 0 = disabled. Recommended: 10000. - LargeFolderThreshold int } type bleveBackend struct { diff --git a/pkg/storage/unified/search/options.go b/pkg/storage/unified/search/options.go index b3b1729c8d9..711c662c318 100644 --- a/pkg/storage/unified/search/options.go +++ b/pkg/storage/unified/search/options.go @@ -55,7 +55,6 @@ func NewSearchOptions( OwnsSubIndex: ownsSubIdx, IndexMinUpdateInterval: cfg.IndexMinUpdateInterval, SubIndexCount: cfg.SubIndexesPerNamespace, - LargeFolderThreshold: cfg.LargeFolderThreshold, }, indexMetrics) if err != nil { diff --git a/pkg/storage/unified/sql/service.go b/pkg/storage/unified/sql/service.go index 17bdaa55e4d..ccc5475f7da 100644 --- a/pkg/storage/unified/sql/service.go +++ b/pkg/storage/unified/sql/service.go @@ -265,7 +265,7 @@ func (s *service) OwnsSubIndex(key resource.NamespacedResource, subIndexID int) } } - rs, err := s.searchRing.GetWithOptions(ringHasher.Sum32(), searchOwnerRead, ring.WithReplicationFactor(s.searchRing.ReplicationFactor())) + rs, err := s.searchRing.GetWithOptions(ringHasher.Sum32(), searchOwnerRead) if err != nil { return false, fmt.Errorf("error getting replicaset from ring: %w", err) }