From 6d750c000e08f12febb5d5201f0a3590619da823 Mon Sep 17 00:00:00 2001 From: Alexander Emelin Date: Thu, 27 May 2021 22:03:18 +0300 Subject: [PATCH] Live: max_connections option with strict default (#34634) this should help Live to be enabled by default but still do not affect setups with lots of simultenious users. To properly handle many WS connections Grafana administrators should tune infrastructure a bit - for example increase a number of open files for a process. Will be in more details in documentation. --- conf/defaults.ini | 7 +++++++ conf/sample.ini | 7 +++++++ packages/grafana-data/src/types/config.ts | 1 + packages/grafana-runtime/src/config.ts | 1 + pkg/api/frontendsettings.go | 1 + pkg/services/live/live.go | 9 +++++++++ pkg/setting/setting.go | 18 ++++++++++++++++++ .../features/live/LiveConnectionWarning.tsx | 4 ++-- public/app/features/live/live.ts | 6 ++++-- 9 files changed, 50 insertions(+), 4 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 559b1066f89..6e0a7a6c41a 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -888,6 +888,13 @@ plugin_admin_enabled = false plugin_admin_external_manage_enabled = false plugin_catalog_url = https://grafana.com/grafana/plugins/ +#################################### Grafana Live ########################################## +[live] +# max_connections to Grafana Live WebSocket endpoint per Grafana server instance. See Grafana Live docs +# if you are planning to make it higher than default 100 since this can require some OS and infrastructure +# tuning. 0 disables Live, -1 means unlimited connections. +max_connections = 100 + #################################### Grafana Image Renderer Plugin ########################## [plugin.grafana-image-renderer] # Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert. diff --git a/conf/sample.ini b/conf/sample.ini index 50d47258814..78e4db378de 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -874,6 +874,13 @@ ;plugin_admin_external_manage_enabled = false ;plugin_catalog_url = https://grafana.com/grafana/plugins/ +#################################### Grafana Live ########################################## +[live] +# max_connections to Grafana Live WebSocket endpoint per Grafana server instance. See Grafana Live docs +# if you are planning to make it higher than default 100 since this can require some OS and infrastructure +# tuning. 0 disables Live, -1 means unlimited connections. +;max_connections = 100 + #################################### Grafana Image Renderer Plugin ########################## [plugin.grafana-image-renderer] # Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert. diff --git a/packages/grafana-data/src/types/config.ts b/packages/grafana-data/src/types/config.ts index 14ed6399305..f23064c626d 100644 --- a/packages/grafana-data/src/types/config.ts +++ b/packages/grafana-data/src/types/config.ts @@ -122,6 +122,7 @@ export interface GrafanaConfig { viewersCanEdit: boolean; editorsCanAdmin: boolean; disableSanitizeHtml: boolean; + liveEnabled: boolean; theme: GrafanaTheme; theme2: GrafanaTheme2; pluginsToPreload: string[]; diff --git a/packages/grafana-runtime/src/config.ts b/packages/grafana-runtime/src/config.ts index 18e2e7a7be1..712b7961710 100644 --- a/packages/grafana-runtime/src/config.ts +++ b/packages/grafana-runtime/src/config.ts @@ -54,6 +54,7 @@ export class GrafanaBootConfig implements GrafanaConfig { viewersCanEdit = false; editorsCanAdmin = false; disableSanitizeHtml = false; + liveEnabled = true; theme: GrafanaTheme; theme2: GrafanaTheme2; pluginsToPreload: string[] = []; diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go index 5acbf05eb6a..f61e1c09357 100644 --- a/pkg/api/frontendsettings.go +++ b/pkg/api/frontendsettings.go @@ -207,6 +207,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i "alertingErrorOrTimeout": setting.AlertingErrorOrTimeout, "alertingNoDataOrNullValues": setting.AlertingNoDataOrNullValues, "alertingMinInterval": setting.AlertingMinInterval, + "liveEnabled": hs.Cfg.LiveMaxConnections != 0, "autoAssignOrg": setting.AutoAssignOrg, "verifyEmailEnabled": setting.VerifyEmailEnabled, "sigV4AuthEnabled": setting.SigV4AuthEnabled, diff --git a/pkg/services/live/live.go b/pkg/services/live/live.go index d5c4dae71c2..f156db04899 100644 --- a/pkg/services/live/live.go +++ b/pkg/services/live/live.go @@ -178,6 +178,15 @@ func (g *GrafanaLive) Init() error { // different goroutines (belonging to different client connections). This is also // true for other event handlers. node.OnConnect(func(client *centrifuge.Client) { + numConnections := g.node.Hub().NumClients() + if g.Cfg.LiveMaxConnections >= 0 && numConnections > g.Cfg.LiveMaxConnections { + logger.Warn( + "Max number of Live connections reached, increase max_connections in [live] configuration section", + "user", client.UserID(), "client", client.ID(), "limit", g.Cfg.LiveMaxConnections, + ) + client.Disconnect(centrifuge.DisconnectConnectionLimit) + return + } var semaphore chan struct{} if clientConcurrency > 1 { semaphore = make(chan struct{}, clientConcurrency) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index e7d83477323..92fe6ef2a7b 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -380,6 +380,11 @@ type Cfg struct { ExpressionsEnabled bool ImageUploadProvider string + + // LiveMaxConnections is a maximum number of WebSocket connections to + // Grafana Live ws endpoint (per Grafana server instance). 0 disables + // Live, -1 means unlimited connections. + LiveMaxConnections int } // IsLiveConfigEnabled returns true if live should be able to save configs to SQL tables @@ -950,6 +955,10 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { cfg.readDateFormats() cfg.readSentryConfig() + if err := cfg.readLiveSettings(iniFile); err != nil { + return err + } + return nil } @@ -1420,3 +1429,12 @@ func (cfg *Cfg) readDataSourcesSettings() { datasources := cfg.Raw.Section("datasources") cfg.DataSourceLimit = datasources.Key("datasource_limit").MustInt(5000) } + +func (cfg *Cfg) readLiveSettings(iniFile *ini.File) error { + section := iniFile.Section("live") + cfg.LiveMaxConnections = section.Key("max_connections").MustInt(100) + if cfg.LiveMaxConnections < -1 { + return fmt.Errorf("unexpected value %d for [live] max_connections", cfg.LiveMaxConnections) + } + return nil +} diff --git a/public/app/features/live/LiveConnectionWarning.tsx b/public/app/features/live/LiveConnectionWarning.tsx index 41b140a7e51..a4d18d34405 100644 --- a/public/app/features/live/LiveConnectionWarning.tsx +++ b/public/app/features/live/LiveConnectionWarning.tsx @@ -45,8 +45,8 @@ export class LiveConnectionWarning extends PureComponent { render() { const { show } = this.state; if (show) { - if (!contextSrv.isSignedIn) { - return null; // do not show the warning for anonomous users (and /login page etc) + if (!contextSrv.isSignedIn || !config.liveEnabled) { + return null; // do not show the warning for anonymous users (and /login page etc) } return ( diff --git a/public/app/features/live/live.ts b/public/app/features/live/live.ts index 261d6b041dc..bc84eb00425 100644 --- a/public/app/features/live/live.ts +++ b/public/app/features/live/live.ts @@ -51,7 +51,7 @@ export class CentrifugeSrv implements GrafanaLiveSrv { readonly connectionState: BehaviorSubject; readonly connectionBlocker: Promise; readonly scopes: Record; - private orgId: number; + private readonly orgId: number; constructor() { const baseURL = window.location.origin.replace('http', 'ws'); @@ -64,7 +64,9 @@ export class CentrifugeSrv implements GrafanaLiveSrv { sessionId, orgId: this.orgId, }); - this.centrifuge.connect(); // do connection + if (config.liveEnabled) { + this.centrifuge.connect(); // do connection + } this.connectionState = new BehaviorSubject(this.centrifuge.isConnected()); this.connectionBlocker = new Promise((resolve) => { if (this.centrifuge.isConnected()) {