diff --git a/conf/defaults.ini b/conf/defaults.ini index caccebbd910..eb8debc0094 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -474,6 +474,10 @@ error_or_timeout = alerting # Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) nodata_or_nullvalues = no_data +# Alert notifications can include images, but rendering many images at the same time can overload the server +# This limit will protect the server from render overloading and make sure notifications are sent out quickly +concurrent_render_limit = 5 + #################################### Explore ############################# [explore] # Enable the Explore section @@ -550,5 +554,3 @@ container_name = # Options to configure external image rendering server like https://github.com/grafana/grafana-image-renderer server_url = callback_url = -concurrent_limit = 10 -concurrent_limit_alerting = 5 diff --git a/conf/sample.ini b/conf/sample.ini index 7a460faca0e..f393c66a20e 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -393,6 +393,10 @@ log_queries = # Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) ;nodata_or_nullvalues = no_data +# Alert notifications can include images, but rendering many images at the same time can overload the server +# This limit will protect the server from render overloading and make sure notifications are sent out quickly +;concurrent_render_limit = 5 + #################################### Explore ############################# [explore] # Enable the Explore section @@ -471,5 +475,3 @@ log_queries = # Options to configure external image rendering server like https://github.com/grafana/grafana-image-renderer ;server_url = ;callback_url = -;concurrent_limit = 10 -;concurrent_limit_alerting = 5 diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 2bf4789257d..5a838e8a321 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -566,3 +566,11 @@ Default setting for new alert rules. Defaults to categorize error and timeouts a > Available in 5.3 and above Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) + +# concurrent_render_limit + +> Available in 5.3 and above + +Alert notifications can include images, but rendering many images at the same time can overload the server. +This limit will protect the server from render overloading and make sure notifications are sent out quickly. Default +value is `5`. diff --git a/pkg/api/render.go b/pkg/api/render.go index b8ef6cc5cb6..cf672af9bea 100644 --- a/pkg/api/render.go +++ b/pkg/api/render.go @@ -41,15 +41,16 @@ func (hs *HTTPServer) RenderToPng(c *m.ReqContext) { } result, err := hs.RenderService.Render(c.Req.Context(), rendering.Opts{ - Width: width, - Height: height, - Timeout: time.Duration(timeout) * time.Second, - OrgId: c.OrgId, - UserId: c.UserId, - OrgRole: c.OrgRole, - Path: c.Params("*") + queryParams, - Timezone: queryReader.Get("tz", ""), - Encoding: queryReader.Get("encoding", ""), + Width: width, + Height: height, + Timeout: time.Duration(timeout) * time.Second, + OrgId: c.OrgId, + UserId: c.UserId, + OrgRole: c.OrgRole, + Path: c.Params("*") + queryParams, + Timezone: queryReader.Get("tz", ""), + Encoding: queryReader.Get("encoding", ""), + ConcurrentLimit: 30, }) if err != nil && err == rendering.ErrTimeout { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 839893f3444..353df1938a2 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/services/rendering" + "github.com/grafana/grafana/pkg/setting" m "github.com/grafana/grafana/pkg/models" ) @@ -108,12 +109,12 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) { } renderOpts := rendering.Opts{ - Width: 1000, - Height: 500, - Timeout: alertTimeout / 2, - OrgId: context.Rule.OrgId, - OrgRole: m.ROLE_ADMIN, - IsAlert: true, + Width: 1000, + Height: 500, + Timeout: alertTimeout / 2, + OrgId: context.Rule.OrgId, + OrgRole: m.ROLE_ADMIN, + ConcurrentLimit: setting.AlertingRenderLimit, } ref, err := context.GetDashboardUID() diff --git a/pkg/services/rendering/interface.go b/pkg/services/rendering/interface.go index 856e6e683ff..39cb1ada0f5 100644 --- a/pkg/services/rendering/interface.go +++ b/pkg/services/rendering/interface.go @@ -13,16 +13,16 @@ var ErrNoRenderer = errors.New("No renderer plugin found nor is an external rend var ErrPhantomJSNotInstalled = errors.New("PhantomJS executable not found") type Opts struct { - Width int - Height int - Timeout time.Duration - OrgId int64 - UserId int64 - OrgRole models.RoleType - Path string - Encoding string - Timezone string - IsAlert bool + Width int + Height int + Timeout time.Duration + OrgId int64 + UserId int64 + OrgRole models.RoleType + Path string + Encoding string + Timezone string + ConcurrentLimit int } type RenderResult struct { diff --git a/pkg/services/rendering/rendering.go b/pkg/services/rendering/rendering.go index 2b9d91771e9..0b4f23e93b4 100644 --- a/pkg/services/rendering/rendering.go +++ b/pkg/services/rendering/rendering.go @@ -90,16 +90,8 @@ func (rs *RenderingService) Run(ctx context.Context) error { return err } -func (rs *RenderingService) getLimit(isAlerting bool) int { - if isAlerting { - return rs.Cfg.RendererLimitAlerting - } else { - return rs.Cfg.RendererLimit - } -} - func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResult, error) { - if rs.inProgressCount > rs.getLimit(opts.IsAlert) { + if rs.inProgressCount > opts.ConcurrentLimit { return &RenderResult{ FilePath: filepath.Join(setting.HomePath, "public/img/rendering_limit.png"), }, nil diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 71e499f9298..27df73a9eed 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -166,6 +166,7 @@ var ( // Alerting AlertingEnabled bool ExecuteAlerts bool + AlertingRenderLimit int AlertingErrorOrTimeout string AlertingNoDataOrNullValues string @@ -648,9 +649,6 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { // Rendering renderSec := iniFile.Section("rendering") - cfg.RendererLimit = renderSec.Key("concurrent_limit").MustInt(10) - cfg.RendererLimitAlerting = renderSec.Key("concurrent_limit").MustInt(5) - cfg.RendererUrl = renderSec.Key("server_url").String() cfg.RendererCallbackUrl = renderSec.Key("callback_url").String() if cfg.RendererCallbackUrl == "" { @@ -683,6 +681,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { alerting := iniFile.Section("alerting") AlertingEnabled = alerting.Key("enabled").MustBool(true) ExecuteAlerts = alerting.Key("execute_alerts").MustBool(true) + AlertingRenderLimit = alerting.Key("concurrent_render_limit").MustInt(5) AlertingErrorOrTimeout = alerting.Key("error_or_timeout").MustString("alerting") AlertingNoDataOrNullValues = alerting.Key("nodata_or_nullvalues").MustString("no_data")