diff --git a/conf/defaults.ini b/conf/defaults.ini index 15b8927e65a..eb8debc0094 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -474,6 +474,10 @@ error_or_timeout = alerting # Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) nodata_or_nullvalues = no_data +# Alert notifications can include images, but rendering many images at the same time can overload the server +# This limit will protect the server from render overloading and make sure notifications are sent out quickly +concurrent_render_limit = 5 + #################################### Explore ############################# [explore] # Enable the Explore section diff --git a/conf/sample.ini b/conf/sample.ini index 2ef254f79b9..f393c66a20e 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -393,6 +393,10 @@ log_queries = # Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) ;nodata_or_nullvalues = no_data +# Alert notifications can include images, but rendering many images at the same time can overload the server +# This limit will protect the server from render overloading and make sure notifications are sent out quickly +;concurrent_render_limit = 5 + #################################### Explore ############################# [explore] # Enable the Explore section diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 2bf4789257d..5a838e8a321 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -566,3 +566,11 @@ Default setting for new alert rules. Defaults to categorize error and timeouts a > Available in 5.3 and above Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok) + +# concurrent_render_limit + +> Available in 5.3 and above + +Alert notifications can include images, but rendering many images at the same time can overload the server. +This limit will protect the server from render overloading and make sure notifications are sent out quickly. Default +value is `5`. diff --git a/pkg/api/render.go b/pkg/api/render.go index b8ef6cc5cb6..cf672af9bea 100644 --- a/pkg/api/render.go +++ b/pkg/api/render.go @@ -41,15 +41,16 @@ func (hs *HTTPServer) RenderToPng(c *m.ReqContext) { } result, err := hs.RenderService.Render(c.Req.Context(), rendering.Opts{ - Width: width, - Height: height, - Timeout: time.Duration(timeout) * time.Second, - OrgId: c.OrgId, - UserId: c.UserId, - OrgRole: c.OrgRole, - Path: c.Params("*") + queryParams, - Timezone: queryReader.Get("tz", ""), - Encoding: queryReader.Get("encoding", ""), + Width: width, + Height: height, + Timeout: time.Duration(timeout) * time.Second, + OrgId: c.OrgId, + UserId: c.UserId, + OrgRole: c.OrgRole, + Path: c.Params("*") + queryParams, + Timezone: queryReader.Get("tz", ""), + Encoding: queryReader.Get("encoding", ""), + ConcurrentLimit: 30, }) if err != nil && err == rendering.ErrTimeout { diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 7fbd956f4f9..353df1938a2 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" "github.com/grafana/grafana/pkg/services/rendering" + "github.com/grafana/grafana/pkg/setting" m "github.com/grafana/grafana/pkg/models" ) @@ -108,11 +109,12 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) { } renderOpts := rendering.Opts{ - Width: 1000, - Height: 500, - Timeout: alertTimeout / 2, - OrgId: context.Rule.OrgId, - OrgRole: m.ROLE_ADMIN, + Width: 1000, + Height: 500, + Timeout: alertTimeout / 2, + OrgId: context.Rule.OrgId, + OrgRole: m.ROLE_ADMIN, + ConcurrentLimit: setting.AlertingRenderLimit, } ref, err := context.GetDashboardUID() diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 363d06d1132..893cca948f9 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -100,7 +100,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } } - handler.notifier.SendIfNeeded(evalContext) + handler.notifier.SendIfNeeded(evalContext) return nil } diff --git a/pkg/services/rendering/interface.go b/pkg/services/rendering/interface.go index 85c139cfc04..39cb1ada0f5 100644 --- a/pkg/services/rendering/interface.go +++ b/pkg/services/rendering/interface.go @@ -13,15 +13,16 @@ var ErrNoRenderer = errors.New("No renderer plugin found nor is an external rend var ErrPhantomJSNotInstalled = errors.New("PhantomJS executable not found") type Opts struct { - Width int - Height int - Timeout time.Duration - OrgId int64 - UserId int64 - OrgRole models.RoleType - Path string - Encoding string - Timezone string + Width int + Height int + Timeout time.Duration + OrgId int64 + UserId int64 + OrgRole models.RoleType + Path string + Encoding string + Timezone string + ConcurrentLimit int } type RenderResult struct { diff --git a/pkg/services/rendering/rendering.go b/pkg/services/rendering/rendering.go index ecef83d74d9..0b4f23e93b4 100644 --- a/pkg/services/rendering/rendering.go +++ b/pkg/services/rendering/rendering.go @@ -24,12 +24,13 @@ func init() { } type RenderingService struct { - log log.Logger - pluginClient *plugin.Client - grpcPlugin pluginModel.RendererPlugin - pluginInfo *plugins.RendererPlugin - renderAction renderFunc - domain string + log log.Logger + pluginClient *plugin.Client + grpcPlugin pluginModel.RendererPlugin + pluginInfo *plugins.RendererPlugin + renderAction renderFunc + domain string + inProgressCount int Cfg *setting.Cfg `inject:""` } @@ -90,6 +91,18 @@ func (rs *RenderingService) Run(ctx context.Context) error { } func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResult, error) { + if rs.inProgressCount > opts.ConcurrentLimit { + return &RenderResult{ + FilePath: filepath.Join(setting.HomePath, "public/img/rendering_limit.png"), + }, nil + } + + defer func() { + rs.inProgressCount -= 1 + }() + + rs.inProgressCount += 1 + if rs.renderAction != nil { return rs.renderAction(ctx, opts) } else { diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 1a253b9b238..27df73a9eed 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -166,6 +166,7 @@ var ( // Alerting AlertingEnabled bool ExecuteAlerts bool + AlertingRenderLimit int AlertingErrorOrTimeout string AlertingNoDataOrNullValues string @@ -196,10 +197,13 @@ type Cfg struct { Smtp SmtpSettings // Rendering - ImagesDir string - PhantomDir string - RendererUrl string - RendererCallbackUrl string + ImagesDir string + PhantomDir string + RendererUrl string + RendererCallbackUrl string + RendererLimit int + RendererLimitAlerting int + DisableBruteForceLoginProtection bool TempDataLifetime time.Duration @@ -677,6 +681,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { alerting := iniFile.Section("alerting") AlertingEnabled = alerting.Key("enabled").MustBool(true) ExecuteAlerts = alerting.Key("execute_alerts").MustBool(true) + AlertingRenderLimit = alerting.Key("concurrent_render_limit").MustInt(5) AlertingErrorOrTimeout = alerting.Key("error_or_timeout").MustString("alerting") AlertingNoDataOrNullValues = alerting.Key("nodata_or_nullvalues").MustString("no_data") diff --git a/public/img/rendering_error.png b/public/img/rendering_error.png new file mode 100644 index 00000000000..cc327c267be Binary files /dev/null and b/public/img/rendering_error.png differ diff --git a/public/img/rendering_limit.png b/public/img/rendering_limit.png new file mode 100644 index 00000000000..f2ba9aad0ba Binary files /dev/null and b/public/img/rendering_limit.png differ diff --git a/public/img/rendering_plugin_not_installed.png b/public/img/rendering_plugin_not_installed.png new file mode 100644 index 00000000000..f135ff7cc9f Binary files /dev/null and b/public/img/rendering_plugin_not_installed.png differ diff --git a/public/img/rendering_timeout.png b/public/img/rendering_timeout.png new file mode 100644 index 00000000000..07a87eb5de3 Binary files /dev/null and b/public/img/rendering_timeout.png differ