Rendering: add CSV support (#33729)

* Rendering: add CSV rendering support

* Rendering: save csv files into a separate folder

* add missing field

* Renderer: get filename from renderer plugin

* apply PR suggestions

* Rendering: remove old PhantomJS error

* Rendering: separate RenderCSV and Render functions

* fix alerting test

* Rendering: fix handling error in HTTP mode

* apply PR feedback

* Update pkg/services/rendering/http_mode.go

Co-authored-by: Joan López de la Franca Beltran <joanjan14@gmail.com>

* apply PR feedback

* Update rendering metrics with type label

* Rendering: return error if not able to parse header

* Rendering: update grpc generated file

* Rendering: use context.WithTimeout to render CSV too

Co-authored-by: Joan López de la Franca Beltran <joanjan14@gmail.com>

pkg/services/rendering/http_mode.go

@@ -5,14 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
+	"mime"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"strconv"
 	"time"
-
-	"github.com/grafana/grafana/pkg/setting"
 )
 
 var netTransport = &http.Transport{
@@ -27,18 +27,18 @@ var netClient = &http.Client{
 	Transport: netTransport,
 }
 
-func (rs *RenderingService) renderViaHttp(ctx context.Context, renderKey string, opts Opts) (*RenderResult, error) {
-	filePath, err := rs.getFilePathForNewImage()
+func (rs *RenderingService) renderViaHTTP(ctx context.Context, renderKey string, opts Opts) (*RenderResult, error) {
+	filePath, err := rs.getNewFilePath(RenderPNG)
 	if err != nil {
 		return nil, err
 	}
 
-	rendererUrl, err := url.Parse(rs.Cfg.RendererUrl)
+	rendererURL, err := url.Parse(rs.Cfg.RendererUrl)
 	if err != nil {
 		return nil, err
 	}
 
-	queryParams := rendererUrl.Query()
+	queryParams := rendererURL.Query()
 	queryParams.Add("url", rs.getURL(opts.Path))
 	queryParams.Add("renderKey", renderKey)
 	queryParams.Add("width", strconv.Itoa(opts.Width))
@@ -48,32 +48,16 @@ func (rs *RenderingService) renderViaHttp(ctx context.Context, renderKey string,
 	queryParams.Add("encoding", opts.Encoding)
 	queryParams.Add("timeout", strconv.Itoa(int(opts.Timeout.Seconds())))
 	queryParams.Add("deviceScaleFactor", fmt.Sprintf("%f", opts.DeviceScaleFactor))
-	rendererUrl.RawQuery = queryParams.Encode()
 
-	req, err := http.NewRequest("GET", rendererUrl.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
-
-	for k, v := range opts.Headers {
-		req.Header[k] = v
-	}
+	rendererURL.RawQuery = queryParams.Encode()
 
 	// gives service some additional time to timeout and return possible errors.
 	reqContext, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2)
 	defer cancel()
 
-	req = req.WithContext(reqContext)
-
-	rs.log.Debug("calling remote rendering service", "url", rendererUrl)
-
-	// make request to renderer server
-	resp, err := netClient.Do(req)
+	resp, err := rs.doRequest(reqContext, rendererURL, opts.Headers)
 	if err != nil {
-		rs.log.Error("Failed to send request to remote rendering service.", "error", err)
-		return nil, fmt.Errorf("failed to send request to remote rendering service: %w", err)
+		return nil, err
 	}
 
 	// save response to file
@@ -83,42 +67,128 @@ func (rs *RenderingService) renderViaHttp(ctx context.Context, renderKey string,
 		}
 	}()
 
+	err = rs.readFileResponse(reqContext, resp, filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &RenderResult{FilePath: filePath}, nil
+}
+
+func (rs *RenderingService) renderCSVViaHTTP(ctx context.Context, renderKey string, opts CSVOpts) (*RenderCSVResult, error) {
+	filePath, err := rs.getNewFilePath(RenderCSV)
+	if err != nil {
+		return nil, err
+	}
+
+	rendererURL, err := url.Parse(rs.Cfg.RendererUrl + "/csv")
+	if err != nil {
+		return nil, err
+	}
+
+	queryParams := rendererURL.Query()
+	queryParams.Add("url", rs.getURL(opts.Path))
+	queryParams.Add("renderKey", renderKey)
+	queryParams.Add("domain", rs.domain)
+	queryParams.Add("timezone", isoTimeOffsetToPosixTz(opts.Timezone))
+	queryParams.Add("encoding", opts.Encoding)
+	queryParams.Add("timeout", strconv.Itoa(int(opts.Timeout.Seconds())))
+
+	rendererURL.RawQuery = queryParams.Encode()
+
+	// gives service some additional time to timeout and return possible errors.
+	reqContext, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2)
+	defer cancel()
+
+	resp, err := rs.doRequest(reqContext, rendererURL, opts.Headers)
+	if err != nil {
+		return nil, err
+	}
+
+	// save response to file
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			rs.log.Warn("Failed to close response body", "err", err)
+		}
+	}()
+
+	_, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
+	if err != nil {
+		return nil, err
+	}
+	downloadFileName := params["filename"]
+
+	err = rs.readFileResponse(reqContext, resp, filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &RenderCSVResult{FilePath: filePath, FileName: downloadFileName}, nil
+}
+
+func (rs *RenderingService) doRequest(ctx context.Context, url *url.URL, headers map[string][]string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", rs.Cfg.BuildVersion))
+	for k, v := range headers {
+		req.Header[k] = v
+	}
+
+	rs.log.Debug("calling remote rendering service", "url", url)
+
+	// make request to renderer server
+	resp, err := netClient.Do(req)
+	if err != nil {
+		rs.log.Error("Failed to send request to remote rendering service", "error", err)
+		return nil, fmt.Errorf("failed to send request to remote rendering service: %w", err)
+	}
+
+	return resp, nil
+}
+
+func (rs *RenderingService) readFileResponse(ctx context.Context, resp *http.Response, filePath string) error {
 	// check for timeout first
-	if errors.Is(reqContext.Err(), context.DeadlineExceeded) {
+	if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 		rs.log.Info("Rendering timed out")
-		return nil, ErrTimeout
+		return ErrTimeout
 	}
 
 	// if we didn't get a 200 response, something went wrong.
 	if resp.StatusCode != http.StatusOK {
 		rs.log.Error("Remote rendering request failed", "error", resp.Status)
-		return nil, fmt.Errorf("remote rendering request failed, status code: %d, status: %s", resp.StatusCode,
+		return fmt.Errorf("remote rendering request failed, status code: %d, status: %s", resp.StatusCode,
 			resp.Status)
 	}
 
 	out, err := os.Create(filePath)
 	if err != nil {
-		return nil, err
+		return err
 	}
+
 	defer func() {
-		if err := out.Close(); err != nil {
+		if err := out.Close(); err != nil && !errors.Is(err, fs.ErrClosed) {
 			// We already close the file explicitly in the non-error path, so shouldn't be a problem
 			rs.log.Warn("Failed to close file", "path", filePath, "err", err)
 		}
 	}()
+
 	_, err = io.Copy(out, resp.Body)
 	if err != nil {
 		// check that we didn't timeout while receiving the response.
-		if errors.Is(reqContext.Err(), context.DeadlineExceeded) {
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 			rs.log.Info("Rendering timed out")
-			return nil, ErrTimeout
+			return ErrTimeout
 		}
+
 		rs.log.Error("Remote rendering request failed", "error", err)
-		return nil, fmt.Errorf("remote rendering request failed: %w", err)
+		return fmt.Errorf("remote rendering request failed: %w", err)
 	}
 	if err := out.Close(); err != nil {
-		return nil, fmt.Errorf("failed to write to %q: %w", filePath, err)
+		return fmt.Errorf("failed to write to %q: %w", filePath, err)
 	}
 
-	return &RenderResult{FilePath: filePath}, err
+	return nil
 }

pkg/services/rendering/interface.go

@@ -9,14 +9,22 @@ import (
 )
 
 var ErrTimeout = errors.New("timeout error - you can set timeout in seconds with &timeout url parameter")
-var ErrPhantomJSNotInstalled = errors.New("PhantomJS executable not found")
+var ErrConcurrentLimitReached = errors.New("rendering concurrent limit reached")
+var ErrRenderUnavailable = errors.New("rendering plugin not available")
+
+type RenderType string
+
+const (
+	RenderCSV RenderType = "csv"
+	RenderPNG RenderType = "png"
+)
 
 type Opts struct {
 	Width             int
 	Height            int
 	Timeout           time.Duration
-	OrgId             int64
-	UserId            int64
+	OrgID             int64
+	UserID            int64
 	OrgRole           models.RoleType
 	Path              string
 	Encoding          string
@@ -26,15 +34,34 @@ type Opts struct {
 	Headers           map[string][]string
 }
 
+type CSVOpts struct {
+	Timeout         time.Duration
+	OrgID           int64
+	UserID          int64
+	OrgRole         models.RoleType
+	Path            string
+	Encoding        string
+	Timezone        string
+	ConcurrentLimit int
+	Headers         map[string][]string
+}
+
 type RenderResult struct {
 	FilePath string
 }
 
+type RenderCSVResult struct {
+	FilePath string
+	FileName string
+}
+
 type renderFunc func(ctx context.Context, renderKey string, options Opts) (*RenderResult, error)
+type renderCSVFunc func(ctx context.Context, renderKey string, options CSVOpts) (*RenderCSVResult, error)
 
 type Service interface {
 	IsAvailable() bool
 	Render(ctx context.Context, opts Opts) (*RenderResult, error)
+	RenderCSV(ctx context.Context, opts CSVOpts) (*RenderCSVResult, error)
 	RenderErrorImage(error error) (*RenderResult, error)
 	GetRenderUser(key string) (*RenderUser, bool)
 }

pkg/services/rendering/plugin_mode.go

@@ -27,7 +27,7 @@ func (rs *RenderingService) renderViaPlugin(ctx context.Context, renderKey strin
 }
 
 func (rs *RenderingService) renderViaPluginV1(ctx context.Context, renderKey string, opts Opts) (*RenderResult, error) {
-	pngPath, err := rs.getFilePathForNewImage()
+	filePath, err := rs.getNewFilePath(RenderPNG)
 	if err != nil {
 		return nil, err
 	}
@@ -36,7 +36,7 @@ func (rs *RenderingService) renderViaPluginV1(ctx context.Context, renderKey str
 		Url:       rs.getURL(opts.Path),
 		Width:     int32(opts.Width),
 		Height:    int32(opts.Height),
-		FilePath:  pngPath,
+		FilePath:  filePath,
 		Timeout:   int32(opts.Timeout.Seconds()),
 		RenderKey: renderKey,
 		Encoding:  opts.Encoding,
@@ -57,11 +57,11 @@ func (rs *RenderingService) renderViaPluginV1(ctx context.Context, renderKey str
 		return nil, fmt.Errorf("rendering failed: %v", rsp.Error)
 	}
 
-	return &RenderResult{FilePath: pngPath}, nil
+	return &RenderResult{FilePath: filePath}, nil
 }
 
 func (rs *RenderingService) renderViaPluginV2(ctx context.Context, renderKey string, opts Opts) (*RenderResult, error) {
-	pngPath, err := rs.getFilePathForNewImage()
+	filePath, err := rs.getNewFilePath(RenderPNG)
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +79,7 @@ func (rs *RenderingService) renderViaPluginV2(ctx context.Context, renderKey str
 		Width:             int32(opts.Width),
 		Height:            int32(opts.Height),
 		DeviceScaleFactor: float32(opts.DeviceScaleFactor),
-		FilePath:          pngPath,
+		FilePath:          filePath,
 		Timeout:           int32(opts.Timeout.Seconds()),
 		RenderKey:         renderKey,
 		Timezone:          isoTimeOffsetToPosixTz(opts.Timezone),
@@ -100,5 +100,50 @@ func (rs *RenderingService) renderViaPluginV2(ctx context.Context, renderKey str
 		return nil, fmt.Errorf("rendering failed: %s", rsp.Error)
 	}
 
-	return &RenderResult{FilePath: pngPath}, err
+	return &RenderResult{FilePath: filePath}, err
+}
+
+func (rs *RenderingService) renderCSVViaPlugin(ctx context.Context, renderKey string, opts CSVOpts) (*RenderCSVResult, error) {
+	// gives plugin some additional time to timeout and return possible errors.
+	ctx, cancel := context.WithTimeout(ctx, opts.Timeout+time.Second*2)
+	defer cancel()
+
+	filePath, err := rs.getNewFilePath(RenderCSV)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := map[string]*pluginextensionv2.StringList{}
+	for k, values := range opts.Headers {
+		headers[k] = &pluginextensionv2.StringList{
+			Values: values,
+		}
+	}
+
+	req := &pluginextensionv2.RenderCSVRequest{
+		Url:       rs.getURL(opts.Path),
+		FilePath:  filePath,
+		RenderKey: renderKey,
+		Domain:    rs.domain,
+		Timeout:   int32(opts.Timeout.Seconds()),
+		Timezone:  isoTimeOffsetToPosixTz(opts.Timezone),
+		Headers:   headers,
+	}
+	rs.log.Debug("Calling renderer plugin", "req", req)
+
+	rsp, err := rs.pluginInfo.GrpcPluginV2.RenderCSV(ctx, req)
+	if err != nil {
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			rs.log.Info("Rendering timed out")
+			return nil, ErrTimeout
+		}
+
+		return nil, err
+	}
+
+	if rsp.Error != "" {
+		return nil, fmt.Errorf("rendering failed: %s", rsp.Error)
+	}
+
+	return &RenderCSVResult{FilePath: filePath, FileName: rsp.FileName}, nil
 }

pkg/services/rendering/rendering.go

@@ -44,6 +44,7 @@ type RenderingService struct {
 	log             log.Logger
 	pluginInfo      *plugins.RendererPlugin
 	renderAction    renderFunc
+	renderCSVAction renderCSVFunc
 	domain          string
 	inProgressCount int
 
@@ -61,6 +62,12 @@ func (rs *RenderingService) Init() error {
 		return fmt.Errorf("failed to create images directory %q: %w", rs.Cfg.ImagesDir, err)
 	}
 
+	// ensure CSVsDir exists
+	err = os.MkdirAll(rs.Cfg.CSVsDir, 0700)
+	if err != nil {
+		return fmt.Errorf("failed to create CSVs directory %q: %w", rs.Cfg.CSVsDir, err)
+	}
+
 	// set value used for domain attribute of renderKey cookie
 	switch {
 	case rs.Cfg.RendererUrl != "":
@@ -80,7 +87,8 @@ func (rs *RenderingService) Run(ctx context.Context) error {
 	if rs.remoteAvailable() {
 		rs.log = rs.log.New("renderer", "http")
 		rs.log.Info("Backend rendering via external http server")
-		rs.renderAction = rs.renderViaHttp
+		rs.renderAction = rs.renderViaHTTP
+		rs.renderCSVAction = rs.renderCSVViaHTTP
 		<-ctx.Done()
 		return nil
 	}
@@ -94,6 +102,7 @@ func (rs *RenderingService) Run(ctx context.Context) error {
 		}
 
 		rs.renderAction = rs.renderViaPlugin
+		rs.renderCSVAction = rs.renderCSVViaPlugin
 		<-ctx.Done()
 		return nil
 	}
@@ -136,23 +145,12 @@ func (rs *RenderingService) renderUnavailableImage() *RenderResult {
 
 func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResult, error) {
 	startTime := time.Now()
-	elapsedTime := time.Since(startTime).Milliseconds()
 	result, err := rs.render(ctx, opts)
-	if err != nil {
-		if errors.Is(err, ErrTimeout) {
-			metrics.MRenderingRequestTotal.WithLabelValues("timeout").Inc()
-			metrics.MRenderingSummary.WithLabelValues("timeout").Observe(float64(elapsedTime))
-		} else {
-			metrics.MRenderingRequestTotal.WithLabelValues("failure").Inc()
-			metrics.MRenderingSummary.WithLabelValues("failure").Observe(float64(elapsedTime))
-		}
 
-		return nil, err
-	}
+	elapsedTime := time.Since(startTime).Milliseconds()
+	saveMetrics(elapsedTime, err, RenderPNG)
 
-	metrics.MRenderingRequestTotal.WithLabelValues("success").Inc()
-	metrics.MRenderingSummary.WithLabelValues("success").Observe(float64(elapsedTime))
-	return result, nil
+	return result, err
 }
 
 func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResult, error) {
@@ -173,7 +171,7 @@ func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResul
 	if math.IsInf(opts.DeviceScaleFactor, 0) || math.IsNaN(opts.DeviceScaleFactor) || opts.DeviceScaleFactor <= 0 {
 		opts.DeviceScaleFactor = 1
 	}
-	renderKey, err := rs.generateAndStoreRenderKey(opts.OrgId, opts.UserId, opts.OrgRole)
+	renderKey, err := rs.generateAndStoreRenderKey(opts.OrgID, opts.UserID, opts.OrgRole)
 	if err != nil {
 		return nil, err
 	}
@@ -190,6 +188,43 @@ func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResul
 	return rs.renderAction(ctx, renderKey, opts)
 }
 
+func (rs *RenderingService) RenderCSV(ctx context.Context, opts CSVOpts) (*RenderCSVResult, error) {
+	startTime := time.Now()
+	result, err := rs.renderCSV(ctx, opts)
+
+	elapsedTime := time.Since(startTime).Milliseconds()
+	saveMetrics(elapsedTime, err, RenderCSV)
+
+	return result, err
+}
+
+func (rs *RenderingService) renderCSV(ctx context.Context, opts CSVOpts) (*RenderCSVResult, error) {
+	if rs.inProgressCount > opts.ConcurrentLimit {
+		return nil, ErrConcurrentLimitReached
+	}
+
+	if !rs.IsAvailable() {
+		return nil, ErrRenderUnavailable
+	}
+
+	rs.log.Info("Rendering", "path", opts.Path)
+	renderKey, err := rs.generateAndStoreRenderKey(opts.OrgID, opts.UserID, opts.OrgRole)
+	if err != nil {
+		return nil, err
+	}
+
+	defer rs.deleteRenderKey(renderKey)
+
+	defer func() {
+		rs.inProgressCount--
+		metrics.MRenderingQueue.Set(float64(rs.inProgressCount))
+	}()
+
+	rs.inProgressCount++
+	metrics.MRenderingQueue.Set(float64(rs.inProgressCount))
+	return rs.renderCSVAction(ctx, renderKey, opts)
+}
+
 func (rs *RenderingService) GetRenderUser(key string) (*RenderUser, bool) {
 	val, err := rs.RemoteCacheService.Get(fmt.Sprintf(renderKeyPrefix, key))
 	if err != nil {
@@ -205,17 +240,20 @@ func (rs *RenderingService) GetRenderUser(key string) (*RenderUser, bool) {
 	return nil, false
 }
 
-func (rs *RenderingService) getFilePathForNewImage() (string, error) {
+func (rs *RenderingService) getNewFilePath(rt RenderType) (string, error) {
 	rand, err := util.GetRandomString(20)
 	if err != nil {
 		return "", err
 	}
-	pngPath, err := filepath.Abs(filepath.Join(rs.Cfg.ImagesDir, rand))
-	if err != nil {
-		return "", err
+
+	ext := "png"
+	folder := rs.Cfg.ImagesDir
+	if rt == RenderCSV {
+		ext = "csv"
+		folder = rs.Cfg.CSVsDir
 	}
 
-	return pngPath + ".png", nil
+	return filepath.Abs(filepath.Join(folder, fmt.Sprintf("%s.%s", rand, ext)))
 }
 
 func (rs *RenderingService) getURL(path string) string {
@@ -282,3 +320,19 @@ func isoTimeOffsetToPosixTz(isoOffset string) string {
 	}
 	return isoOffset
 }
+
+func saveMetrics(elapsedTime int64, err error, renderType RenderType) {
+	if err == nil {
+		metrics.MRenderingRequestTotal.WithLabelValues("success", string(renderType)).Inc()
+		metrics.MRenderingSummary.WithLabelValues("success", string(renderType)).Observe(float64(elapsedTime))
+		return
+	}
+
+	if errors.Is(err, ErrTimeout) {
+		metrics.MRenderingRequestTotal.WithLabelValues("timeout", string(renderType)).Inc()
+		metrics.MRenderingSummary.WithLabelValues("timeout", string(renderType)).Observe(float64(elapsedTime))
+	} else {
+		metrics.MRenderingRequestTotal.WithLabelValues("failure", string(renderType)).Inc()
+		metrics.MRenderingSummary.WithLabelValues("failure", string(renderType)).Observe(float64(elapsedTime))
+	}
+}