Renderer: Add sanitize API (#50936)

* svg fun

* #50597: add proto

* #50597: add sanitizer methods

* #50597: add provider

* #50597: use sanitizer

* #50597: use sanitizer

* update grafana to match new api

* add comments

* add capability check

* add timing

* update sanitize path

* improve log message

* strings.HasPrefix rather than filepath.IsAbs

* filepath.Clean + filepath.ToSlash for windows

* read 404

* remove `path.clean` from `getPathAndScope`

* add resp body close

* remove unneeded prop

* Update pkg/services/rendering/rendering.go

Co-authored-by: Agnès Toulet <35176601+AgnesToulet@users.noreply.github.com>

* remove test files

* filepath.ToSlash correct wrapping

* filepath.ToSlash correct wrapping

* filepath.ToSlash comment

* compilation error

* lint fix

* fix error message

* Update pkg/services/rendering/rendering.go

Co-authored-by: Agnès Toulet <35176601+AgnesToulet@users.noreply.github.com>

* add `image/svg+xml` mime type

* refactored log

* refactored log

Co-authored-by: Agnès Toulet <35176601+AgnesToulet@users.noreply.github.com>
This commit is contained in:
Artur Wierzbicki
2022-07-07 15:32:18 +04:00
committed by GitHub
co-authored by Agnès Toulet
parent f279699beb
commit e96f67ae2e
21 changed files with 710 additions and 28 deletions
+4
View File
@@ -339,6 +339,10 @@ type testRenderService struct {
renderErrorImageProvider func(error error) (*rendering.RenderResult, error)
}
func (s *testRenderService) SanitizeSVG(ctx context.Context, req *rendering.SanitizeSVGRequest) (*rendering.SanitizeSVGResponse, error) {
return &rendering.SanitizeSVGResponse{Sanitized: req.Content}, nil
}
func (s *testRenderService) HasCapability(feature rendering.CapabilityName) (rendering.CapabilitySupportRequestResult, error) {
return rendering.CapabilitySupportRequestResult{}, nil
}
+1
View File
@@ -16,6 +16,7 @@ type CapabilityName string
const (
ScalingDownImages CapabilityName = "ScalingDownImages"
FullHeightImages CapabilityName = "FullHeightImages"
SvgSanitization CapabilityName = "SvgSanitization"
)
var ErrUnknownCapability = errors.New("unknown capability")
+11
View File
@@ -62,6 +62,15 @@ type ErrorOpts struct {
ErrorRenderUnavailable bool
}
type SanitizeSVGRequest struct {
Filename string
Content []byte
}
type SanitizeSVGResponse struct {
Sanitized []byte
}
type CSVOpts struct {
TimeoutOpts
AuthOpts
@@ -83,6 +92,7 @@ type RenderCSVResult struct {
type renderFunc func(ctx context.Context, renderKey string, options Opts) (*RenderResult, error)
type renderCSVFunc func(ctx context.Context, renderKey string, options CSVOpts) (*RenderCSVResult, error)
type sanitizeFunc func(ctx context.Context, req *SanitizeSVGRequest) (*SanitizeSVGResponse, error)
type renderKeyProvider interface {
get(ctx context.Context, opts AuthOpts) (string, error)
@@ -114,4 +124,5 @@ type Service interface {
GetRenderUser(ctx context.Context, key string) (*RenderUser, bool)
HasCapability(capability CapabilityName) (CapabilitySupportRequestResult, error)
CreateRenderingSession(ctx context.Context, authOpts AuthOpts, sessionOpts SessionOpts) (Session, error)
SanitizeSVG(ctx context.Context, req *SanitizeSVGRequest) (*SanitizeSVGResponse, error)
}
+15
View File
@@ -139,6 +139,21 @@ func (mr *MockServiceMockRecorder) RenderErrorImage(arg0, arg1 interface{}) *gom
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenderErrorImage", reflect.TypeOf((*MockService)(nil).RenderErrorImage), arg0, arg1)
}
// SanitizeSVG mocks base method.
func (m *MockService) SanitizeSVG(arg0 context.Context, arg1 *SanitizeSVGRequest) (*SanitizeSVGResponse, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SanitizeSVG", arg0, arg1)
ret0, _ := ret[0].(*SanitizeSVGResponse)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// SanitizeSVG indicates an expected call of SanitizeSVG.
func (mr *MockServiceMockRecorder) SanitizeSVG(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SanitizeSVG", reflect.TypeOf((*MockService)(nil).SanitizeSVG), arg0, arg1)
}
// Version mocks base method.
func (m *MockService) Version() string {
m.ctrl.T.Helper()
+53 -11
View File
@@ -27,18 +27,22 @@ func init() {
remotecache.Register(&RenderUser{})
}
var _ Service = (*RenderingService)(nil)
const ServiceName = "RenderingService"
type RenderingService struct {
log log.Logger
pluginInfo *plugins.Plugin
renderAction renderFunc
renderCSVAction renderCSVFunc
domain string
inProgressCount int32
version string
versionMutex sync.RWMutex
capabilities []Capability
log log.Logger
pluginInfo *plugins.Plugin
renderAction renderFunc
renderCSVAction renderCSVFunc
sanitizeSVGAction sanitizeFunc
sanitizeURL string
domain string
inProgressCount int32
version string
versionMutex sync.RWMutex
capabilities []Capability
perRequestRenderKeyProvider renderKeyProvider
Cfg *setting.Cfg
@@ -59,8 +63,14 @@ func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, rm p
return nil, fmt.Errorf("failed to create CSVs directory %q: %w", cfg.CSVsDir, err)
}
logger := log.New("rendering")
// URL for HTTP sanitize API
var sanitizeURL string
// value used for domain attribute of renderKey cookie
var domain string
// set value used for domain attribute of renderKey cookie
switch {
case cfg.RendererUrl != "":
// RendererCallbackUrl has already been passed, it won't generate an error.
@@ -69,6 +79,7 @@ func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, rm p
return nil, err
}
sanitizeURL = getSanitizerURL(cfg.RendererUrl)
domain = u.Hostname()
case cfg.HTTPAddr != setting.DefaultHTTPAddr:
domain = cfg.HTTPAddr
@@ -76,7 +87,6 @@ func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, rm p
domain = "localhost"
}
logger := log.New("rendering")
s := &RenderingService{
perRequestRenderKeyProvider: &perRequestRenderKeyProvider{
cache: remoteCache,
@@ -92,16 +102,26 @@ func ProvideService(cfg *setting.Cfg, remoteCache *remotecache.RemoteCache, rm p
name: ScalingDownImages,
semverConstraint: ">= 3.4.0",
},
{
name: SvgSanitization,
semverConstraint: ">= 3.5.0",
},
},
Cfg: cfg,
RemoteCacheService: remoteCache,
RendererPluginManager: rm,
log: logger,
domain: domain,
sanitizeURL: sanitizeURL,
}
return s, nil
}
func getSanitizerURL(rendererURL string) string {
rendererBaseURL := strings.TrimSuffix(rendererURL, "/render")
return rendererBaseURL + "/sanitize"
}
func (rs *RenderingService) Run(ctx context.Context) error {
if rs.remoteAvailable() {
rs.log = rs.log.New("renderer", "http")
@@ -120,6 +140,7 @@ func (rs *RenderingService) Run(ctx context.Context) error {
})
rs.renderAction = rs.renderViaHTTP
rs.renderCSVAction = rs.renderCSVViaHTTP
rs.sanitizeSVGAction = rs.sanitizeViaHTTP
refreshTicker := time.NewTicker(remoteVersionRefreshInterval)
@@ -146,6 +167,7 @@ func (rs *RenderingService) Run(ctx context.Context) error {
rs.version = rs.pluginInfo.Info.Version
rs.renderAction = rs.renderViaPlugin
rs.renderCSVAction = rs.renderCSVViaPlugin
rs.sanitizeSVGAction = rs.sanitizeSVGViaPlugin
<-ctx.Done()
// On Windows, Chromium is generating a debug.log file that breaks signature check on next restart
@@ -293,6 +315,26 @@ func (rs *RenderingService) RenderCSV(ctx context.Context, opts CSVOpts, session
return result, err
}
func (rs *RenderingService) SanitizeSVG(ctx context.Context, req *SanitizeSVGRequest) (*SanitizeSVGResponse, error) {
capability, err := rs.HasCapability(SvgSanitization)
if err != nil {
return nil, err
}
if !capability.IsSupported {
return nil, fmt.Errorf("svg sanitization unsupported, requires image renderer version: %s", capability.SemverConstraint)
}
start := time.Now()
action, err := rs.sanitizeSVGAction(ctx, req)
if err != nil {
defer rs.log.Info("svg sanitization finished", "duration", time.Since(start), "filename", req.Filename, "isError", err != nil)
}
return action, err
}
func (rs *RenderingService) renderCSV(ctx context.Context, opts CSVOpts, renderKeyProvider renderKeyProvider) (*RenderCSVResult, error) {
if int(atomic.LoadInt32(&rs.inProgressCount)) > opts.ConcurrentLimit {
return nil, ErrConcurrentLimitReached
+178
View File
@@ -0,0 +1,178 @@
package rendering
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"time"
"github.com/grafana/grafana/pkg/plugins/backendplugin/pluginextensionv2"
)
var (
domPurifySvgConfig = map[string]interface{}{
// domPurifyConfig is passed directly to DOMPurify https://github.com/cure53/DOMPurify#can-i-configure-dompurify
"domPurifyConfig": map[string]interface{}{
"USE_PROFILES": map[string]bool{"svg": true, "svgFilters": true},
"ADD_TAGS": []string{"use"},
},
// allowAllLinksInSvgUseTags will preserve all `use` tags.
// By default, we remove all non-self-referential `use` tags, i.e. those which `href` attribute does not start with `#`
"allowAllLinksInSvgUseTags": false,
}
domPurifyConfigType = "DOMPurify"
)
type formFile struct {
fileName string
key string
contentType string
content io.Reader
}
func createMultipartRequestBody(values []formFile) (bytes.Buffer, string, error) {
var b bytes.Buffer
w := multipart.NewWriter(&b)
for _, f := range values {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, f.key, f.fileName))
h.Set("Content-Type", f.contentType)
formWriter, err := w.CreatePart(h)
if err != nil {
return bytes.Buffer{}, "", err
}
if _, err := io.Copy(formWriter, f.content); err != nil {
return bytes.Buffer{}, "", err
}
if x, ok := f.content.(io.Closer); ok {
_ = x.Close()
}
}
if err := w.Close(); err != nil {
return bytes.Buffer{}, "", err
}
return b, w.FormDataContentType(), nil
}
func (rs *RenderingService) sanitizeViaHTTP(ctx context.Context, req *SanitizeSVGRequest) (*SanitizeSVGResponse, error) {
sanitizerUrl, err := url.Parse(rs.sanitizeURL)
if err != nil {
return nil, err
}
configJson, err := json.Marshal(map[string]interface{}{
"config": domPurifySvgConfig,
"configType": domPurifyConfigType,
})
if err != nil {
rs.log.Error("Sanitizer - HTTP: failed to create the request config", "error", err, "filename", req.Filename)
return nil, fmt.Errorf("config creation fail: %s", err)
}
body, contentType, err := createMultipartRequestBody([]formFile{
{
fileName: "config",
key: "config",
contentType: "application/json",
content: bytes.NewReader(configJson),
},
{
fileName: req.Filename,
key: "file",
contentType: "image/svg+xml",
content: bytes.NewReader(req.Content),
},
})
if err != nil {
rs.log.Error("Sanitizer - HTTP: failed to create the request body", "error", err, "filename", req.Filename)
return nil, fmt.Errorf("body creation fail: %s", err)
}
reqContext, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
httpReq, err := http.NewRequestWithContext(reqContext, "POST", sanitizerUrl.String(), &body)
if err != nil {
rs.log.Error("Sanitizer - HTTP: failed to create the HTTP request", "error", err, "filename", req.Filename)
return nil, err
}
httpReq.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", rs.Cfg.BuildVersion))
httpReq.Header.Set("Content-Type", contentType)
rs.log.Debug("Sanitizer - HTTP: calling", "filename", req.Filename, "contentLength", len(req.Content), "url", sanitizerUrl)
// make request to renderer server
resp, err := netClient.Do(httpReq)
if err != nil {
rs.log.Error("Sanitizer - HTTP: failed to send request", "error", err)
return nil, fmt.Errorf("sanitizer - HTTP: failed to send request: %w", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
rs.log.Error("Sanitizer - HTTP: failed to close response body", "statusCode", resp.StatusCode, "error", err)
}
}()
if resp.StatusCode != http.StatusOK {
if body, err := io.ReadAll(resp.Body); body != nil {
rs.log.Error("Sanitizer - HTTP: failed to sanitize", "statusCode", resp.StatusCode, "error", err, "resp", string(body))
} else {
rs.log.Error("Sanitizer - HTTP: failed to sanitize", "statusCode", resp.StatusCode, "error", err)
}
return nil, fmt.Errorf("sanitizer - HTTP: failed to sanitize %s", req.Filename)
}
sanitized, err := io.ReadAll(resp.Body)
if err != nil {
rs.log.Error("Sanitizer - HTTP: failed to read response body", "error", err, "filename", req.Filename)
return nil, fmt.Errorf("sanitizer - HTTP: failed to read response body: %s", err)
}
return &SanitizeSVGResponse{Sanitized: sanitized}, nil
}
func (rs *RenderingService) sanitizeSVGViaPlugin(ctx context.Context, req *SanitizeSVGRequest) (*SanitizeSVGResponse, error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*20)
defer cancel()
domPurifyConfig, err := json.Marshal(domPurifySvgConfig)
if err != nil {
rs.log.Error("Sanitizer - plugin: failed to parse domPurifyConfig")
return nil, fmt.Errorf("sanitizer - plugin: failed to parse domPurifyConfig %s", err)
}
grpcReq := &pluginextensionv2.SanitizeRequest{
Filename: req.Filename,
Content: req.Content,
ConfigType: domPurifyConfigType,
Config: domPurifyConfig,
}
rs.log.Debug("Sanitizer - plugin: calling", "filename", req.Filename, "contentLength", len(req.Content))
rsp, err := rs.pluginInfo.Renderer.Sanitize(ctx, grpcReq)
if err != nil {
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
rs.log.Info("Sanitizer - plugin: time out")
return nil, ErrTimeout
}
return nil, err
}
if rsp.Error != "" {
return nil, fmt.Errorf("sanitizer - plugin: failed to sanitize: %s", rsp.Error)
}
return &SanitizeSVGResponse{Sanitized: rsp.Sanitized}, nil
}
+5
View File
@@ -126,6 +126,11 @@ func (s *httpStorage) Read(c *models.ReqContext) response.Response {
if err != nil {
return response.Error(400, "cannot call read", err)
}
if file == nil || file.Contents == nil {
return response.Error(404, "file does not exist", err)
}
// set the correct content type for svg
if strings.HasSuffix(path, ".svg") {
c.Resp.Header().Set("Content-Type", "image/svg+xml")
+28 -4
View File
@@ -6,20 +6,44 @@ import (
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/services/store/sanitizer"
)
func (s *standardStorageService) sanitizeUploadRequest(ctx context.Context, user *models.SignedInUser, req *UploadRequest, storagePath string) (*filestorage.UpsertFileCommand, error) {
func (s *standardStorageService) sanitizeContents(ctx context.Context, user *models.SignedInUser, req *UploadRequest, storagePath string) ([]byte, error) {
if req.EntityType == EntityTypeImage {
ext := filepath.Ext(req.Path)
//nolint: staticcheck
if ext == ".svg" {
// TODO: sanitize svg
resp, err := sanitizer.SanitizeSVG(ctx, &rendering.SanitizeSVGRequest{
Filename: storagePath,
Content: req.Contents,
})
if err != nil {
if s.cfg.allowUnsanitizedSvgUpload {
grafanaStorageLogger.Debug("allowing unsanitized svg upload", "filename", req.Path, "sanitizationError", err)
return req.Contents, nil
} else {
grafanaStorageLogger.Debug("disallowing unsanitized svg upload", "filename", req.Path, "sanitizationError", err)
return nil, err
}
}
return resp.Sanitized, nil
}
}
return req.Contents, nil
}
func (s *standardStorageService) sanitizeUploadRequest(ctx context.Context, user *models.SignedInUser, req *UploadRequest, storagePath string) (*filestorage.UpsertFileCommand, error) {
contents, err := s.sanitizeContents(ctx, user, req, storagePath)
if err != nil {
return nil, err
}
return &filestorage.UpsertFileCommand{
Path: storagePath,
Contents: req.Contents,
Contents: contents,
MimeType: req.MimeType,
CacheControl: req.CacheControl,
ContentDisposition: req.ContentDisposition,
+23
View File
@@ -0,0 +1,23 @@
package sanitizer
import (
"context"
"errors"
"github.com/grafana/grafana/pkg/services/rendering"
)
// workaround for cyclic dep between the store and the renderer
type Provider struct{}
var SanitizeSVG = func(ctx context.Context, req *rendering.SanitizeSVGRequest) (*rendering.SanitizeSVGResponse, error) {
return nil, errors.New("not implemented")
}
func ProvideService(
renderer rendering.Service,
) *Provider {
SanitizeSVG = renderer.SanitizeSVG
return &Provider{}
}
+8
View File
@@ -48,9 +48,14 @@ type StorageService interface {
sanitizeUploadRequest(ctx context.Context, user *models.SignedInUser, req *UploadRequest, storagePath string) (*filestorage.UpsertFileCommand, error)
}
type storageServiceConfig struct {
allowUnsanitizedSvgUpload bool
}
type standardStorageService struct {
sql *sqlstore.SQLStore
tree *nestedTree
cfg storageServiceConfig
}
func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService {
@@ -83,6 +88,9 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
s := newStandardStorageService(globalRoots, initializeOrgStorages)
s.sql = sql
s.cfg = storageServiceConfig{
allowUnsanitizedSvgUpload: false,
}
return s
}
+1 -2
View File
@@ -1,7 +1,6 @@
package store
import (
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/models"
@@ -27,5 +26,5 @@ func getPathAndScope(c *models.ReqContext) (string, string) {
if path == "" {
return "", ""
}
return splitFirstSegment(filepath.Clean(path))
return splitFirstSegment(path)
}
+3 -1
View File
@@ -13,6 +13,7 @@ var (
allowedImageExtensions = map[string]bool{
".jpg": true,
".jpeg": true,
".svg": true,
".gif": true,
".png": true,
".webp": true,
@@ -23,6 +24,7 @@ var (
".gif": {"image/gif": true},
".png": {"image/png": true},
".webp": {"image/webp": true},
".svg": {"text/xml; charset=utf-8": true, "text/plain; charset=utf-8": true, "image/svg+xml": true},
}
)
@@ -68,7 +70,7 @@ func (s *standardStorageService) validateUploadRequest(ctx context.Context, user
// TODO: validateProperties
if err := filestorage.ValidatePath(storagePath); err != nil {
return fail("path validation failed: " + err.Error())
return fail("path validation failed. error:" + err.Error() + ". path: " + storagePath)
}
switch req.EntityType {