Instrumentation: Proxy status code correction and various improvements (#47473)

For a proxied request, e.g. Grafana's datasource or plugin proxy:
If the request is cancelled, e.g. from the browser, the HTTP status code is 
now 499 Client closed request instead of 502 Bad gateway.
If the request times out, e.g. takes longer time than allowed, the HTTP status 
code is now 504 Gateway timeout instead of 502 Bad gateway.
This also means that request metrics and logs will get their status codes 
adjusted according to above.

Fixes #46337
Fixes #46338

pkg/api/pluginproxy/ds_proxy.go

@@ -5,9 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
-	"net/http/httputil"
 	"net/url"
 	"strconv"
 	"strings"
@@ -48,35 +46,10 @@ type DataSourceProxy struct {
 	secretsService     secrets.Service
 }
 
-type handleResponseTransport struct {
-	transport http.RoundTripper
-}
-
-func (t *handleResponseTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	res, err := t.transport.RoundTrip(req)
-	if err != nil {
-		return nil, err
-	}
-	res.Header.Del("Set-Cookie")
-	proxyutil.SetProxyResponseHeaders(res.Header)
-	return res, nil
-}
-
 type httpClient interface {
 	Do(req *http.Request) (*http.Response, error)
 }
 
-type logWrapper struct {
-	logger glog.Logger
-}
-
-// Write writes log messages as bytes from proxy
-func (lw *logWrapper) Write(p []byte) (n int, err error) {
-	withoutNewline := strings.TrimSuffix(string(p), "\n")
-	lw.logger.Error("Data proxy error", "error", withoutNewline)
-	return len(p), nil
-}
-
 // NewDataSourceProxy creates a new Datasource proxy
 func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext,
 	proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider,
@@ -115,8 +88,14 @@ func (proxy *DataSourceProxy) HandleRequest() {
 		return
 	}
 
-	proxyErrorLogger := logger.New("userId", proxy.ctx.UserId, "orgId", proxy.ctx.OrgId, "uname", proxy.ctx.Login,
-		"path", proxy.ctx.Req.URL.Path, "remote_addr", proxy.ctx.RemoteAddr(), "referer", proxy.ctx.Req.Referer())
+	proxyErrorLogger := logger.New(
+		"userId", proxy.ctx.UserId,
+		"orgId", proxy.ctx.OrgId,
+		"uname", proxy.ctx.Login,
+		"path", proxy.ctx.Req.URL.Path,
+		"remote_addr", proxy.ctx.RemoteAddr(),
+		"referer", proxy.ctx.Req.Referer(),
+	)
 
 	transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ds, proxy.clientProvider)
 	if err != nil {
@@ -124,36 +103,36 @@ func (proxy *DataSourceProxy) HandleRequest() {
 		return
 	}
 
-	reverseProxy := &httputil.ReverseProxy{
-		Director:      proxy.director,
-		FlushInterval: time.Millisecond * 200,
-		ErrorLog:      log.New(&logWrapper{logger: proxyErrorLogger}, "", 0),
-		Transport: &handleResponseTransport{
-			transport: transport,
-		},
-		ModifyResponse: func(resp *http.Response) error {
-			if resp.StatusCode == 401 {
-				// The data source rejected the request as unauthorized, convert to 400 (bad request)
-				body, err := ioutil.ReadAll(resp.Body)
-				if err != nil {
-					return fmt.Errorf("failed to read data source response body: %w", err)
-				}
-				_ = resp.Body.Close()
-
-				proxyErrorLogger.Info("Authentication to data source failed", "body", string(body), "statusCode",
-					resp.StatusCode)
-				msg := "Authentication to data source failed"
-				*resp = http.Response{
-					StatusCode:    400,
-					Status:        "Bad Request",
-					Body:          ioutil.NopCloser(strings.NewReader(msg)),
-					ContentLength: int64(len(msg)),
-				}
+	modifyResponse := func(resp *http.Response) error {
+		if resp.StatusCode == 401 {
+			// The data source rejected the request as unauthorized, convert to 400 (bad request)
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return fmt.Errorf("failed to read data source response body: %w", err)
 			}
-			return nil
-		},
+			_ = resp.Body.Close()
+
+			proxyErrorLogger.Info("Authentication to data source failed", "body", string(body), "statusCode",
+				resp.StatusCode)
+			msg := "Authentication to data source failed"
+			*resp = http.Response{
+				StatusCode:    400,
+				Status:        "Bad Request",
+				Body:          ioutil.NopCloser(strings.NewReader(msg)),
+				ContentLength: int64(len(msg)),
+				Header:        http.Header{},
+			}
+		}
+		return nil
 	}
 
+	reverseProxy := proxyutil.NewReverseProxy(
+		proxyErrorLogger,
+		proxy.director,
+		proxyutil.WithTransport(transport),
+		proxyutil.WithModifyResponse(modifyResponse),
+	)
+
 	proxy.logRequest()
 	ctx, span := proxy.tracer.Start(proxy.ctx.Req.Context(), "datasource reverse proxy")
 	defer span.End()
@@ -236,14 +215,8 @@ func (proxy *DataSourceProxy) director(req *http.Request) {
 	}
 
 	proxyutil.ClearCookieHeader(req, keepCookieNames)
-	proxyutil.PrepareProxyRequest(req)
-
 	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
 
-	// Clear Origin and Referer to avoid CORS issues
-	req.Header.Del("Origin")
-	req.Header.Del("Referer")
-
 	jsonData := make(map[string]interface{})
 	if proxy.ds.JsonData != nil {
 		jsonData, err = proxy.ds.JsonData.Map()