Plugins: Remove connection/hop-by-hop request/response headers for call resource (#60077)

Removes request/response connection/hop headers for call resource in similar 
manner as Go's reverse proxy functions. Also removes Prometheus datasource 
custom call resource header manipulation in regards to hop-by-hop headers.

Fixes #60076
Ref #58646

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

pkg/plugins/manager/client/client.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/textproto"
+	"strings"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 
@@ -82,7 +84,19 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
 		return backendplugin.ErrPluginNotRegistered
 	}
 	err := instrumentation.InstrumentCallResourceRequest(ctx, &req.PluginContext, s.cfg, func() error {
-		if err := p.CallResource(ctx, req, sender); err != nil {
+		removeConnectionHeaders(req.Headers)
+		removeHopByHopHeaders(req.Headers)
+
+		wrappedSender := callResourceResponseSenderFunc(func(res *backend.CallResourceResponse) error {
+			if res != nil && len(res.Headers) > 0 {
+				removeConnectionHeaders(res.Headers)
+				removeHopByHopHeaders(res.Headers)
+			}
+
+			return sender.Send(res)
+		})
+
+		if err := p.CallResource(ctx, req, wrappedSender); err != nil {
 			return err
 		}
 		return nil
@@ -204,3 +218,63 @@ func (s *Service) plugin(ctx context.Context, pluginID string) (*plugins.Plugin,
 
 	return p, true
 }
+
+// removeConnectionHeaders removes hop-by-hop headers listed in the "Connection" header of h.
+// See RFC 7230, section 6.1
+//
+// Based on https://github.com/golang/go/blob/dc04f3ba1f25313bc9c97e728620206c235db9ee/src/net/http/httputil/reverseproxy.go#L411-L421
+func removeConnectionHeaders(h map[string][]string) {
+	for _, f := range h["Connection"] {
+		for _, sf := range strings.Split(f, ",") {
+			if sf = textproto.TrimString(sf); sf != "" {
+				for k := range h {
+					if textproto.CanonicalMIMEHeaderKey(sf) == textproto.CanonicalMIMEHeaderKey(k) {
+						delete(h, k)
+						break
+					}
+				}
+			}
+		}
+	}
+}
+
+// Hop-by-hop headers. These are removed when sent to the backend.
+// As of RFC 7230, hop-by-hop headers are required to appear in the
+// Connection header field. These are the headers defined by the
+// obsoleted RFC 2616 (section 13.5.1) and are used for backward
+// compatibility.
+//
+// Copied from https://github.com/golang/go/blob/dc04f3ba1f25313bc9c97e728620206c235db9ee/src/net/http/httputil/reverseproxy.go#L171-L186
+var hopHeaders = []string{
+	"Connection",
+	"Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google
+	"Keep-Alive",
+	"Proxy-Authenticate",
+	"Proxy-Authorization",
+	"Te",      // canonicalized version of "TE"
+	"Trailer", // not Trailers per URL above; https://www.rfc-editor.org/errata_search.php?eid=4522
+	"Transfer-Encoding",
+	"Upgrade",
+}
+
+// removeHopByHopHeaders removes hop-by-hop headers. Especially
+// important is "Connection" because we want a persistent
+// connection, regardless of what the client sent to us.
+//
+// Based on https://github.com/golang/go/blob/dc04f3ba1f25313bc9c97e728620206c235db9ee/src/net/http/httputil/reverseproxy.go#L276-L281
+func removeHopByHopHeaders(h map[string][]string) {
+	for _, hh := range hopHeaders {
+		for k := range h {
+			if hh == textproto.CanonicalMIMEHeaderKey(k) {
+				delete(h, k)
+				break
+			}
+		}
+	}
+}
+
+type callResourceResponseSenderFunc func(res *backend.CallResourceResponse) error
+
+func (fn callResourceResponseSenderFunc) Send(res *backend.CallResourceResponse) error {
+	return fn(res)
+}