Plugins: Automatically forward plugin request HTTP headers in outgoing HTTP requests (#60417)

Automatically forward core plugin request HTTP headers in outgoing HTTP requests. 
Core datasource plugin authors don't have to specifically handle forwarding of HTTP 
headers, e.g. do not have to "hardcode" the header-names in the datasource plugin, 
if not having custom needs.

Fixes #57065

pkg/services/pluginsintegration/clientmiddleware/clear_auth_headers_middleware.go

@@ -4,8 +4,6 @@ import (
 	"context"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 )
@@ -25,19 +23,19 @@ type ClearAuthHeadersMiddleware struct {
 	next plugins.Client
 }
 
-func (m *ClearAuthHeadersMiddleware) clearHeaders(ctx context.Context, pCtx backend.PluginContext, req interface{}) context.Context {
+func (m *ClearAuthHeadersMiddleware) clearHeaders(ctx context.Context, h backend.ForwardHTTPHeaders) {
 	reqCtx := contexthandler.FromContext(ctx)
 	// if no HTTP request context skip middleware
-	if req == nil || reqCtx == nil || reqCtx.Req == nil || reqCtx.SignedInUser == nil {
-		return ctx
+	if h == nil || reqCtx == nil || reqCtx.Req == nil || reqCtx.SignedInUser == nil {
+		return
 	}
 
 	list := contexthandler.AuthHTTPHeaderListFromContext(ctx)
 	if list != nil {
-		ctx = sdkhttpclient.WithContextualMiddleware(ctx, httpclientprovider.DeleteHeadersMiddleware(list.Items...))
+		for _, k := range list.Items {
+			h.DeleteHTTPHeader(k)
+		}
 	}
-
-	return ctx
 }
 
 func (m *ClearAuthHeadersMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -45,7 +43,7 @@ func (m *ClearAuthHeadersMiddleware) QueryData(ctx context.Context, req *backend
 		return m.next.QueryData(ctx, req)
 	}
 
-	ctx = m.clearHeaders(ctx, req.PluginContext, req)
+	m.clearHeaders(ctx, req)
 
 	return m.next.QueryData(ctx, req)
 }
@@ -55,7 +53,7 @@ func (m *ClearAuthHeadersMiddleware) CallResource(ctx context.Context, req *back
 		return m.next.CallResource(ctx, req, sender)
 	}
 
-	ctx = m.clearHeaders(ctx, req.PluginContext, req)
+	m.clearHeaders(ctx, req)
 
 	return m.next.CallResource(ctx, req, sender)
 }
@@ -65,7 +63,7 @@ func (m *ClearAuthHeadersMiddleware) CheckHealth(ctx context.Context, req *backe
 		return m.next.CheckHealth(ctx, req)
 	}
 
-	ctx = m.clearHeaders(ctx, req.PluginContext, req)
+	m.clearHeaders(ctx, req)
 
 	return m.next.CheckHealth(ctx, req)
 }

pkg/services/pluginsintegration/clientmiddleware/clear_auth_headers_middleware_test.go

@@ -1,14 +1,10 @@
 package clientmiddleware
 
 import (
-	"bytes"
-	"io"
 	"net/http"
 	"testing"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins/manager/client/clienttest"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/services/user"
@@ -42,9 +38,6 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Len(t, cdt.QueryDataReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 0)
 			})
 
 			t.Run("Should not attach delete headers middleware when calling CallResource", func(t *testing.T) {
@@ -55,9 +48,6 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Len(t, cdt.CallResourceReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 0)
 			})
 
 			t.Run("Should not attach delete headers middleware when calling CheckHealth", func(t *testing.T) {
@@ -68,9 +58,6 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Len(t, cdt.CheckHealthReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 0)
 			})
 		})
 
@@ -92,9 +79,6 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Len(t, cdt.QueryDataReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 0)
 			})
 
 			t.Run("Should not attach delete headers middleware when calling CallResource", func(t *testing.T) {
@@ -105,9 +89,6 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Len(t, cdt.CallResourceReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 0)
 			})
 
 			t.Run("Should not attach delete headers middleware when calling CheckHealth", func(t *testing.T) {
@@ -118,9 +99,6 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Len(t, cdt.CheckHealthReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 0)
 			})
 		})
 	})
@@ -155,18 +133,7 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Len(t, cdt.QueryDataReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-				reqClone := req.Clone(req.Context())
-				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-				require.NoError(t, err)
-				require.NoError(t, res.Body.Close())
-				require.Len(t, reqClone.Header, 1)
-				require.Empty(t, reqClone.Header[customHeader])
-				require.Equal(t, "test", reqClone.Header.Get(otherHeader))
+				require.Equal(t, "test", cdt.QueryDataReq.Headers[otherHeader])
 			})
 
 			t.Run("Should attach delete headers middleware when calling CallResource", func(t *testing.T) {
@@ -177,18 +144,7 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Len(t, cdt.CallResourceReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-				reqClone := req.Clone(req.Context())
-				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-				require.NoError(t, err)
-				require.NoError(t, res.Body.Close())
-				require.Len(t, reqClone.Header, 1)
-				require.Empty(t, reqClone.Header[customHeader])
-				require.Equal(t, "test", reqClone.Header.Get(otherHeader))
+				require.Equal(t, []string{"test"}, cdt.CallResourceReq.Headers[otherHeader])
 			})
 
 			t.Run("Should attach delete headers middleware when calling CheckHealth", func(t *testing.T) {
@@ -199,18 +155,7 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Len(t, cdt.CheckHealthReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-				reqClone := req.Clone(req.Context())
-				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-				require.NoError(t, err)
-				require.NoError(t, res.Body.Close())
-				require.Len(t, reqClone.Header, 1)
-				require.Empty(t, reqClone.Header[customHeader])
-				require.Equal(t, "test", reqClone.Header.Get(otherHeader))
+				require.Equal(t, "test", cdt.CheckHealthReq.Headers[otherHeader])
 			})
 		})
 
@@ -220,12 +165,12 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				clienttest.WithMiddlewares(NewClearAuthHeadersMiddleware()),
 			)
 
-			const customHeader = "X-Custom"
+			const customHeader = "x-Custom"
 			req.Header.Set(customHeader, "val")
 			ctx := contexthandler.WithAuthHTTPHeader(req.Context(), customHeader)
 			req = req.WithContext(ctx)
 
-			const otherHeader = "X-Other"
+			const otherHeader = "x-Other"
 			req.Header.Set(otherHeader, "test")
 
 			pluginCtx := backend.PluginContext{
@@ -240,18 +185,7 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Len(t, cdt.QueryDataReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-				reqClone := req.Clone(req.Context())
-				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-				require.NoError(t, err)
-				require.NoError(t, res.Body.Close())
-				require.Len(t, reqClone.Header, 1)
-				require.Empty(t, reqClone.Header[customHeader])
-				require.Equal(t, "test", reqClone.Header.Get(otherHeader))
+				require.Equal(t, "test", cdt.QueryDataReq.Headers[otherHeader])
 			})
 
 			t.Run("Should attach delete headers middleware when calling CallResource", func(t *testing.T) {
@@ -262,18 +196,7 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Len(t, cdt.CallResourceReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-				reqClone := req.Clone(req.Context())
-				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-				require.NoError(t, err)
-				require.NoError(t, res.Body.Close())
-				require.Len(t, reqClone.Header, 1)
-				require.Empty(t, reqClone.Header[customHeader])
-				require.Equal(t, "test", reqClone.Header.Get(otherHeader))
+				require.Equal(t, []string{"test"}, cdt.CallResourceReq.Headers[otherHeader])
 			})
 
 			t.Run("Should attach delete headers middleware when calling CheckHealth", func(t *testing.T) {
@@ -284,27 +207,8 @@ func TestClearAuthHeadersMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Len(t, cdt.CheckHealthReq.Headers, 1)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-				reqClone := req.Clone(req.Context())
-				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-				require.NoError(t, err)
-				require.NoError(t, res.Body.Close())
-				require.Len(t, reqClone.Header, 1)
-				require.Empty(t, reqClone.Header[customHeader])
-				require.Equal(t, "test", reqClone.Header.Get(otherHeader))
+				require.Equal(t, "test", cdt.CheckHealthReq.Headers[otherHeader])
 			})
 		})
 	})
 }
-
-var finalRoundTripper = httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
-	return &http.Response{
-		StatusCode: http.StatusOK,
-		Request:    req,
-		Body:       io.NopCloser(bytes.NewBufferString("")),
-	}, nil
-})

pkg/services/pluginsintegration/clientmiddleware/cookies_middleware.go

@@ -4,9 +4,7 @@ import (
 	"context"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/services/datasources"
@@ -16,8 +14,8 @@ import (
 const cookieHeaderName = "Cookie"
 
 // NewCookiesMiddleware creates a new plugins.ClientMiddleware that will
-// forward incoming HTTP request Cookies to outgoing plugins.Client and
-// HTTP requests if the datasource has enabled forwarding of cookies (keepCookies).
+// forward incoming HTTP request Cookies to outgoing plugins.Client requests
+// if the datasource has enabled forwarding of cookies (keepCookies).
 func NewCookiesMiddleware(skipCookiesNames []string) plugins.ClientMiddleware {
 	return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client {
 		return &CookiesMiddleware{
@@ -32,17 +30,17 @@ type CookiesMiddleware struct {
 	skipCookiesNames []string
 }
 
-func (m *CookiesMiddleware) applyCookies(ctx context.Context, pCtx backend.PluginContext, req interface{}) (context.Context, error) {
+func (m *CookiesMiddleware) applyCookies(ctx context.Context, pCtx backend.PluginContext, req interface{}) error {
 	reqCtx := contexthandler.FromContext(ctx)
 	// if request not for a datasource or no HTTP request context skip middleware
 	if req == nil || pCtx.DataSourceInstanceSettings == nil || reqCtx == nil || reqCtx.Req == nil {
-		return ctx, nil
+		return nil
 	}
 
 	settings := pCtx.DataSourceInstanceSettings
 	jsonDataBytes, err := simplejson.NewJson(settings.JSONData)
 	if err != nil {
-		return ctx, err
+		return err
 	}
 
 	ds := &datasources.DataSource{
@@ -54,20 +52,29 @@ func (m *CookiesMiddleware) applyCookies(ctx context.Context, pCtx backend.Plugi
 
 	proxyutil.ClearCookieHeader(reqCtx.Req, ds.AllowedCookies(), m.skipCookiesNames)
 
-	if cookieStr := reqCtx.Req.Header.Get(cookieHeaderName); cookieStr != "" {
-		switch t := req.(type) {
-		case *backend.QueryDataRequest:
+	cookieStr := reqCtx.Req.Header.Get(cookieHeaderName)
+	switch t := req.(type) {
+	case *backend.QueryDataRequest:
+		if cookieStr == "" {
+			delete(t.Headers, cookieHeaderName)
+		} else {
 			t.Headers[cookieHeaderName] = cookieStr
-		case *backend.CheckHealthRequest:
+		}
+	case *backend.CheckHealthRequest:
+		if cookieStr == "" {
+			delete(t.Headers, cookieHeaderName)
+		} else {
 			t.Headers[cookieHeaderName] = cookieStr
-		case *backend.CallResourceRequest:
+		}
+	case *backend.CallResourceRequest:
+		if cookieStr == "" {
+			delete(t.Headers, cookieHeaderName)
+		} else {
 			t.Headers[cookieHeaderName] = []string{cookieStr}
 		}
 	}
 
-	ctx = httpclient.WithContextualMiddleware(ctx, httpclientprovider.ForwardedCookiesMiddleware(reqCtx.Req.Cookies(), ds.AllowedCookies(), m.skipCookiesNames))
-
-	return ctx, nil
+	return nil
 }
 
 func (m *CookiesMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -75,12 +82,12 @@ func (m *CookiesMiddleware) QueryData(ctx context.Context, req *backend.QueryDat
 		return m.next.QueryData(ctx, req)
 	}
 
-	newCtx, err := m.applyCookies(ctx, req.PluginContext, req)
+	err := m.applyCookies(ctx, req.PluginContext, req)
 	if err != nil {
 		return nil, err
 	}
 
-	return m.next.QueryData(newCtx, req)
+	return m.next.QueryData(ctx, req)
 }
 
 func (m *CookiesMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
@@ -88,12 +95,12 @@ func (m *CookiesMiddleware) CallResource(ctx context.Context, req *backend.CallR
 		return m.next.CallResource(ctx, req, sender)
 	}
 
-	newCtx, err := m.applyCookies(ctx, req.PluginContext, req)
+	err := m.applyCookies(ctx, req.PluginContext, req)
 	if err != nil {
 		return err
 	}
 
-	return m.next.CallResource(newCtx, req, sender)
+	return m.next.CallResource(ctx, req, sender)
 }
 
 func (m *CookiesMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
@@ -101,12 +108,12 @@ func (m *CookiesMiddleware) CheckHealth(ctx context.Context, req *backend.CheckH
 		return m.next.CheckHealth(ctx, req)
 	}
 
-	newCtx, err := m.applyCookies(ctx, req.PluginContext, req)
+	err := m.applyCookies(ctx, req.PluginContext, req)
 	if err != nil {
 		return nil, err
 	}
 
-	return m.next.CheckHealth(newCtx, req)
+	return m.next.CheckHealth(ctx, req)
 }
 
 func (m *CookiesMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) {

pkg/services/pluginsintegration/clientmiddleware/cookies_middleware_test.go

@@ -6,8 +6,6 @@ import (
 	"testing"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins/manager/client/clienttest"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/stretchr/testify/require"
@@ -54,39 +52,19 @@ func TestCookiesMiddleware(t *testing.T) {
 			require.NotNil(t, cdt.QueryDataReq)
 			require.Len(t, cdt.QueryDataReq.Headers, 1)
 			require.Equal(t, "test", cdt.QueryDataReq.Headers[otherHeader])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.ForwardedCookiesMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 1)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
 		})
 
 		t.Run("Should not forward cookies when calling CallResource", func(t *testing.T) {
-			err = cdt.Decorator.CallResource(req.Context(), &backend.CallResourceRequest{
+			pReq := &backend.CallResourceRequest{
 				PluginContext: pluginCtx,
 				Headers:       map[string][]string{otherHeader: {"test"}},
-			}, nopCallResourceSender)
+			}
+			pReq.Headers[backend.CookiesHeaderName] = []string{req.Header.Get(backend.CookiesHeaderName)}
+			err = cdt.Decorator.CallResource(req.Context(), pReq, nopCallResourceSender)
 			require.NoError(t, err)
 			require.NotNil(t, cdt.CallResourceReq)
 			require.Len(t, cdt.CallResourceReq.Headers, 1)
 			require.Equal(t, "test", cdt.CallResourceReq.Headers[otherHeader][0])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.ForwardedCookiesMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 1)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
 		})
 
 		t.Run("Should not forward cookies when calling CheckHealth", func(t *testing.T) {
@@ -98,17 +76,6 @@ func TestCookiesMiddleware(t *testing.T) {
 			require.NotNil(t, cdt.CheckHealthReq)
 			require.Len(t, cdt.CheckHealthReq.Headers, 1)
 			require.Equal(t, "test", cdt.CheckHealthReq.Headers[otherHeader])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.ForwardedCookiesMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 1)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
 		})
 	})
 
@@ -157,18 +124,6 @@ func TestCookiesMiddleware(t *testing.T) {
 			require.Len(t, cdt.QueryDataReq.Headers, 2)
 			require.Equal(t, "test", cdt.QueryDataReq.Headers[otherHeader])
 			require.EqualValues(t, "cookie2=", cdt.QueryDataReq.Headers[cookieHeaderName])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.ForwardedCookiesMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 2)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
-			require.Equal(t, "cookie2=", reqClone.Header.Get(cookieHeaderName))
 		})
 
 		t.Run("Should forward cookies when calling CallResource", func(t *testing.T) {
@@ -182,18 +137,6 @@ func TestCookiesMiddleware(t *testing.T) {
 			require.Equal(t, "test", cdt.CallResourceReq.Headers[otherHeader][0])
 			require.Len(t, cdt.CallResourceReq.Headers[cookieHeaderName], 1)
 			require.EqualValues(t, "cookie2=", cdt.CallResourceReq.Headers[cookieHeaderName][0])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.ForwardedCookiesMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 2)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
-			require.Equal(t, "cookie2=", reqClone.Header.Get(cookieHeaderName))
 		})
 
 		t.Run("Should forward cookies when calling CheckHealth", func(t *testing.T) {
@@ -206,18 +149,6 @@ func TestCookiesMiddleware(t *testing.T) {
 			require.Len(t, cdt.CheckHealthReq.Headers, 2)
 			require.Equal(t, "test", cdt.CheckHealthReq.Headers[otherHeader])
 			require.EqualValues(t, "cookie2=", cdt.CheckHealthReq.Headers[cookieHeaderName])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.ForwardedCookiesMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 2)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
-			require.Equal(t, "cookie2=", reqClone.Header.Get(cookieHeaderName))
 		})
 	})
 }

pkg/services/pluginsintegration/clientmiddleware/httpclient_middleware.go

@@ -0,0 +1,108 @@
+package clientmiddleware
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/plugins"
+	ngalertmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+)
+
+const forwardPluginRequestHTTPHeaders = "forward-plugin-request-http-headers"
+
+// NewHTTPClientMiddleware creates a new plugins.ClientMiddleware
+// that will forward plugin request headers as outgoing HTTP headers.
+func NewHTTPClientMiddleware() plugins.ClientMiddleware {
+	return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client {
+		return &HTTPClientMiddleware{
+			next: next,
+		}
+	})
+}
+
+type HTTPClientMiddleware struct {
+	next plugins.Client
+}
+
+func (m *HTTPClientMiddleware) applyHeaders(ctx context.Context, pReq interface{}) context.Context {
+	if pReq == nil {
+		return ctx
+	}
+
+	mw := httpclient.NamedMiddlewareFunc(forwardPluginRequestHTTPHeaders, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+		return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			switch t := pReq.(type) {
+			case *backend.QueryDataRequest:
+				if val, exists := t.Headers[ngalertmodels.FromAlertHeaderName]; exists {
+					req.Header.Set(ngalertmodels.FromAlertHeaderName, val)
+				}
+			case *backend.CallResourceRequest:
+				if val, exists := t.Headers[ngalertmodels.FromAlertHeaderName]; exists {
+					req.Header.Set(ngalertmodels.FromAlertHeaderName, val[0])
+				}
+			case *backend.CheckHealthRequest:
+				if val, exists := t.Headers[ngalertmodels.FromAlertHeaderName]; exists {
+					req.Header.Set(ngalertmodels.FromAlertHeaderName, val)
+				}
+			}
+
+			if h, ok := pReq.(backend.ForwardHTTPHeaders); ok {
+				for k, v := range h.GetHTTPHeaders() {
+					req.Header[k] = v
+				}
+			}
+
+			return next.RoundTrip(req)
+		})
+	})
+
+	return httpclient.WithContextualMiddleware(ctx, mw)
+}
+
+func (m *HTTPClientMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+	if req == nil {
+		return m.next.QueryData(ctx, req)
+	}
+
+	ctx = m.applyHeaders(ctx, req)
+
+	return m.next.QueryData(ctx, req)
+}
+
+func (m *HTTPClientMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
+	if req == nil {
+		return m.next.CallResource(ctx, req, sender)
+	}
+
+	ctx = m.applyHeaders(ctx, req)
+
+	return m.next.CallResource(ctx, req, sender)
+}
+
+func (m *HTTPClientMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
+	if req == nil {
+		return m.next.CheckHealth(ctx, req)
+	}
+
+	ctx = m.applyHeaders(ctx, req)
+
+	return m.next.CheckHealth(ctx, req)
+}
+
+func (m *HTTPClientMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) {
+	return m.next.CollectMetrics(ctx, req)
+}
+
+func (m *HTTPClientMiddleware) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) {
+	return m.next.SubscribeStream(ctx, req)
+}
+
+func (m *HTTPClientMiddleware) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) {
+	return m.next.PublishStream(ctx, req)
+}
+
+func (m *HTTPClientMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error {
+	return m.next.RunStream(ctx, req, sender)
+}

pkg/services/pluginsintegration/clientmiddleware/httpclient_middleware_test.go

@@ -0,0 +1,289 @@
+package clientmiddleware
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/plugins/manager/client/clienttest"
+	ngalertmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/user"
+	"github.com/grafana/grafana/pkg/util/proxyutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPClientMiddleware(t *testing.T) {
+	const otherHeader = "test"
+
+	t.Run("When no http headers in plugin request", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "/some/thing", nil)
+		require.NoError(t, err)
+
+		t.Run("And requests are for a datasource", func(t *testing.T) {
+			cdt := clienttest.NewClientDecoratorTest(t,
+				clienttest.WithReqContext(req, &user.SignedInUser{}),
+				clienttest.WithMiddlewares(NewHTTPClientMiddleware()),
+			)
+
+			pluginCtx := backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
+			}
+
+			t.Run("Should not forward headers when calling QueryData", func(t *testing.T) {
+				_, err = cdt.Decorator.QueryData(req.Context(), &backend.QueryDataRequest{
+					PluginContext: pluginCtx,
+					Headers:       map[string]string{otherHeader: "val"},
+				})
+				require.NoError(t, err)
+				require.NotNil(t, cdt.QueryDataReq)
+				require.Len(t, cdt.QueryDataReq.Headers, 1)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 0)
+			})
+
+			t.Run("Should not forward headers when calling CallResource", func(t *testing.T) {
+				err = cdt.Decorator.CallResource(req.Context(), &backend.CallResourceRequest{
+					PluginContext: pluginCtx,
+					Headers:       map[string][]string{otherHeader: {"val"}},
+				}, nopCallResourceSender)
+				require.NoError(t, err)
+				require.NotNil(t, cdt.CallResourceReq)
+				require.Len(t, cdt.CallResourceReq.Headers, 1)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 0)
+			})
+
+			t.Run("Should not forward headers when calling CheckHealth", func(t *testing.T) {
+				_, err = cdt.Decorator.CheckHealth(req.Context(), &backend.CheckHealthRequest{
+					PluginContext: pluginCtx,
+					Headers:       map[string]string{otherHeader: "val"},
+				})
+				require.NoError(t, err)
+				require.NotNil(t, cdt.CheckHealthReq)
+				require.Len(t, cdt.CheckHealthReq.Headers, 1)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 0)
+			})
+		})
+
+		t.Run("And requests are for an app", func(t *testing.T) {
+			cdt := clienttest.NewClientDecoratorTest(t,
+				clienttest.WithReqContext(req, &user.SignedInUser{}),
+				clienttest.WithMiddlewares(NewHTTPClientMiddleware()),
+			)
+
+			pluginCtx := backend.PluginContext{
+				AppInstanceSettings: &backend.AppInstanceSettings{},
+			}
+
+			t.Run("Should not forward headers when calling QueryData", func(t *testing.T) {
+				_, err = cdt.Decorator.QueryData(req.Context(), &backend.QueryDataRequest{
+					PluginContext: pluginCtx,
+					Headers:       map[string]string{},
+				})
+				require.NoError(t, err)
+				require.NotNil(t, cdt.QueryDataReq)
+				require.Len(t, cdt.QueryDataReq.Headers, 0)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 0)
+			})
+
+			t.Run("Should not forward headers when calling CallResource", func(t *testing.T) {
+				err = cdt.Decorator.CallResource(req.Context(), &backend.CallResourceRequest{
+					PluginContext: pluginCtx,
+					Headers:       map[string][]string{},
+				}, nopCallResourceSender)
+				require.NoError(t, err)
+				require.NotNil(t, cdt.CallResourceReq)
+				require.Len(t, cdt.CallResourceReq.Headers, 0)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 0)
+			})
+
+			t.Run("Should not forward headers when calling CheckHealth", func(t *testing.T) {
+				_, err = cdt.Decorator.CheckHealth(req.Context(), &backend.CheckHealthRequest{
+					PluginContext: pluginCtx,
+					Headers:       map[string]string{},
+				})
+				require.NoError(t, err)
+				require.NotNil(t, cdt.CheckHealthReq)
+				require.Len(t, cdt.CheckHealthReq.Headers, 0)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 0)
+			})
+		})
+	})
+
+	t.Run("When HTTP headers in plugin request", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "/some/thing", nil)
+		require.NoError(t, err)
+
+		headers := map[string]string{
+			ngalertmodels.FromAlertHeaderName:      "true",
+			backend.OAuthIdentityTokenHeaderName:   "bearer token",
+			backend.OAuthIdentityIDTokenHeaderName: "id-token",
+			"http_" + proxyutil.UserHeaderName:     "uname",
+			backend.CookiesHeaderName:              "cookie1=; cookie2=; cookie3=",
+			otherHeader:                            "val",
+		}
+
+		crHeaders := map[string][]string{}
+		for k, v := range headers {
+			crHeaders[k] = []string{v}
+		}
+
+		t.Run("And requests are for a datasource", func(t *testing.T) {
+			cdt := clienttest.NewClientDecoratorTest(t,
+				clienttest.WithReqContext(req, &user.SignedInUser{}),
+				clienttest.WithMiddlewares(NewHTTPClientMiddleware()),
+			)
+
+			pluginCtx := backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
+			}
+
+			t.Run("Should forward headers when calling QueryData", func(t *testing.T) {
+				_, err = cdt.Decorator.QueryData(req.Context(), &backend.QueryDataRequest{
+					PluginContext: pluginCtx,
+					Headers:       headers,
+				})
+				require.NoError(t, err)
+				require.NotNil(t, cdt.QueryDataReq)
+				require.Len(t, cdt.QueryDataReq.Headers, 6)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 5)
+				require.Equal(t, "true", reqClone.Header.Get(ngalertmodels.FromAlertHeaderName))
+				require.Equal(t, "bearer token", reqClone.Header.Get(backend.OAuthIdentityTokenHeaderName))
+				require.Equal(t, "id-token", reqClone.Header.Get(backend.OAuthIdentityIDTokenHeaderName))
+				require.Equal(t, "uname", reqClone.Header.Get(proxyutil.UserHeaderName))
+				require.Len(t, reqClone.Cookies(), 3)
+				require.Equal(t, "cookie1", reqClone.Cookies()[0].Name)
+				require.Equal(t, "cookie2", reqClone.Cookies()[1].Name)
+				require.Equal(t, "cookie3", reqClone.Cookies()[2].Name)
+			})
+
+			t.Run("Should forward headers when calling CallResource", func(t *testing.T) {
+				err = cdt.Decorator.CallResource(req.Context(), &backend.CallResourceRequest{
+					PluginContext: pluginCtx,
+					Headers:       crHeaders,
+				}, nopCallResourceSender)
+				require.NoError(t, err)
+				require.NotNil(t, cdt.CallResourceReq)
+				require.Len(t, cdt.CallResourceReq.Headers, 6)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 5)
+				require.Equal(t, "true", reqClone.Header.Get(ngalertmodels.FromAlertHeaderName))
+				require.Equal(t, "bearer token", reqClone.Header.Get(backend.OAuthIdentityTokenHeaderName))
+				require.Equal(t, "id-token", reqClone.Header.Get(backend.OAuthIdentityIDTokenHeaderName))
+				require.Equal(t, "uname", reqClone.Header.Get(proxyutil.UserHeaderName))
+				require.Len(t, reqClone.Cookies(), 3)
+				require.Equal(t, "cookie1", reqClone.Cookies()[0].Name)
+				require.Equal(t, "cookie2", reqClone.Cookies()[1].Name)
+				require.Equal(t, "cookie3", reqClone.Cookies()[2].Name)
+			})
+
+			t.Run("Should forward headers when calling CheckHealth", func(t *testing.T) {
+				_, err = cdt.Decorator.CheckHealth(req.Context(), &backend.CheckHealthRequest{
+					PluginContext: pluginCtx,
+					Headers:       headers,
+				})
+				require.NoError(t, err)
+				require.NotNil(t, cdt.CheckHealthReq)
+				require.Len(t, cdt.CheckHealthReq.Headers, 6)
+
+				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
+				require.Len(t, middlewares, 1)
+				require.Equal(t, forwardPluginRequestHTTPHeaders, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
+
+				reqClone := req.Clone(req.Context())
+				res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
+				require.NoError(t, err)
+				require.NoError(t, res.Body.Close())
+				require.Len(t, reqClone.Header, 5)
+				require.Equal(t, "true", reqClone.Header.Get(ngalertmodels.FromAlertHeaderName))
+				require.Equal(t, "bearer token", reqClone.Header.Get(backend.OAuthIdentityTokenHeaderName))
+				require.Equal(t, "id-token", reqClone.Header.Get(backend.OAuthIdentityIDTokenHeaderName))
+				require.Equal(t, "uname", reqClone.Header.Get(proxyutil.UserHeaderName))
+				require.Len(t, reqClone.Cookies(), 3)
+				require.Equal(t, "cookie1", reqClone.Cookies()[0].Name)
+				require.Equal(t, "cookie2", reqClone.Cookies()[1].Name)
+				require.Equal(t, "cookie3", reqClone.Cookies()[2].Name)
+			})
+		})
+	})
+}
+
+var finalRoundTripper = httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Request:    req,
+		Body:       io.NopCloser(bytes.NewBufferString("")),
+	}, nil
+})

pkg/services/pluginsintegration/clientmiddleware/oauthtoken_middleware.go

@@ -3,12 +3,9 @@ package clientmiddleware
 import (
 	"context"
 	"fmt"
-	"net/http"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/services/datasources"
@@ -16,8 +13,8 @@ import (
 )
 
 // NewOAuthTokenMiddleware creates a new plugins.ClientMiddleware that will
-// set OAuth token headers on outgoing plugins.Client and HTTP requests if
-// the datasource has enabled Forward OAuth Identity (oauthPassThru).
+// set OAuth token headers on outgoing plugins.Client requests if the
+// datasource has enabled Forward OAuth Identity (oauthPassThru).
 func NewOAuthTokenMiddleware(oAuthTokenService oauthtoken.OAuthTokenService) plugins.ClientMiddleware {
 	return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client {
 		return &OAuthTokenMiddleware{
@@ -37,17 +34,17 @@ type OAuthTokenMiddleware struct {
 	next              plugins.Client
 }
 
-func (m *OAuthTokenMiddleware) applyToken(ctx context.Context, pCtx backend.PluginContext, req interface{}) (context.Context, error) {
+func (m *OAuthTokenMiddleware) applyToken(ctx context.Context, pCtx backend.PluginContext, req interface{}) error {
 	reqCtx := contexthandler.FromContext(ctx)
 	// if request not for a datasource or no HTTP request context skip middleware
 	if req == nil || pCtx.DataSourceInstanceSettings == nil || reqCtx == nil || reqCtx.Req == nil {
-		return ctx, nil
+		return nil
 	}
 
 	settings := pCtx.DataSourceInstanceSettings
 	jsonDataBytes, err := simplejson.NewJson(settings.JSONData)
 	if err != nil {
-		return ctx, err
+		return err
 	}
 
 	ds := &datasources.DataSource{
@@ -84,19 +81,10 @@ func (m *OAuthTokenMiddleware) applyToken(ctx context.Context, pCtx backend.Plug
 					t.Headers[idTokenHeaderName] = []string{idTokenHeader}
 				}
 			}
-
-			httpHeaders := http.Header{}
-			httpHeaders.Set(tokenHeaderName, authorizationHeader)
-
-			if idTokenHeader != "" {
-				httpHeaders.Set(idTokenHeaderName, idTokenHeader)
-			}
-
-			ctx = httpclient.WithContextualMiddleware(ctx, httpclientprovider.SetHeadersMiddleware(httpHeaders))
 		}
 	}
 
-	return ctx, nil
+	return nil
 }
 
 func (m *OAuthTokenMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -104,12 +92,12 @@ func (m *OAuthTokenMiddleware) QueryData(ctx context.Context, req *backend.Query
 		return m.next.QueryData(ctx, req)
 	}
 
-	newCtx, err := m.applyToken(ctx, req.PluginContext, req)
+	err := m.applyToken(ctx, req.PluginContext, req)
 	if err != nil {
 		return nil, err
 	}
 
-	return m.next.QueryData(newCtx, req)
+	return m.next.QueryData(ctx, req)
 }
 
 func (m *OAuthTokenMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
@@ -117,12 +105,12 @@ func (m *OAuthTokenMiddleware) CallResource(ctx context.Context, req *backend.Ca
 		return m.next.CallResource(ctx, req, sender)
 	}
 
-	newCtx, err := m.applyToken(ctx, req.PluginContext, req)
+	err := m.applyToken(ctx, req.PluginContext, req)
 	if err != nil {
 		return err
 	}
 
-	return m.next.CallResource(newCtx, req, sender)
+	return m.next.CallResource(ctx, req, sender)
 }
 
 func (m *OAuthTokenMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
@@ -130,12 +118,12 @@ func (m *OAuthTokenMiddleware) CheckHealth(ctx context.Context, req *backend.Che
 		return m.next.CheckHealth(ctx, req)
 	}
 
-	newCtx, err := m.applyToken(ctx, req.PluginContext, req)
+	err := m.applyToken(ctx, req.PluginContext, req)
 	if err != nil {
 		return nil, err
 	}
 
-	return m.next.CheckHealth(newCtx, req)
+	return m.next.CheckHealth(ctx, req)
 }
 
 func (m *OAuthTokenMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) {

pkg/services/pluginsintegration/clientmiddleware/oauthtoken_middleware_test.go

@@ -6,8 +6,6 @@ import (
 	"testing"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins/manager/client/clienttest"
 	"github.com/grafana/grafana/pkg/services/oauthtoken/oauthtokentest"
 	"github.com/grafana/grafana/pkg/services/user"
@@ -49,9 +47,6 @@ func TestOAuthTokenMiddleware(t *testing.T) {
 			require.NotNil(t, cdt.QueryDataReq)
 			require.Len(t, cdt.QueryDataReq.Headers, 1)
 			require.Equal(t, "test", cdt.QueryDataReq.Headers[otherHeader])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-			require.Len(t, middlewares, 0)
 		})
 
 		t.Run("Should not forward OAuth Identity when calling CallResource", func(t *testing.T) {
@@ -63,9 +58,6 @@ func TestOAuthTokenMiddleware(t *testing.T) {
 			require.NotNil(t, cdt.CallResourceReq)
 			require.Len(t, cdt.CallResourceReq.Headers, 1)
 			require.Equal(t, "test", cdt.CallResourceReq.Headers[otherHeader][0])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-			require.Len(t, middlewares, 0)
 		})
 
 		t.Run("Should not forward OAuth Identity when calling CheckHealth", func(t *testing.T) {
@@ -77,9 +69,6 @@ func TestOAuthTokenMiddleware(t *testing.T) {
 			require.NotNil(t, cdt.CheckHealthReq)
 			require.Len(t, cdt.CheckHealthReq.Headers, 1)
 			require.Equal(t, "test", cdt.CheckHealthReq.Headers[otherHeader])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-			require.Len(t, middlewares, 0)
 		})
 	})
 
@@ -125,19 +114,6 @@ func TestOAuthTokenMiddleware(t *testing.T) {
 			require.Equal(t, "test", cdt.QueryDataReq.Headers[otherHeader])
 			require.Equal(t, "Bearer access-token", cdt.QueryDataReq.Headers[tokenHeaderName])
 			require.Equal(t, "id-token", cdt.QueryDataReq.Headers[idTokenHeaderName])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 3)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
-			require.Equal(t, "Bearer access-token", reqClone.Header.Get(tokenHeaderName))
-			require.Equal(t, "id-token", reqClone.Header.Get(idTokenHeaderName))
 		})
 
 		t.Run("Should forward OAuth Identity when calling CallResource", func(t *testing.T) {
@@ -153,19 +129,6 @@ func TestOAuthTokenMiddleware(t *testing.T) {
 			require.Equal(t, "Bearer access-token", cdt.CallResourceReq.Headers[tokenHeaderName][0])
 			require.Len(t, cdt.CallResourceReq.Headers[idTokenHeaderName], 1)
 			require.Equal(t, "id-token", cdt.CallResourceReq.Headers[idTokenHeaderName][0])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 3)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
-			require.Equal(t, "Bearer access-token", reqClone.Header.Get(tokenHeaderName))
-			require.Equal(t, "id-token", reqClone.Header.Get(idTokenHeaderName))
 		})
 
 		t.Run("Should forward OAuth Identity when calling CheckHealth", func(t *testing.T) {
@@ -179,19 +142,6 @@ func TestOAuthTokenMiddleware(t *testing.T) {
 			require.Equal(t, "test", cdt.CheckHealthReq.Headers[otherHeader])
 			require.Equal(t, "Bearer access-token", cdt.CheckHealthReq.Headers[tokenHeaderName])
 			require.Equal(t, "id-token", cdt.CheckHealthReq.Headers[idTokenHeaderName])
-
-			middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-			require.Len(t, middlewares, 1)
-			require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
-
-			reqClone := req.Clone(req.Context())
-			res, err := middlewares[0].CreateMiddleware(httpclient.Options{}, finalRoundTripper).RoundTrip(reqClone)
-			require.NoError(t, err)
-			require.NoError(t, res.Body.Close())
-			require.Len(t, reqClone.Header, 3)
-			require.Equal(t, "test", reqClone.Header.Get(otherHeader))
-			require.Equal(t, "Bearer access-token", reqClone.Header.Get(tokenHeaderName))
-			require.Equal(t, "id-token", reqClone.Header.Get(idTokenHeaderName))
 		})
 	})
 }

pkg/services/pluginsintegration/clientmiddleware/user_header_middleware.go

@@ -2,19 +2,15 @@ package clientmiddleware
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/util/proxyutil"
 )
 
 // NewUserHeaderMiddleware creates a new plugins.ClientMiddleware that will
-// populate the X-Grafana-User header on outgoing plugins.Client and HTTP
-// requests.
+// populate the X-Grafana-User header on outgoing plugins.Client requests.
 func NewUserHeaderMiddleware() plugins.ClientMiddleware {
 	return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client {
 		return &UserHeaderMiddleware{
@@ -27,33 +23,17 @@ type UserHeaderMiddleware struct {
 	next plugins.Client
 }
 
-func (m *UserHeaderMiddleware) applyToken(ctx context.Context, pCtx backend.PluginContext, h backend.ForwardHTTPHeaders) context.Context {
+func (m *UserHeaderMiddleware) applyUserHeader(ctx context.Context, h backend.ForwardHTTPHeaders) {
 	reqCtx := contexthandler.FromContext(ctx)
 	// if no HTTP request context skip middleware
 	if h == nil || reqCtx == nil || reqCtx.Req == nil || reqCtx.SignedInUser == nil {
-		return ctx
+		return
 	}
 
 	h.DeleteHTTPHeader(proxyutil.UserHeaderName)
 	if !reqCtx.IsAnonymous {
 		h.SetHTTPHeader(proxyutil.UserHeaderName, reqCtx.Login)
 	}
-
-	middlewares := []sdkhttpclient.Middleware{}
-
-	if !reqCtx.IsAnonymous {
-		httpHeaders := http.Header{
-			proxyutil.UserHeaderName: []string{reqCtx.Login},
-		}
-
-		middlewares = append(middlewares, httpclientprovider.SetHeadersMiddleware(httpHeaders))
-	} else {
-		middlewares = append(middlewares, httpclientprovider.DeleteHeadersMiddleware(proxyutil.UserHeaderName))
-	}
-
-	ctx = sdkhttpclient.WithContextualMiddleware(ctx, middlewares...)
-
-	return ctx
 }
 
 func (m *UserHeaderMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -61,7 +41,7 @@ func (m *UserHeaderMiddleware) QueryData(ctx context.Context, req *backend.Query
 		return m.next.QueryData(ctx, req)
 	}
 
-	ctx = m.applyToken(ctx, req.PluginContext, req)
+	m.applyUserHeader(ctx, req)
 
 	return m.next.QueryData(ctx, req)
 }
@@ -71,7 +51,7 @@ func (m *UserHeaderMiddleware) CallResource(ctx context.Context, req *backend.Ca
 		return m.next.CallResource(ctx, req, sender)
 	}
 
-	ctx = m.applyToken(ctx, req.PluginContext, req)
+	m.applyUserHeader(ctx, req)
 
 	return m.next.CallResource(ctx, req, sender)
 }
@@ -81,7 +61,7 @@ func (m *UserHeaderMiddleware) CheckHealth(ctx context.Context, req *backend.Che
 		return m.next.CheckHealth(ctx, req)
 	}
 
-	ctx = m.applyToken(ctx, req.PluginContext, req)
+	m.applyUserHeader(ctx, req)
 
 	return m.next.CheckHealth(ctx, req)
 }

pkg/services/pluginsintegration/clientmiddleware/user_header_middleware_test.go

@@ -5,8 +5,6 @@ import (
 	"testing"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
-	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/plugins/manager/client/clienttest"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/util/proxyutil"
@@ -39,10 +37,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Empty(t, cdt.QueryDataReq.Headers)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should not forward user header when calling CallResource", func(t *testing.T) {
@@ -53,10 +47,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Empty(t, cdt.CallResourceReq.Headers)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should not forward user header when calling CheckHealth", func(t *testing.T) {
@@ -67,10 +57,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Empty(t, cdt.CheckHealthReq.Headers)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 		})
 
@@ -95,10 +81,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Empty(t, cdt.QueryDataReq.Headers)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should not forward user header when calling CallResource", func(t *testing.T) {
@@ -109,10 +91,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Empty(t, cdt.CallResourceReq.Headers)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should not forward user header when calling CheckHealth", func(t *testing.T) {
@@ -123,10 +101,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NoError(t, err)
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Empty(t, cdt.CheckHealthReq.Headers)
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.DeleteHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 		})
 	})
@@ -156,10 +130,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Len(t, cdt.QueryDataReq.Headers, 1)
 				require.Equal(t, "admin", cdt.QueryDataReq.GetHTTPHeader(proxyutil.UserHeaderName))
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should forward user header when calling CallResource", func(t *testing.T) {
@@ -171,10 +141,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Len(t, cdt.CallResourceReq.Headers, 1)
 				require.Equal(t, "admin", cdt.CallResourceReq.GetHTTPHeader(proxyutil.UserHeaderName))
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should forward user header when calling CheckHealth", func(t *testing.T) {
@@ -186,10 +152,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Len(t, cdt.CheckHealthReq.Headers, 1)
 				require.Equal(t, "admin", cdt.CheckHealthReq.GetHTTPHeader(proxyutil.UserHeaderName))
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 		})
 
@@ -214,10 +176,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NotNil(t, cdt.QueryDataReq)
 				require.Len(t, cdt.QueryDataReq.Headers, 1)
 				require.Equal(t, "admin", cdt.QueryDataReq.GetHTTPHeader(proxyutil.UserHeaderName))
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.QueryDataCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should forward user header when calling CallResource", func(t *testing.T) {
@@ -229,10 +187,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NotNil(t, cdt.CallResourceReq)
 				require.Len(t, cdt.CallResourceReq.Headers, 1)
 				require.Equal(t, "admin", cdt.CallResourceReq.GetHTTPHeader(proxyutil.UserHeaderName))
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CallResourceCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("Should forward user header when calling CheckHealth", func(t *testing.T) {
@@ -244,10 +198,6 @@ func TestUserHeaderMiddleware(t *testing.T) {
 				require.NotNil(t, cdt.CheckHealthReq)
 				require.Len(t, cdt.CheckHealthReq.Headers, 1)
 				require.Equal(t, "admin", cdt.CheckHealthReq.GetHTTPHeader(proxyutil.UserHeaderName))
-
-				middlewares := httpclient.ContextualMiddlewareFromContext(cdt.CheckHealthCtx)
-				require.Len(t, middlewares, 1)
-				require.Equal(t, httpclientprovider.SetHeadersMiddlewareName, middlewares[0].(httpclient.MiddlewareName).MiddlewareName())
 			})
 		})
 	})

pkg/services/pluginsintegration/pluginsintegration.go

@@ -81,5 +81,7 @@ func CreateMiddlewares(cfg *setting.Cfg, oAuthTokenService oauthtoken.OAuthToken
 		middlewares = append(middlewares, clientmiddleware.NewUserHeaderMiddleware())
 	}
 
+	middlewares = append(middlewares, clientmiddleware.NewHTTPClientMiddleware())
+
 	return middlewares
 }