Ensure that datasource apiservers receive and forwards headers (#92304)

* Ensure that datasource apiservers receive and forwards headers for datasources:
- adds log line for prometheus to see when from alert header is received
- add logging to the datasource apiserver
- Updates the Connect func in sub query to forward expected headers to datasources and log unexpected ones.

pkg/registry/apis/datasource/sub_query.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
@@ -52,7 +53,6 @@ func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Ob
 	if err != nil {
 		return nil, err
 	}
-
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		dqr := data.QueryDataRequest{}
 		err := web.Bind(req, &dqr)
@@ -73,9 +73,30 @@ func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Ob
 
 		ctx = backend.WithGrafanaConfig(ctx, pluginCtx.GrafanaConfig)
 		ctx = contextualMiddlewares(ctx)
+
+		// only forward expected headers, log unexpected ones
+		headers := make(map[string]string)
+		// headers are case insensitive, however some datasources still check for camel casing so we have to send them camel cased
+		expectedHeaders := map[string]string{
+			"fromalert":      "FromAlert",
+			"content-type":   "Content-Type",
+			"content-length": "Content-Length",
+			"user-agent":     "User-Agent",
+			"accept":         "Accept",
+		}
+		for k, v := range req.Header {
+			headerToSend, ok := expectedHeaders[strings.ToLower(k)]
+			if ok {
+				headers[headerToSend] = v[0]
+			} else {
+				r.builder.log.Warn("datasource received an unexpected header, ignoring it", "header", k)
+			}
+		}
+
 		rsp, err := r.builder.client.QueryData(ctx, &backend.QueryDataRequest{
 			Queries:       queries,
 			PluginContext: pluginCtx,
+			Headers:       headers,
 		})
 		if err != nil {
 			responder.Error(err)