[v7.5.x] HTTP Client: Introduce go-conntrack (#35321)

* Introduce go-conntrack into http transport

* Fix linting

* Add sanitize label name for datasource names

* Fix tests - add one to check missing datasource name

* Fix more tests

* Remove DialWithTracing()

pkg/models/datasource_cache.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
@@ -11,9 +12,9 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-aws-sdk/pkg/sigv4"
-
 	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/mwitkow/go-conntrack"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -154,13 +155,15 @@ func (ds *DataSource) GetHttpTransport() (*dataSourceTransport, error) {
 
 	// Create transport which adds all
 	customHeaders := ds.getCustomHeaders()
+	datasourceLabelName, err := metricutil.SanitizeLabelName(ds.Name)
+	if err != nil {
+		return nil, err
+	}
+
 	transport := &http.Transport{
-		TLSClientConfig: tlsConfig,
-		Proxy:           http.ProxyFromEnvironment,
-		Dial: (&net.Dialer{
-			Timeout:   time.Duration(setting.DataProxyTimeout) * time.Second,
-			KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second,
-		}).Dial,
+		TLSClientConfig:       tlsConfig,
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           newConntrackDialContext(datasourceLabelName),
 		TLSHandshakeTimeout:   time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second,
 		ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second,
 		MaxIdleConns:          setting.DataProxyMaxIdleConns,
@@ -332,3 +335,15 @@ func awsServiceNamespace(dsType string) string {
 		panic(fmt.Sprintf("Unsupported datasource %q", dsType))
 	}
 }
+
+// newConntrackDialContext takes a http.DefaultTransport and adds the Conntrack Dialer
+// so we can instrument outbound connections
+func newConntrackDialContext(name string) func(context.Context, string, string) (net.Conn, error) {
+	return conntrack.NewDialContextFunc(
+		conntrack.DialWithName(name),
+		conntrack.DialWithDialer(&net.Dialer{
+			Timeout:   time.Duration(setting.DataProxyTimeout) * time.Second,
+			KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second,
+		}),
+	)
+}