diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go index 61cb6836e22..3f349cee60c 100644 --- a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go @@ -5,6 +5,7 @@ import ( sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana/pkg/infra/httpclient" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/setting" ) @@ -12,8 +13,10 @@ var newProviderFunc = sdkhttpclient.NewProvider // New creates a new HTTP client provider with pre-configured middlewares. func New(cfg *setting.Cfg) httpclient.Provider { + logger := log.New("httpclient") userAgent := fmt.Sprintf("Grafana/%s", cfg.BuildVersion) middlewares := []sdkhttpclient.Middleware{ + TracingMiddleware(logger), DataSourceMetricsMiddleware(), SetUserAgentMiddleware(userAgent), sdkhttpclient.BasicAuthenticationMiddleware(), diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go index d5359eaf567..6b6c310368b 100644 --- a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go @@ -22,11 +22,12 @@ func TestHTTPClientProvider(t *testing.T) { _ = New(&setting.Cfg{SigV4AuthEnabled: false}) require.Len(t, providerOpts, 1) o := providerOpts[0] - require.Len(t, o.Middlewares, 4) - require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Len(t, o.Middlewares, 5) + require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName()) }) t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) { @@ -42,11 +43,12 @@ func TestHTTPClientProvider(t *testing.T) { _ = New(&setting.Cfg{SigV4AuthEnabled: true}) require.Len(t, providerOpts, 1) o := providerOpts[0] - require.Len(t, o.Middlewares, 5) - require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, SigV4MiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Len(t, o.Middlewares, 6) + require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, SigV4MiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName()) }) } diff --git a/pkg/infra/httpclient/httpclientprovider/testing.go b/pkg/infra/httpclient/httpclientprovider/testing.go index d729c02d563..6a1b6cff478 100644 --- a/pkg/infra/httpclient/httpclientprovider/testing.go +++ b/pkg/infra/httpclient/httpclientprovider/testing.go @@ -13,6 +13,6 @@ type testContext struct { func (c *testContext) createRoundTripper(name string) http.RoundTripper { return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { c.callChain = append(c.callChain, name) - return &http.Response{StatusCode: http.StatusOK}, nil + return &http.Response{StatusCode: http.StatusOK, Request: req}, nil }) } diff --git a/pkg/infra/httpclient/httpclientprovider/tracing_middleware.go b/pkg/infra/httpclient/httpclientprovider/tracing_middleware.go new file mode 100644 index 00000000000..f95cc102dec --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/tracing_middleware.go @@ -0,0 +1,62 @@ +package httpclientprovider + +import ( + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" +) + +const ( + TracingMiddlewareName = "tracing" + httpContentLengthTagKey = "http.content_length" +) + +func TracingMiddleware(logger log.Logger) httpclient.Middleware { + return httpclient.NamedMiddlewareFunc(TracingMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper { + return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + span, ctx := opentracing.StartSpanFromContext(req.Context(), "HTTP Outgoing Request") + defer span.Finish() + + req = req.WithContext(ctx) + for k, v := range opts.Labels { + span.SetTag(k, v) + } + + if err := opentracing.GlobalTracer().Inject( + span.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)); err != nil { + logger.Error("Failed to inject span context instance", "err", err) + } + + res, err := next.RoundTrip(req) + + ext.HTTPUrl.Set(span, req.URL.String()) + ext.HTTPMethod.Set(span, req.Method) + ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum) + + if err != nil { + ext.Error.Set(span, true) + return res, err + } + + if res != nil { + // we avoid measuring contentlength less than zero because it indicates + // that the content size is unknown. https://godoc.org/github.com/badu/http#Response + if res.ContentLength > 0 { + span.SetTag(httpContentLengthTagKey, res.ContentLength) + } + + ext.HTTPStatusCode.Set(span, uint16(res.StatusCode)) + if res.StatusCode >= 400 { + ext.Error.Set(span, true) + } + } + + return res, err + }) + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go new file mode 100644 index 00000000000..4f9e29fdde8 --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go @@ -0,0 +1,173 @@ +package httpclientprovider + +import ( + "bytes" + "context" + "net/http" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" + "github.com/stretchr/testify/require" + jaeger "github.com/uber/jaeger-client-go" +) + +func TestTracingMiddleware(t *testing.T) { + setupTracing(t) + + t.Run("GET request that returns 200 OK should start and capture span", func(t *testing.T) { + finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Request: req}, nil + }) + + mw := TracingMiddleware(log.New("test")) + rt := mw.CreateMiddleware(httpclient.Options{ + Labels: map[string]string{ + "l1": "v1", + "l2": "v2", + }, + }, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, TracingMiddlewareName, middlewareName.MiddlewareName()) + + ctx := context.Background() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://test.com/query", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + + sp := opentracing.SpanFromContext(res.Request.Context()) + require.NotNil(t, sp) + jsp, ok := sp.(*jaeger.Span) + require.True(t, ok) + require.Equal(t, "HTTP Outgoing Request", jsp.OperationName()) + require.Len(t, jsp.Tags(), 8) + expectedTags := opentracing.Tags{ + string(ext.HTTPMethod): http.MethodGet, + string(ext.HTTPStatusCode): uint16(http.StatusOK), + string(ext.HTTPUrl): "http://test.com/query", + "l1": "v1", + "l2": "v2", + jaeger.SamplerParamTagKey: true, + jaeger.SamplerTypeTagKey: jaeger.SamplerTypeConst, + string(ext.SpanKind): ext.SpanKindRPCClientEnum, + } + require.EqualValues(t, expectedTags, jsp.Tags()) + require.Contains(t, req.Header, "Uber-Trace-Id") + }) + + t.Run("GET request that returns 400 Bad Request should start and capture span", func(t *testing.T) { + finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusBadRequest, Request: req}, nil + }) + + mw := TracingMiddleware(log.New("test")) + rt := mw.CreateMiddleware(httpclient.Options{ + Labels: map[string]string{ + "l1": "v1", + "l2": "v2", + }, + }, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, TracingMiddlewareName, middlewareName.MiddlewareName()) + + ctx := context.Background() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://test.com/query", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + + sp := opentracing.SpanFromContext(res.Request.Context()) + require.NotNil(t, sp) + jsp, ok := sp.(*jaeger.Span) + require.True(t, ok) + require.Equal(t, "HTTP Outgoing Request", jsp.OperationName()) + require.Len(t, jsp.Tags(), 9) + expectedTags := opentracing.Tags{ + string(ext.Error): true, + string(ext.HTTPMethod): http.MethodGet, + string(ext.HTTPStatusCode): uint16(http.StatusBadRequest), + string(ext.HTTPUrl): "http://test.com/query", + "l1": "v1", + "l2": "v2", + jaeger.SamplerParamTagKey: true, + jaeger.SamplerTypeTagKey: jaeger.SamplerTypeConst, + string(ext.SpanKind): ext.SpanKindRPCClientEnum, + } + require.EqualValues(t, expectedTags, jsp.Tags()) + require.Contains(t, req.Header, "Uber-Trace-Id") + }) + + t.Run("POST request that returns 200 OK should start and capture span", func(t *testing.T) { + finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Request: req, ContentLength: 10}, nil + }) + + mw := TracingMiddleware(log.New("test")) + rt := mw.CreateMiddleware(httpclient.Options{ + Labels: map[string]string{ + "l1": "v1", + "l2": "v2", + }, + }, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, TracingMiddlewareName, middlewareName.MiddlewareName()) + + ctx := context.Background() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://test.com/query", bytes.NewBufferString("{ \"message\": \"ok\"}")) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + + sp := opentracing.SpanFromContext(res.Request.Context()) + require.NotNil(t, sp) + jsp, ok := sp.(*jaeger.Span) + require.True(t, ok) + require.Equal(t, "HTTP Outgoing Request", jsp.OperationName()) + require.Len(t, jsp.Tags(), 9) + expectedTags := opentracing.Tags{ + httpContentLengthTagKey: int64(10), + string(ext.HTTPMethod): http.MethodPost, + string(ext.HTTPStatusCode): uint16(http.StatusOK), + string(ext.HTTPUrl): "http://test.com/query", + "l1": "v1", + "l2": "v2", + jaeger.SamplerParamTagKey: true, + jaeger.SamplerTypeTagKey: jaeger.SamplerTypeConst, + string(ext.SpanKind): ext.SpanKindRPCClientEnum, + } + require.EqualValues(t, expectedTags, jsp.Tags()) + require.Contains(t, req.Header, "Uber-Trace-Id") + }) +} + +func setupTracing(t *testing.T) { + t.Helper() + + tracer, closer := jaeger.NewTracer("test", jaeger.NewConstSampler(true), jaeger.NewNullReporter()) + opentracing.SetGlobalTracer(tracer) + t.Cleanup(func() { + require.NoError(t, closer.Close()) + opentracing.SetGlobalTracer(opentracing.NoopTracer{}) + }) +}