Instrumentation: Add status_source label to request metrics/logs (#74114)

Ref #68480

Co-authored-by: Giuseppe Guerra <giuseppe.guerra@grafana.com>

pkg/util/errutil/errors.go

@@ -15,6 +15,7 @@ type Base struct {
 	messageID     string
 	publicMessage string
 	logLevel      LogLevel
+	source        Source
 }
 
 // NewBase initializes a [Base] that is used to construct [Error].
@@ -32,6 +33,7 @@ func NewBase(reason StatusReason, msgID string, opts ...BaseOpt) Base {
 		reason:    reason,
 		messageID: msgID,
 		logLevel:  reason.Status().LogLevel(),
+		source:    SourceServer,
 	}
 
 	for _, opt := range opts {
@@ -143,6 +145,32 @@ func NotImplemented(msgID string, opts ...BaseOpt) Base {
 	return NewBase(StatusNotImplemented, msgID, opts...)
 }
 
+// BadGateway initializes a new [Base] error with reason StatusBadGateway
+// and source SourceDownstream that is used to construct [Error]. The msgID
+// is passed to the caller to serve as the base for user facing error messages.
+//
+// msgID should be structured as component.errorBrief, for example
+//
+//	area.downstreamError
+func BadGateway(msgID string, opts ...BaseOpt) Base {
+	newOpts := []BaseOpt{WithDownstream()}
+	newOpts = append(newOpts, opts...)
+	return NewBase(StatusBadGateway, msgID, newOpts...)
+}
+
+// GatewayTimeout initializes a new [Base] error with reason StatusGatewayTimeout
+// and source SourceDownstream that is used to construct [Error]. The msgID
+// is passed to the caller to serve as the base for user facing error messages.
+//
+// msgID should be structured as component.errorBrief, for example
+//
+//	area.downstreamTimeout
+func GatewayTimeout(msgID string, opts ...BaseOpt) Base {
+	newOpts := []BaseOpt{WithDownstream()}
+	newOpts = append(newOpts, opts...)
+	return NewBase(StatusGatewayTimeout, msgID, newOpts...)
+}
+
 type BaseOpt func(Base) Base
 
 // WithLogLevel sets a custom log level for all errors instantiated from
@@ -167,6 +195,17 @@ func WithPublicMessage(message string) BaseOpt {
 	}
 }
 
+// WithDownstream sets the source as SourceDownstream that will be used
+// for errors based on this [Base].
+//
+// Used as a functional option to [NewBase].
+func WithDownstream() BaseOpt {
+	return func(b Base) Base {
+		b.source = SourceDownstream
+		return b
+	}
+}
+
 // Errorf creates a new [Error] with Reason and MessageID from [Base],
 // and Message and Underlying will be populated using the rules of
 // [fmt.Errorf].
@@ -180,6 +219,7 @@ func (b Base) Errorf(format string, args ...any) Error {
 		MessageID:     b.messageID,
 		Underlying:    errors.Unwrap(err),
 		LogLevel:      b.logLevel,
+		Source:        b.source,
 	}
 }
 
@@ -273,6 +313,8 @@ type Error struct {
 	PublicPayload map[string]any
 	// LogLevel provides a suggested level of logging for the error.
 	LogLevel LogLevel
+	// Source identifies from where the error originates.
+	Source Source
 }
 
 // MarshalJSON returns an error, we do not want raw [Error]s being

pkg/util/errutil/source.go

@@ -0,0 +1,18 @@
+package errutil
+
+// Source identifies from where an error originates.
+type Source string
+
+const (
+	// SourceServer implies error originates from within the server, i.e. this application.
+	SourceServer Source = "server"
+
+	// SourceDownstream implies error originates from response error while server acting
+	// as a proxy, i.e. from a downstream service.
+	SourceDownstream Source = "downstream"
+)
+
+// IsDownstream checks if Source is SourceDownstream.
+func (s Source) IsDownstream() bool {
+	return s == SourceDownstream
+}

pkg/util/errutil/status.go

@@ -46,6 +46,15 @@ const (
 	// features.
 	// HTTP status code 501.
 	StatusNotImplemented CoreStatus = "Not implemented"
+	// StatusBadGateway means that the server, while acting as a proxy,
+	// received an invalid response from the downstream server.
+	// HTTP status code 502.
+	StatusBadGateway CoreStatus = "Bad gateway"
+	// StatusGatewayTimeout means that the server, while acting as a proxy,
+	// did not receive a timely response from a downstream server it needed
+	// to access in order to complete the request.
+	// HTTP status code 504.
+	StatusGatewayTimeout CoreStatus = "Gateway timeout"
 )
 
 // StatusReason allows for wrapping of CoreStatus.
@@ -69,7 +78,7 @@ func (s CoreStatus) HTTPStatus() int {
 		return http.StatusForbidden
 	case StatusNotFound:
 		return http.StatusNotFound
-	case StatusTimeout:
+	case StatusTimeout, StatusGatewayTimeout:
 		return http.StatusGatewayTimeout
 	case StatusTooManyRequests:
 		return http.StatusTooManyRequests
@@ -77,6 +86,8 @@ func (s CoreStatus) HTTPStatus() int {
 		return http.StatusBadRequest
 	case StatusNotImplemented:
 		return http.StatusNotImplemented
+	case StatusBadGateway:
+		return http.StatusBadGateway
 	case StatusUnknown, StatusInternal:
 		return http.StatusInternalServerError
 	default:

pkg/util/proxyutil/reverse_proxy.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	glog "github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/middleware/requestmeta"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 )
 
@@ -103,6 +104,8 @@ func modifyResponse(logger glog.Logger) func(resp *http.Response) error {
 
 		SetProxyResponseHeaders(resp.Header)
 		SetViaHeader(resp.Header, resp.ProtoMajor, resp.ProtoMinor)
+
+		requestmeta.WithStatusSource(resp.Request.Context(), resp.StatusCode)
 		return nil
 	}
 }
@@ -120,6 +123,7 @@ type timeoutError interface {
 func errorHandler(logger glog.Logger) func(http.ResponseWriter, *http.Request, error) {
 	return func(w http.ResponseWriter, r *http.Request, err error) {
 		ctxLogger := logger.FromContext(r.Context())
+		requestmeta.WithDownstreamStatusSource(r.Context())
 
 		if errors.Is(err, context.Canceled) {
 			ctxLogger.Debug("Proxy request cancelled by client")

pkg/util/proxyutil/reverse_proxy_test.go

@@ -3,6 +3,7 @@ package proxyutil
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -11,6 +12,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/middleware/requestmeta"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -97,7 +99,7 @@ func TestReverseProxy(t *testing.T) {
 		require.NoError(t, resp.Body.Close())
 	})
 
-	t.Run("Error handling should convert status codes depending on what kind of error it is", func(t *testing.T) {
+	t.Run("Error handling should convert status codes depending on what kind of error it is and set downstream status source", func(t *testing.T) {
 		timedOutTransport := http.DefaultTransport.(*http.Transport)
 		timedOutTransport.ResponseHeaderTimeout = time.Millisecond
 
@@ -136,7 +138,12 @@ func TestReverseProxy(t *testing.T) {
 				}))
 				t.Cleanup(upstream.Close)
 				rec := httptest.NewRecorder()
+
+				ctx := requestmeta.SetRequestMetaData(context.Background(), requestmeta.RequestMetaData{
+					StatusSource: requestmeta.StatusSourceServer,
+				})
 				req := httptest.NewRequest(http.MethodGet, upstream.URL, nil)
+				req = req.WithContext(ctx)
 
 				rp := NewReverseProxy(
 					log.New("test"),
@@ -151,6 +158,55 @@ func TestReverseProxy(t *testing.T) {
 				resp := rec.Result()
 				require.Equal(t, tc.expectedStatusCode, resp.StatusCode)
 				require.NoError(t, resp.Body.Close())
+
+				rmd := requestmeta.GetRequestMetaData(ctx)
+				require.Equal(t, requestmeta.StatusSourceDownstream, rmd.StatusSource)
+			})
+		}
+	})
+
+	t.Run("5xx response status codes should set downstream status source", func(t *testing.T) {
+		testCases := []struct {
+			status         int
+			expectedSource requestmeta.StatusSource
+		}{
+			{status: http.StatusOK, expectedSource: requestmeta.StatusSourceServer},
+			{status: http.StatusBadRequest, expectedSource: requestmeta.StatusSourceServer},
+			{status: http.StatusForbidden, expectedSource: requestmeta.StatusSourceServer},
+			{status: http.StatusUnauthorized, expectedSource: requestmeta.StatusSourceServer},
+			{status: http.StatusInternalServerError, expectedSource: requestmeta.StatusSourceDownstream},
+			{status: http.StatusBadGateway, expectedSource: requestmeta.StatusSourceDownstream},
+			{status: http.StatusGatewayTimeout, expectedSource: requestmeta.StatusSourceDownstream},
+			{status: 599, expectedSource: requestmeta.StatusSourceDownstream},
+		}
+
+		for _, tc := range testCases {
+			t.Run(fmt.Sprintf("status %d => source %s ", tc.status, tc.expectedSource), func(t *testing.T) {
+				upstream := newUpstreamServer(t, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+					w.WriteHeader(tc.status)
+				}))
+				t.Cleanup(upstream.Close)
+				rec := httptest.NewRecorder()
+
+				ctx := requestmeta.SetRequestMetaData(context.Background(), requestmeta.RequestMetaData{
+					StatusSource: requestmeta.StatusSourceServer,
+				})
+				req := httptest.NewRequest(http.MethodGet, upstream.URL, nil)
+				req = req.WithContext(ctx)
+
+				rp := NewReverseProxy(
+					log.New("test"),
+					func(req *http.Request) {},
+				)
+				require.NotNil(t, rp)
+				rp.ServeHTTP(rec, req)
+
+				resp := rec.Result()
+				require.Equal(t, tc.status, resp.StatusCode)
+				require.NoError(t, resp.Body.Close())
+
+				rmd := requestmeta.GetRequestMetaData(ctx)
+				require.Equal(t, tc.expectedSource, rmd.StatusSource)
 			})
 		}
 	})