Instrumentation: Add status_source label to request metrics/logs (#74114)

Ref #68480

Co-authored-by: Giuseppe Guerra <giuseppe.guerra@grafana.com>

pkg/util/errutil/errors.go

@@ -15,6 +15,7 @@ type Base struct {
 	messageID     string
 	publicMessage string
 	logLevel      LogLevel
+	source        Source
 }
 
 // NewBase initializes a [Base] that is used to construct [Error].
@@ -32,6 +33,7 @@ func NewBase(reason StatusReason, msgID string, opts ...BaseOpt) Base {
 		reason:    reason,
 		messageID: msgID,
 		logLevel:  reason.Status().LogLevel(),
+		source:    SourceServer,
 	}
 
 	for _, opt := range opts {
@@ -143,6 +145,32 @@ func NotImplemented(msgID string, opts ...BaseOpt) Base {
 	return NewBase(StatusNotImplemented, msgID, opts...)
 }
 
+// BadGateway initializes a new [Base] error with reason StatusBadGateway
+// and source SourceDownstream that is used to construct [Error]. The msgID
+// is passed to the caller to serve as the base for user facing error messages.
+//
+// msgID should be structured as component.errorBrief, for example
+//
+//	area.downstreamError
+func BadGateway(msgID string, opts ...BaseOpt) Base {
+	newOpts := []BaseOpt{WithDownstream()}
+	newOpts = append(newOpts, opts...)
+	return NewBase(StatusBadGateway, msgID, newOpts...)
+}
+
+// GatewayTimeout initializes a new [Base] error with reason StatusGatewayTimeout
+// and source SourceDownstream that is used to construct [Error]. The msgID
+// is passed to the caller to serve as the base for user facing error messages.
+//
+// msgID should be structured as component.errorBrief, for example
+//
+//	area.downstreamTimeout
+func GatewayTimeout(msgID string, opts ...BaseOpt) Base {
+	newOpts := []BaseOpt{WithDownstream()}
+	newOpts = append(newOpts, opts...)
+	return NewBase(StatusGatewayTimeout, msgID, newOpts...)
+}
+
 type BaseOpt func(Base) Base
 
 // WithLogLevel sets a custom log level for all errors instantiated from
@@ -167,6 +195,17 @@ func WithPublicMessage(message string) BaseOpt {
 	}
 }
 
+// WithDownstream sets the source as SourceDownstream that will be used
+// for errors based on this [Base].
+//
+// Used as a functional option to [NewBase].
+func WithDownstream() BaseOpt {
+	return func(b Base) Base {
+		b.source = SourceDownstream
+		return b
+	}
+}
+
 // Errorf creates a new [Error] with Reason and MessageID from [Base],
 // and Message and Underlying will be populated using the rules of
 // [fmt.Errorf].
@@ -180,6 +219,7 @@ func (b Base) Errorf(format string, args ...any) Error {
 		MessageID:     b.messageID,
 		Underlying:    errors.Unwrap(err),
 		LogLevel:      b.logLevel,
+		Source:        b.source,
 	}
 }
 
@@ -273,6 +313,8 @@ type Error struct {
 	PublicPayload map[string]any
 	// LogLevel provides a suggested level of logging for the error.
 	LogLevel LogLevel
+	// Source identifies from where the error originates.
+	Source Source
 }
 
 // MarshalJSON returns an error, we do not want raw [Error]s being

pkg/util/errutil/source.go

@@ -0,0 +1,18 @@
+package errutil
+
+// Source identifies from where an error originates.
+type Source string
+
+const (
+	// SourceServer implies error originates from within the server, i.e. this application.
+	SourceServer Source = "server"
+
+	// SourceDownstream implies error originates from response error while server acting
+	// as a proxy, i.e. from a downstream service.
+	SourceDownstream Source = "downstream"
+)
+
+// IsDownstream checks if Source is SourceDownstream.
+func (s Source) IsDownstream() bool {
+	return s == SourceDownstream
+}

pkg/util/errutil/status.go

@@ -46,6 +46,15 @@ const (
 	// features.
 	// HTTP status code 501.
 	StatusNotImplemented CoreStatus = "Not implemented"
+	// StatusBadGateway means that the server, while acting as a proxy,
+	// received an invalid response from the downstream server.
+	// HTTP status code 502.
+	StatusBadGateway CoreStatus = "Bad gateway"
+	// StatusGatewayTimeout means that the server, while acting as a proxy,
+	// did not receive a timely response from a downstream server it needed
+	// to access in order to complete the request.
+	// HTTP status code 504.
+	StatusGatewayTimeout CoreStatus = "Gateway timeout"
 )
 
 // StatusReason allows for wrapping of CoreStatus.
@@ -69,7 +78,7 @@ func (s CoreStatus) HTTPStatus() int {
 		return http.StatusForbidden
 	case StatusNotFound:
 		return http.StatusNotFound
-	case StatusTimeout:
+	case StatusTimeout, StatusGatewayTimeout:
 		return http.StatusGatewayTimeout
 	case StatusTooManyRequests:
 		return http.StatusTooManyRequests
@@ -77,6 +86,8 @@ func (s CoreStatus) HTTPStatus() int {
 		return http.StatusBadRequest
 	case StatusNotImplemented:
 		return http.StatusNotImplemented
+	case StatusBadGateway:
+		return http.StatusBadGateway
 	case StatusUnknown, StatusInternal:
 		return http.StatusInternalServerError
 	default: