Instrumentation: Add status_source label to request metrics/logs (#74114)

Ref #68480

Co-authored-by: Giuseppe Guerra <giuseppe.guerra@grafana.com>

pkg/util/errutil/errors.go

@@ -15,6 +15,7 @@ type Base struct {
 	messageID     string
 	publicMessage string
 	logLevel      LogLevel
+	source        Source
 }
 
 // NewBase initializes a [Base] that is used to construct [Error].
@@ -32,6 +33,7 @@ func NewBase(reason StatusReason, msgID string, opts ...BaseOpt) Base {
 		reason:    reason,
 		messageID: msgID,
 		logLevel:  reason.Status().LogLevel(),
+		source:    SourceServer,
 	}
 
 	for _, opt := range opts {
@@ -143,6 +145,32 @@ func NotImplemented(msgID string, opts ...BaseOpt) Base {
 	return NewBase(StatusNotImplemented, msgID, opts...)
 }
 
+// BadGateway initializes a new [Base] error with reason StatusBadGateway
+// and source SourceDownstream that is used to construct [Error]. The msgID
+// is passed to the caller to serve as the base for user facing error messages.
+//
+// msgID should be structured as component.errorBrief, for example
+//
+//	area.downstreamError
+func BadGateway(msgID string, opts ...BaseOpt) Base {
+	newOpts := []BaseOpt{WithDownstream()}
+	newOpts = append(newOpts, opts...)
+	return NewBase(StatusBadGateway, msgID, newOpts...)
+}
+
+// GatewayTimeout initializes a new [Base] error with reason StatusGatewayTimeout
+// and source SourceDownstream that is used to construct [Error]. The msgID
+// is passed to the caller to serve as the base for user facing error messages.
+//
+// msgID should be structured as component.errorBrief, for example
+//
+//	area.downstreamTimeout
+func GatewayTimeout(msgID string, opts ...BaseOpt) Base {
+	newOpts := []BaseOpt{WithDownstream()}
+	newOpts = append(newOpts, opts...)
+	return NewBase(StatusGatewayTimeout, msgID, newOpts...)
+}
+
 type BaseOpt func(Base) Base
 
 // WithLogLevel sets a custom log level for all errors instantiated from
@@ -167,6 +195,17 @@ func WithPublicMessage(message string) BaseOpt {
 	}
 }
 
+// WithDownstream sets the source as SourceDownstream that will be used
+// for errors based on this [Base].
+//
+// Used as a functional option to [NewBase].
+func WithDownstream() BaseOpt {
+	return func(b Base) Base {
+		b.source = SourceDownstream
+		return b
+	}
+}
+
 // Errorf creates a new [Error] with Reason and MessageID from [Base],
 // and Message and Underlying will be populated using the rules of
 // [fmt.Errorf].
@@ -180,6 +219,7 @@ func (b Base) Errorf(format string, args ...any) Error {
 		MessageID:     b.messageID,
 		Underlying:    errors.Unwrap(err),
 		LogLevel:      b.logLevel,
+		Source:        b.source,
 	}
 }
 
@@ -273,6 +313,8 @@ type Error struct {
 	PublicPayload map[string]any
 	// LogLevel provides a suggested level of logging for the error.
 	LogLevel LogLevel
+	// Source identifies from where the error originates.
+	Source Source
 }
 
 // MarshalJSON returns an error, we do not want raw [Error]s being