From 729f0a6ba5bcc34373705bc92eab0b8ad34e99ca Mon Sep 17 00:00:00 2001 From: Adam Simpson Date: Thu, 29 May 2025 17:30:45 -0400 Subject: [PATCH] ds-querier: handle downstream 500s (#105890) * ds-querier: add new metric for the total request Co-authored-by: Sarah Zinger * fix logger and trace Co-authored-by: Sarah Zinger * ds-querier: rewrite downstream 500s to 400 --------- Co-authored-by: Sarah Zinger --- pkg/registry/apis/query/query.go | 41 ++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/pkg/registry/apis/query/query.go b/pkg/registry/apis/query/query.go index 8018b1ec129..f99065ce898 100644 --- a/pkg/registry/apis/query/query.go +++ b/pkg/registry/apis/query/query.go @@ -92,17 +92,39 @@ func (r *queryREST) Connect(connectCtx context.Context, name string, _ runtime.O ctx, span := b.tracer.Start(httpreq.Context(), "QueryService.Query") defer span.End() ctx = request.WithNamespace(ctx, request.NamespaceValue(connectCtx)) - + traceId := span.SpanContext().TraceID() + connectLogger := b.log.New("traceId", traceId.String()) responder := newResponderWrapper(incomingResponder, - func(statusCode int, obj runtime.Object) { - if statusCode >= 400 { - r.logger.Debug("error found in success handler in connect", "statuscode", strconv.Itoa(statusCode)) - span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(statusCode))) + func(statusCode *int, obj runtime.Object) { + if *statusCode/100 == 4 { + span.SetStatus(codes.Error, strconv.Itoa(*statusCode)) + } + + if *statusCode >= 500 { + o, ok := obj.(*query.QueryDataResponse) + if ok && o.Responses != nil { + for refId, response := range o.Responses { + if response.ErrorSource == backend.ErrorSourceDownstream { + *statusCode = http.StatusBadRequest //force this to be a 400 since it's downstream + span.SetStatus(codes.Error, strconv.Itoa(*statusCode)) + span.SetAttributes(attribute.String("error.source", "downstream")) + break + } else if response.Error != nil { + connectLogger.Debug("500 error without downstream error source", "error", response.Error, "errorSource", response.ErrorSource, "refId", refId) + span.SetStatus(codes.Error, "500 error without downstream error source") + } else { + span.SetStatus(codes.Error, "500 error without downstream error source and no Error message") + span.SetAttributes(attribute.String("error.ref_id", refId)) + } + } + } } }, + func(err error) { - r.logger.Debug("error caught in handler", "err", err) + connectLogger.Error("error caught in handler", "err", err) span.SetStatus(codes.Error, "query error") + if err == nil { return } @@ -215,7 +237,6 @@ func (b *QueryAPIBuilder) execute(ctx context.Context, req parsedRequestInfo, in b.log.Debug("error in executeConcurrentQueries", "err", err) } } - if err != nil { b.log.Debug("error in query phase, skipping expressions", "error", err) return qdr, err //return early here to prevent expressions from being executed if we got an error during the query phase @@ -476,11 +497,11 @@ func (b *QueryAPIBuilder) convertQueryFromAlerting(ctx context.Context, req data type responderWrapper struct { wrapped rest.Responder - onObjectFn func(statusCode int, obj runtime.Object) + onObjectFn func(statusCode *int, obj runtime.Object) onErrorFn func(err error) } -func newResponderWrapper(responder rest.Responder, onObjectFn func(statusCode int, obj runtime.Object), onErrorFn func(err error)) *responderWrapper { +func newResponderWrapper(responder rest.Responder, onObjectFn func(statusCode *int, obj runtime.Object), onErrorFn func(err error)) *responderWrapper { return &responderWrapper{ wrapped: responder, onObjectFn: onObjectFn, @@ -490,7 +511,7 @@ func newResponderWrapper(responder rest.Responder, onObjectFn func(statusCode in func (r responderWrapper) Object(statusCode int, obj runtime.Object) { if r.onObjectFn != nil { - r.onObjectFn(statusCode, obj) + r.onObjectFn(&statusCode, obj) } r.wrapped.Object(statusCode, obj)