public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ds-querier: handle downstream 500s (#105890)

Browse Source 
* ds-querier: add new metric for the total request

Co-authored-by: Sarah Zinger <sarah.zinger@grafana.com>

* fix logger and trace

Co-authored-by: Sarah Zinger <sarah.zinger@grafana.com>

* ds-querier: rewrite downstream 500s to 400

---------

Co-authored-by: Sarah Zinger <sarah.zinger@grafana.com>
This commit is contained in:
Adam Simpson
2025-05-29 17:30:45 -04:00
committed by GitHub
co-authored by Sarah Zinger
parent eae77aa695
commit 729f0a6ba5
1 changed files with 31 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/registry/apis/query/query.go
+31 -10
View File
@@ -92,17 +92,39 @@ func (r *queryREST) Connect(connectCtx context.Context, name string, _ runtime.O
ctx, span := b.tracer.Start(httpreq.Context(), "QueryService.Query")
defer span.End()
ctx = request.WithNamespace(ctx, request.NamespaceValue(connectCtx))
traceId := span.SpanContext().TraceID()
connectLogger := b.log.New("traceId", traceId.String())
responder := newResponderWrapper(incomingResponder,
func(statusCode int, obj runtime.Object) {
if statusCode >= 400 {
r.logger.Debug("error found in success handler in connect", "statuscode", strconv.Itoa(statusCode))
span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(statusCode)))
func(statusCode *int, obj runtime.Object) {
if *statusCode/100 == 4 {
span.SetStatus(codes.Error, strconv.Itoa(*statusCode))
}
if *statusCode >= 500 {
o, ok := obj.(*query.QueryDataResponse)
if ok && o.Responses != nil {
for refId, response := range o.Responses {
if response.ErrorSource == backend.ErrorSourceDownstream {
*statusCode = http.StatusBadRequest //force this to be a 400 since it's downstream
span.SetStatus(codes.Error, strconv.Itoa(*statusCode))
span.SetAttributes(attribute.String("error.source", "downstream"))
break
} else if response.Error != nil {
connectLogger.Debug("500 error without downstream error source", "error", response.Error, "errorSource", response.ErrorSource, "refId", refId)
span.SetStatus(codes.Error, "500 error without downstream error source")
} else {
span.SetStatus(codes.Error, "500 error without downstream error source and no Error message")
span.SetAttributes(attribute.String("error.ref_id", refId))
}
}
}
}
},
func(err error) {
r.logger.Debug("error caught in handler", "err", err)
connectLogger.Error("error caught in handler", "err", err)
span.SetStatus(codes.Error, "query error")
if err == nil {
return
}
@@ -215,7 +237,6 @@ func (b *QueryAPIBuilder) execute(ctx context.Context, req parsedRequestInfo, in
b.log.Debug("error in executeConcurrentQueries", "err", err)
}
}
if err != nil {
b.log.Debug("error in query phase, skipping expressions", "error", err)
return qdr, err //return early here to prevent expressions from being executed if we got an error during the query phase
@@ -476,11 +497,11 @@ func (b *QueryAPIBuilder) convertQueryFromAlerting(ctx context.Context, req data
type responderWrapper struct {
wrapped rest.Responder
onObjectFn func(statusCode int, obj runtime.Object)
onObjectFn func(statusCode *int, obj runtime.Object)
onErrorFn func(err error)
}
func newResponderWrapper(responder rest.Responder, onObjectFn func(statusCode int, obj runtime.Object), onErrorFn func(err error)) *responderWrapper {
func newResponderWrapper(responder rest.Responder, onObjectFn func(statusCode *int, obj runtime.Object), onErrorFn func(err error)) *responderWrapper {
return &responderWrapper{
wrapped: responder,
onObjectFn: onObjectFn,
@@ -490,7 +511,7 @@ func newResponderWrapper(responder rest.Responder, onObjectFn func(statusCode in
func (r responderWrapper) Object(statusCode int, obj runtime.Object) {
if r.onObjectFn != nil {
r.onObjectFn(statusCode, obj)
r.onObjectFn(&statusCode, obj)
}
r.wrapped.Object(statusCode, obj)