public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Instrumentation: Add status_source label to request metrics/logs (#74114)

Browse Source 
Ref #68480

Co-authored-by: Giuseppe Guerra <giuseppe.guerra@grafana.com>
This commit is contained in:
Marcus Efraimsson
2023-09-11 12:13:13 +02:00
committed by GitHub
co-authored by Giuseppe Guerra
parent 97d568e60a
commit 8ee43f3705
27 changed files with 487 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/metrics.go
+9 -2
View File
@@ -1,6 +1,7 @@
package api
import (
"context"
"errors"
"fmt"
"net/http"
@@ -9,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/api/dtos"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/middleware/requestmeta"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -60,10 +62,10 @@ func (hs *HTTPServer) QueryMetricsV2(c *contextmodel.ReqContext) response.Respon
if err != nil {
return hs.handleQueryMetricsError(err)
}
return hs.toJsonStreamingResponse(resp)
return hs.toJsonStreamingResponse(c.Req.Context(), resp)
}
func (hs *HTTPServer) toJsonStreamingResponse(qdr *backend.QueryDataResponse) response.Response {
func (hs *HTTPServer) toJsonStreamingResponse(ctx context.Context, qdr *backend.QueryDataResponse) response.Response {
statusWhenError := http.StatusBadRequest
if hs.Features.IsEnabled(featuremgmt.FlagDatasourceQueryMultiStatus) {
statusWhenError = http.StatusMultiStatus
@@ -76,6 +78,11 @@ func (hs *HTTPServer) toJsonStreamingResponse(qdr *backend.QueryDataResponse) re
}
}
if statusCode == statusWhenError {
// an error in the response we treat as downstream.
requestmeta.WithDownstreamStatusSource(ctx)
}
return response.JSONStreaming(statusCode, qdr)
}
pkg/api/metrics_test.go
-1
View File
@@ -308,7 +308,6 @@ func TestDataSourceQueryError(t *testing.T) {
resp, err := srv.SendJSON(req)
require.NoError(t, err)
require.Equal(t, tc.expectedStatus, resp.StatusCode)
require.Equal(t, tc.expectedStatus, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
pkg/api/plugin_resource.go
+12 -3
View File
@@ -9,6 +9,8 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/middleware/requestmeta"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/httpresponsesender"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
@@ -44,7 +46,10 @@ func (hs *HTTPServer) callPluginResource(c *contextmodel.ReqContext, pluginID st
if err = hs.makePluginResourceRequest(c.Resp, req, pCtx); err != nil {
handleCallResourceError(err, c)
return
}
requestmeta.WithStatusSource(c.Req.Context(), c.Resp.Status())
}
func (hs *HTTPServer) callPluginResourceWithDataSource(c *contextmodel.ReqContext, pluginID string, ds *datasources.DataSource) {
@@ -77,7 +82,10 @@ func (hs *HTTPServer) callPluginResourceWithDataSource(c *contextmodel.ReqContex
if err = hs.makePluginResourceRequest(c.Resp, req, pCtx); err != nil {
handleCallResourceError(err, c)
return
}
requestmeta.WithStatusSource(c.Req.Context(), c.Resp.Status())
}
func (hs *HTTPServer) pluginResourceRequest(c *contextmodel.ReqContext) (*http.Request, error) {
@@ -118,14 +126,15 @@ func (hs *HTTPServer) makePluginResourceRequest(w http.ResponseWriter, req *http
func handleCallResourceError(err error, reqCtx *contextmodel.ReqContext) {
if errors.Is(err, backendplugin.ErrPluginUnavailable) {
reqCtx.JsonApiErr(503, "Plugin unavailable", err)
reqCtx.JsonApiErr(http.StatusServiceUnavailable, "Plugin unavailable", err)
return
}
if errors.Is(err, backendplugin.ErrMethodNotImplemented) {
reqCtx.JsonApiErr(404, "Not found", err)
reqCtx.JsonApiErr(http.StatusNotFound, "Not found", err)
return
}
reqCtx.JsonApiErr(500, "Failed to call resource", err)
resp := response.ErrOrFallback(http.StatusInternalServerError, "Failed to call resource", err)
resp.WriteTo(reqCtx)
}
pkg/api/pluginproxy/ds_proxy.go
+1
View File
@@ -120,6 +120,7 @@ func (proxy *DataSourceProxy) HandleRequest() {
Body: io.NopCloser(strings.NewReader(msg)),
ContentLength: int64(len(msg)),
Header: http.Header{},
Request: resp.Request,
}
}
return nil
pkg/api/plugins.go
+8 -12
View File
@@ -395,9 +395,9 @@ func (hs *HTTPServer) CheckHealth(c *contextmodel.ReqContext) response.Response
pCtx, err := hs.pluginContextProvider.Get(c.Req.Context(), pluginID, c.SignedInUser, c.OrgID)
if err != nil {
if errors.Is(err, plugincontext.ErrPluginNotFound) {
return response.Error(404, "Plugin not found", nil)
return response.Error(http.StatusNotFound, "Plugin not found", nil)
}
return response.Error(500, "Failed to get plugin settings", err)
return response.Error(http.StatusInternalServerError, "Failed to get plugin settings", err)
}
resp, err := hs.pluginClient.CheckHealth(c.Req.Context(), &backend.CheckHealthRequest{
PluginContext: pCtx,
@@ -417,14 +417,14 @@ func (hs *HTTPServer) CheckHealth(c *contextmodel.ReqContext) response.Response
var jsonDetails map[string]any
err = json.Unmarshal(resp.JSONDetails, &jsonDetails)
if err != nil {
return response.Error(500, "Failed to unmarshal detailed response from backend plugin", err)
return response.Error(http.StatusInternalServerError, "Failed to unmarshal detailed response from backend plugin", err)
}
payload["details"] = jsonDetails
}
if resp.Status != backend.HealthStatusOk {
return response.JSON(503, payload)
return response.JSON(http.StatusBadRequest, payload)
}
return response.JSON(http.StatusOK, payload)
@@ -492,22 +492,18 @@ func (hs *HTTPServer) UninstallPlugin(c *contextmodel.ReqContext) response.Respo
func translatePluginRequestErrorToAPIError(err error) response.Response {
if errors.Is(err, backendplugin.ErrPluginNotRegistered) {
return response.Error(404, "Plugin not found", err)
return response.Error(http.StatusNotFound, "Plugin not found", err)
}
if errors.Is(err, backendplugin.ErrMethodNotImplemented) {
return response.Error(404, "Not found", err)
}
if errors.Is(err, backendplugin.ErrHealthCheckFailed) {
return response.Error(500, "Plugin health check failed", err)
return response.Error(http.StatusNotFound, "Not found", err)
}
if errors.Is(err, backendplugin.ErrPluginUnavailable) {
return response.Error(503, "Plugin unavailable", err)
return response.Error(http.StatusServiceUnavailable, "Plugin unavailable", err)
}
return response.Error(500, "Plugin request failed", err)
return response.ErrOrFallback(http.StatusInternalServerError, "Plugin request failed", err)
}
func (hs *HTTPServer) pluginMarkdown(ctx context.Context, pluginID string, name string) ([]byte, error) {
pkg/api/response/response.go
+6
View File
@@ -12,6 +12,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/middleware/requestmeta"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util/errutil"
@@ -80,6 +81,11 @@ func (r *NormalResponse) ErrMessage() string {
func (r *NormalResponse) WriteTo(ctx *contextmodel.ReqContext) {
if r.err != nil {
grafanaErr := errutil.Error{}
if errors.As(r.err, &grafanaErr) && grafanaErr.Source.IsDownstream() {
requestmeta.WithDownstreamStatusSource(ctx.Req.Context())
}
if errutil.HasUnifiedLogging(ctx.Req.Context()) {
ctx.Error = r.err
} else {