Plugins: Improve instrumentation by adding metrics and tracing (#61035)

* WIP: Plugins tracing * Trace ID middleware * Add prometheus metrics and tracing to plugins updater * Add TODOs * Add instrumented http client * Add tracing to grafana update checker * Goimports * Moved plugins tracing to middleware * goimports, fix tests * Removed X-Trace-Id header * Fix comment in NewTracingHeaderMiddleware * Add metrics to instrumented http client * Add instrumented http client options * Removed unused function * Switch to contextual logger * Refactoring, fix tests * Moved InstrumentedHTTPClient and PrometheusMetrics to their own package * Tracing middleware: handle errors * Report span status codes when recording errors * Add tests for tracing middleware * Moved fakeSpan and fakeTracer to pkg/infra/tracing * Add TestHTTPClientTracing * Lint * Changes after PR review * Tests: Made "ended" in FakeSpan private, allow calling End only once * Testing: panic in FakeSpan if span already ended * Refactoring: Simplify Grafana updater checks * Refactoring: Simplify plugins updater error checks and logs * Fix wrong call to checkForUpdates -> instrumentedCheckForUpdates * Tests: Fix wrong call to checkForUpdates -> instrumentedCheckForUpdates * Log update checks duration, use Info log level for check succeeded logs * Add plugin context span attributes in tracing_middleware * Refactor prometheus metrics as httpclient middleware * Fix call to ProvidePluginsService in plugins_test.go * Propagate context to update checker outgoing http requests * Plugin client tracing middleware: Removed operation name in status * Fix tests * Goimports tracing_middleware.go * Goimports * Fix imports * Changed span name to plugins client middleware * Add span name assertion in TestTracingMiddleware * Removed Prometheus metrics middleware from grafana and plugins updatechecker * Add span attributes for ds name, type, uid, panel and dashboard ids * Fix http header reading in tracing middlewares * Use contexthandler.FromContext, add X-Query-Group-Id * Add test for RunStream * Fix imports * Changes from PR review * TestTracingMiddleware: Changed assert to require for didPanic assertion * Lint * Fix imports
2023-03-28 11:01:06 +02:00
parent 0beb768427
commit a89202eab2
16 changed files with 1028 additions and 72 deletions
@@ -18,9 +18,13 @@ import (

 type TestClient struct {
 	plugins.Client
-	QueryDataFunc    backend.QueryDataHandlerFunc
-	CallResourceFunc backend.CallResourceHandlerFunc
-	CheckHealthFunc  backend.CheckHealthHandlerFunc
+	QueryDataFunc       backend.QueryDataHandlerFunc
+	CallResourceFunc    backend.CallResourceHandlerFunc
+	CheckHealthFunc     backend.CheckHealthHandlerFunc
+	CollectMetricsFunc  backend.CollectMetricsHandlerFunc
+	SubscribeStreamFunc func(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error)
+	PublishStreamFunc   func(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error)
+	RunStreamFunc       func(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error
 }

 func (c *TestClient) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
@@ -47,6 +51,37 @@ func (c *TestClient) CheckHealth(ctx context.Context, req *backend.CheckHealthRe
 	return nil, nil
 }

+func (c *TestClient) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) {
+	if c.CollectMetricsFunc != nil {
+		return c.CollectMetricsFunc(ctx, req)
+	}
+
+	return nil, nil
+}
+
+func (c *TestClient) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) {
+	if c.PublishStreamFunc != nil {
+		return c.PublishStreamFunc(ctx, req)
+	}
+
+	return nil, nil
+}
+
+func (c *TestClient) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) {
+	if c.SubscribeStreamFunc != nil {
+		return c.SubscribeStreamFunc(ctx, req)
+	}
+
+	return nil, nil
+}
+
+func (c *TestClient) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error {
+	if c.RunStreamFunc != nil {
+		return c.RunStreamFunc(ctx, req, sender)
+	}
+	return nil
+}
+
 type MiddlewareScenarioContext struct {
 	QueryDataCallChain       []string
 	CallResourceCallChain    []string
@@ -125,18 +160,24 @@ func (m *TestMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRe
 var _ plugins.Client = &TestClient{}

 type ClientDecoratorTest struct {
-	T               *testing.T
-	Context         context.Context
-	TestClient      *TestClient
-	Middlewares     []plugins.ClientMiddleware
-	Decorator       *client.Decorator
-	ReqContext      *contextmodel.ReqContext
-	QueryDataReq    *backend.QueryDataRequest
-	QueryDataCtx    context.Context
-	CallResourceReq *backend.CallResourceRequest
-	CallResourceCtx context.Context
-	CheckHealthReq  *backend.CheckHealthRequest
-	CheckHealthCtx  context.Context
+	T                  *testing.T
+	Context            context.Context
+	TestClient         *TestClient
+	Middlewares        []plugins.ClientMiddleware
+	Decorator          *client.Decorator
+	ReqContext         *contextmodel.ReqContext
+	QueryDataReq       *backend.QueryDataRequest
+	QueryDataCtx       context.Context
+	CallResourceReq    *backend.CallResourceRequest
+	CallResourceCtx    context.Context
+	CheckHealthReq     *backend.CheckHealthRequest
+	CheckHealthCtx     context.Context
+	CollectMetricsReq  *backend.CollectMetricsRequest
+	CollectMetricsCtx  context.Context
+	SubscribeStreamReq *backend.SubscribeStreamRequest
+	SubscribeStreamCtx context.Context
+	PublishStreamReq   *backend.PublishStreamRequest
+	PublishStreamCtx   context.Context
 }

 type ClientDecoratorTestOption func(*ClientDecoratorTest)
@@ -162,6 +203,21 @@ func NewClientDecoratorTest(t *testing.T, opts ...ClientDecoratorTestOption) *Cl
 			cdt.CheckHealthCtx = ctx
 			return nil, nil
 		},
+		CollectMetricsFunc: func(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) {
+			cdt.CollectMetricsReq = req
+			cdt.CollectMetricsCtx = ctx
+			return nil, nil
+		},
+		SubscribeStreamFunc: func(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) {
+			cdt.SubscribeStreamReq = req
+			cdt.SubscribeStreamCtx = ctx
+			return nil, nil
+		},
+		PublishStreamFunc: func(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) {
+			cdt.PublishStreamReq = req
+			cdt.PublishStreamCtx = ctx
+			return nil, nil
+		},
 	}
 	require.NotNil(t, cdt)