Instrumentation: Check embedded errors in query data response for plugin metrics/logs status label (#77613)

Check embedded errors in query data response for plugin metrics/logs status label.
Plugin Request Completed log messages are now logged with info level if status=ok, 
otherwise error level.

Fixes #76769

pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go

@@ -2,7 +2,6 @@ package clientmiddleware
 
 import (
 	"context"
-	"errors"
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -37,31 +36,32 @@ type LoggerMiddleware struct {
 	features featuremgmt.FeatureToggles
 }
 
-func (m *LoggerMiddleware) logRequest(ctx context.Context, fn func(ctx context.Context) error) error {
-	status := statusOK
+func (m *LoggerMiddleware) logRequest(ctx context.Context, fn func(ctx context.Context) (requestStatus, error)) error {
 	start := time.Now()
 	timeBeforePluginRequest := log.TimeSinceStart(ctx, start)
 
-	err := fn(ctx)
-	if err != nil {
-		status = statusError
-		if errors.Is(err, context.Canceled) {
-			status = statusCancelled
-		}
-	}
+	status, err := fn(ctx)
 	logParams := []any{
 		"status", status,
 		"duration", time.Since(start),
 		"eventName", "grafana-data-egress",
 		"time_before_plugin_request", timeBeforePluginRequest,
 	}
-	if status == statusError {
+	if err != nil {
 		logParams = append(logParams, "error", err)
 	}
 	if m.features.IsEnabled(featuremgmt.FlagPluginsInstrumentationStatusSource) {
 		logParams = append(logParams, "status_source", pluginrequestmeta.StatusSourceFromContext(ctx))
 	}
-	m.logger.FromContext(ctx).Info("Plugin Request Completed", logParams...)
+
+	ctxLogger := m.logger.FromContext(ctx)
+	logFunc := ctxLogger.Info
+	if status > requestStatusOK {
+		logFunc = ctxLogger.Error
+	}
+
+	logFunc("Plugin Request Completed", logParams...)
+
 	return err
 }
 
@@ -71,11 +71,11 @@ func (m *LoggerMiddleware) QueryData(ctx context.Context, req *backend.QueryData
 	}
 
 	var resp *backend.QueryDataResponse
-	err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) {
+	err := m.logRequest(ctx, func(ctx context.Context) (status requestStatus, innerErr error) {
 		resp, innerErr = m.next.QueryData(ctx, req)
 
 		if innerErr != nil {
-			return innerErr
+			return requestStatusFromError(innerErr), innerErr
 		}
 
 		ctxLogger := m.logger.FromContext(ctx)
@@ -85,7 +85,7 @@ func (m *LoggerMiddleware) QueryData(ctx context.Context, req *backend.QueryData
 			}
 		}
 
-		return nil
+		return requestStatusFromQueryDataResponse(resp, innerErr), innerErr
 	})
 
 	return resp, err
@@ -96,9 +96,9 @@ func (m *LoggerMiddleware) CallResource(ctx context.Context, req *backend.CallRe
 		return m.next.CallResource(ctx, req, sender)
 	}
 
-	err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) {
+	err := m.logRequest(ctx, func(ctx context.Context) (status requestStatus, innerErr error) {
 		innerErr = m.next.CallResource(ctx, req, sender)
-		return innerErr
+		return requestStatusFromError(innerErr), innerErr
 	})
 
 	return err
@@ -110,9 +110,9 @@ func (m *LoggerMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHe
 	}
 
 	var resp *backend.CheckHealthResult
-	err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) {
+	err := m.logRequest(ctx, func(ctx context.Context) (status requestStatus, innerErr error) {
 		resp, innerErr = m.next.CheckHealth(ctx, req)
-		return innerErr
+		return requestStatusFromError(innerErr), innerErr
 	})
 
 	return resp, err
@@ -124,9 +124,9 @@ func (m *LoggerMiddleware) CollectMetrics(ctx context.Context, req *backend.Coll
 	}
 
 	var resp *backend.CollectMetricsResult
-	err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) {
+	err := m.logRequest(ctx, func(ctx context.Context) (status requestStatus, innerErr error) {
 		resp, innerErr = m.next.CollectMetrics(ctx, req)
-		return innerErr
+		return requestStatusFromError(innerErr), innerErr
 	})
 
 	return resp, err