Plugins: Refactoring: Implement plugin instrumentation as a middleware (#76011)

* Plugins: Refactor instrumentation as plugin client middleware

* Simplify repeated code

* Fix compilation error

* Add comments

* Moved status and endpoint consts to utils.go

* Fix wrong endpoint name in CheckHealth InstrumentationMiddleware

* Add tests

* Fix wrong endpoint value in instrumentPluginRequestSize

* removed todo

* PR review feedback: use MustRegister

* PR review feedback: move tracing middleware before instrumentation middleware

* PR review feedback: removed decommissioned check

* PR review feedback: extract prometheus metrics into separate variables

pkg/api/plugin_resource_test.go

@@ -11,12 +11,14 @@ import (
 
 	"github.com/grafana/grafana-azure-sdk-go/azsettings"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/localcache"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
 	pluginClient "github.com/grafana/grafana/pkg/plugins/manager/client"
 	"github.com/grafana/grafana/pkg/plugins/manager/fakes"
@@ -87,13 +89,20 @@ func TestCallResource(t *testing.T) {
 		require.NoError(t, resp.Body.Close())
 		require.Equal(t, 200, resp.StatusCode)
 	})
-
+	pluginRegistry := fakes.NewFakePluginRegistry()
+	require.NoError(t, pluginRegistry.Add(context.Background(), &plugins.Plugin{
+		JSONData: plugins.JSONData{
+			ID:      "grafana-testdata-datasource",
+			Backend: true,
+		},
+	}))
+	middlewares := pluginsintegration.CreateMiddlewares(cfg, &oauthtokentest.Service{}, tracing.InitializeTracerForTest(), &caching.OSSCachingService{}, &featuremgmt.FeatureManager{}, prometheus.DefaultRegisterer, pluginRegistry)
 	pc, err := pluginClient.NewDecorator(&fakes.FakePluginClient{
 		CallResourceHandlerFunc: backend.CallResourceHandlerFunc(func(ctx context.Context,
 			req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
 			return errors.New("something went wrong")
 		}),
-	}, pluginsintegration.CreateMiddlewares(cfg, &oauthtokentest.Service{}, tracing.InitializeTracerForTest(), &caching.OSSCachingService{}, &featuremgmt.FeatureManager{})...)
+	}, middlewares...)
 	require.NoError(t, err)
 
 	srv = SetupAPITestServer(t, func(hs *HTTPServer) {