Plugins: Add status_source label to plugin request metrics (#76236)

* Plugins: Chore: Renamed instrumentation middleware to metrics middleware

* Removed repeated logger attributes in middleware and contextual logger

* renamed loggerParams to logParams

* PR review suggestion

* Add pluginsInstrumentationStatusSource feature toggle

* Plugin error source prometheus metrics

* Add error_source to logs

* re-generate feature toggles

* fix compilation issues

* remove unwanted changes

* Removed logger middleware changes, implement error source using context

* Renamed pluginmeta to pluginrequestmeta, changed some method names

* Fix comment

* pluginrequestmeta.go -> plugin_request_meta.go

* Replaced plugin request meta with status source

* Add tests for pluginrequestmeta status source

* Fix potential nil pointer dereference in instrmentation middleware

* Add metrics middleware tests

* Sort imports in clienttest.go

* Add StatusSourceFromContext test

* Add error_source label to plugin_request_duration_seconds

* Re-generate feature flags

* lint

* Use StatusSourcePlugin by default

* re-generate feature flags

pkg/plugins/pluginrequestmeta/plugin_request_meta.go

@@ -0,0 +1,44 @@
+package pluginrequestmeta
+
+import (
+	"context"
+	"errors"
+)
+
+// StatusSource is an enum-like string value representing the source of a
+// plugin query data response status code
+type StatusSource string
+
+const (
+	StatusSourcePlugin     StatusSource = "plugin"
+	StatusSourceDownstream StatusSource = "downstream"
+)
+
+type statusSourceCtxKey struct{}
+
+// StatusSourceFromContext returns the plugin request status source stored in the context.
+// If no plugin request status source is stored in the context, [StatusSourcePlugin] is returned.
+func StatusSourceFromContext(ctx context.Context) StatusSource {
+	value, ok := ctx.Value(statusSourceCtxKey{}).(*StatusSource)
+	if ok {
+		return *value
+	}
+	return StatusSourcePlugin
+}
+
+// WithStatusSource sets the plugin request status source for the context.
+func WithStatusSource(ctx context.Context, s StatusSource) context.Context {
+	return context.WithValue(ctx, statusSourceCtxKey{}, &s)
+}
+
+// WithDownstreamStatusSource mutates the provided context by setting the plugin request status source to
+// StatusSourceDownstream. If the provided context does not have a plugin request status source, the context
+// will not be mutated. This means that [WithStatusSource] has to be called before this function.
+func WithDownstreamStatusSource(ctx context.Context) error {
+	v, ok := ctx.Value(statusSourceCtxKey{}).(*StatusSource)
+	if !ok {
+		return errors.New("the provided context does not have a plugin request status source")
+	}
+	*v = StatusSourceDownstream
+	return nil
+}