Plugins: Make backend plugin metrics endpoints available with optional authentication (#46467)

* add new endpoint without auth+config

* add cfg check

* fit lint issue

* Add basic auth support

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

* WIP docs

* Update docs/sources/administration/view-server/internal-metrics.md

Co-authored-by: Dave Henderson <dhenderson@gmail.com>

* update instructions

Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
Co-authored-by: Dave Henderson <dhenderson@gmail.com>

conf/defaults.ini

@@ -896,14 +896,14 @@ enabled = true
 enabled = false
 
 #################################### Internal Grafana Metrics ############
-# Metrics available at HTTP API Url /metrics
+# Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId
 [metrics]
 enabled              = true
 interval_seconds     = 10
 # Disable total stats (stat_totals_*) metrics to be generated
 disable_total_stats = false
 
-#If both are set, basic auth will be required for the metrics endpoint.
+#If both are set, basic auth will be required for the metrics endpoints.
 basic_auth_username =
 basic_auth_password =
 

conf/sample.ini

@@ -878,7 +878,7 @@
 ;enabled = false
 
 #################################### Internal Grafana Metrics ##########################
-# Metrics available at HTTP API Url /metrics
+# Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId
 [metrics]
 # Disable / Enable internal metrics
 ;enabled           = true
@@ -887,7 +887,7 @@
 # Disable total stats (stat_totals_*) metrics to be generated
 ;disable_total_stats = false
 
-#If both are set, basic auth will be required for the metrics endpoint.
+#If both are set, basic auth will be required for the metrics endpoints.
 ; basic_auth_username =
 ; basic_auth_password =
 

devenv/docker/blocks/prometheus/prometheus.yml

@@ -37,3 +37,8 @@ scrape_configs:
   - job_name: 'prometheus-random-data'
     static_configs:
       - targets: ['prometheus-random-data:8080']
+
+  # - job_name: 'grafana-test-datasource'
+  #   metrics_path: /metrics/plugins/grafana-test-datasource
+  #   static_configs:
+  #     - targets: ['host.docker.internal:3000']

docs/sources/administration/view-server/internal-metrics.md

@@ -30,7 +30,7 @@ These instructions assume you have already added Prometheus as a data source in
 1. Enable Prometheus to scrape metrics from Grafana. In your configuration file (`grafana.ini` or `custom.ini` depending on your operating system) remove the semicolon to enable the following configuration options:
 
    ```
-   # Metrics available at HTTP API Url /metrics
+   # Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId
    [metrics]
    # Disable / Enable internal metrics
    enabled           = true
@@ -39,7 +39,7 @@ These instructions assume you have already added Prometheus as a data source in
    disable_total_stats = false
    ```
 
-1. (optional) If you want to require authorization to view the metrics endpoint, then uncomment and set the following options:
+1. (optional) If you want to require authorization to view the metrics endpoints, then uncomment and set the following options:
 
    ```
    basic_auth_username =
@@ -92,3 +92,48 @@ These instructions assume you have already added Graphite as a data source in Gr
    ```
 
 1. Restart Grafana. Grafana now exposes metrics at http://localhost:3000/metrics and sends them to the Graphite location you specified.
+
+## Pull metrics from Grafana backend plugin into Prometheus
+
+Any installed [backend plugin]({{< relref "../../developers/plugins/backend/_index.md" >}}) exposes a metrics endpoint through Grafana that you can configure Prometheus to scrape.
+
+These instructions assume you have already added Prometheus as a data source in Grafana.
+
+1. Enable Prometheus to scrape backend plugin metrics from Grafana. In your configuration file (`grafana.ini` or `custom.ini` depending on your operating system) remove the semicolon to enable the following configuration options:
+
+   ```
+   # Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId
+   [metrics]
+   # Disable / Enable internal metrics
+   enabled           = true
+
+   # Disable total stats (stat_totals_*) metrics to be generated
+   disable_total_stats = false
+   ```
+
+1. (optional) If you want to require authorization to view the metrics endpoints, then uncomment and set the following options:
+
+   ```
+   basic_auth_username =
+   basic_auth_password =
+   ```
+
+1. Restart Grafana. Grafana now exposes metrics at `http://localhost:3000/metrics/plugins/<plugin id>`, e.g. http://localhost:3000/metrics/plugins/grafana-github-datasource if you have the [Grafana GitHub datasource](https://grafana.com/grafana/plugins/grafana-github-datasource/) installed.
+1. Add the job to your prometheus.yml file.
+   Example:
+
+   ```
+   - job_name: 'grafana_github_datasource'
+
+      scrape_interval: 15s
+      scrape_timeout: 5s
+      metrics_path: /metrics/plugins/grafana-test-datasource
+
+      static_configs:
+        - targets: ['localhost:3000']
+   ```
+
+1. Restart Prometheus. Your new job should appear on the Targets tab.
+1. In Grafana, hover your mouse over the **Configuration** (gear) icon on the left sidebar and then click **Data Sources**.
+1. Select the **Prometheus** data source.
+1. Import a Golang application metrics dashboard - for example [Go Processes](https://grafana.com/grafana/dashboards/6671).

pkg/api/http_server.go

@@ -497,6 +497,7 @@ func (hs *HTTPServer) addMiddlewaresAndStaticRoutes() {
 	m.Use(hs.healthzHandler)
 	m.Use(hs.apiHealthHandler)
 	m.Use(hs.metricsEndpoint)
+	m.Use(hs.pluginMetricsEndpoint)
 
 	m.Use(hs.ContextHandler.Middleware)
 	m.Use(middleware.OrgRedirect(hs.Cfg))

pkg/api/http_server_test.go

@@ -3,9 +3,8 @@ package api
 import (
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestHTTPServer_MetricsBasicAuth(t *testing.T) {

pkg/api/plugin_metrics.go

@@ -0,0 +1,45 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/plugins/backendplugin"
+	"github.com/grafana/grafana/pkg/web"
+)
+
+func (hs *HTTPServer) pluginMetricsEndpoint(ctx *web.Context) {
+	if !hs.Cfg.MetricsEndpointEnabled {
+		return
+	}
+
+	if ctx.Req.Method != http.MethodGet || !strings.HasPrefix(ctx.Req.URL.Path, "/metrics/plugins/") {
+		return
+	}
+
+	if hs.metricsEndpointBasicAuthEnabled() && !BasicAuthenticatedRequest(ctx.Req, hs.Cfg.MetricsEndpointBasicAuthUsername, hs.Cfg.MetricsEndpointBasicAuthPassword) {
+		ctx.Resp.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+
+	pathParts := strings.SplitAfter(ctx.Req.URL.Path, "/")
+	pluginID := pathParts[len(pathParts)-1]
+
+	resp, err := hs.pluginClient.CollectMetrics(ctx.Req.Context(), &backend.CollectMetricsRequest{PluginContext: backend.PluginContext{PluginID: pluginID}})
+	if err != nil {
+		if errors.Is(err, backendplugin.ErrPluginNotRegistered) {
+			ctx.Resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		ctx.Resp.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Type", "text/plain")
+	if _, err := ctx.Resp.Write(resp.PrometheusMetrics); err != nil {
+		hs.log.Error("Failed to write to response", "err", err)
+	}
+}

pkg/api/plugin_metrics_test.go

@@ -0,0 +1,160 @@
+package api
+
+import (
+	"context"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/backendplugin"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/web/webtest"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPluginMetricsEndpoint(t *testing.T) {
+	t.Run("Endpoint is enabled, basic auth disabled", func(t *testing.T) {
+		hs := &HTTPServer{
+			Cfg: &setting.Cfg{
+				MetricsEndpointEnabled:           true,
+				MetricsEndpointBasicAuthUsername: "",
+				MetricsEndpointBasicAuthPassword: "",
+			},
+			pluginClient: &fakePluginClientMetrics{
+				store: map[string][]byte{
+					"test-plugin": []byte("http_errors=2"),
+				},
+			},
+		}
+
+		s := webtest.NewServer(t, routing.NewRouteRegister())
+		s.Mux.Use(hs.pluginMetricsEndpoint)
+
+		t.Run("Endpoint matches and plugin is registered", func(t *testing.T) {
+			req := s.NewGetRequest("/metrics/plugins/test-plugin")
+			resp, err := s.Send(req)
+			require.NoError(t, err)
+			require.NotNil(t, resp)
+
+			body, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, "http_errors=2", string(body))
+			require.NoError(t, resp.Body.Close())
+			require.Equal(t, http.StatusOK, resp.StatusCode)
+			require.Equal(t, "text/plain", resp.Header.Get("Content-Type"))
+		})
+
+		t.Run("Endpoint matches and plugin is not registered", func(t *testing.T) {
+			req := s.NewGetRequest("/metrics/plugins/plugin-not-registered")
+			resp, err := s.Send(req)
+			require.NoError(t, err)
+			require.NotNil(t, resp)
+
+			body, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Empty(t, string(body))
+			require.NoError(t, resp.Body.Close())
+			require.Equal(t, http.StatusNotFound, resp.StatusCode)
+		})
+
+		t.Run("Endpoint does not match", func(t *testing.T) {
+			req := s.NewGetRequest("/foo")
+			resp, err := s.Send(req)
+			require.NoError(t, err)
+			require.NotNil(t, resp)
+			require.NoError(t, resp.Body.Close())
+			require.Equal(t, http.StatusNotFound, resp.StatusCode)
+		})
+	})
+
+	t.Run("Endpoint and basic auth is enabled", func(t *testing.T) {
+		hs := &HTTPServer{
+			Cfg: &setting.Cfg{
+				MetricsEndpointEnabled:           true,
+				MetricsEndpointBasicAuthUsername: "user",
+				MetricsEndpointBasicAuthPassword: "pwd",
+			},
+			pluginClient: &fakePluginClientMetrics{
+				store: map[string][]byte{
+					"test-plugin": []byte("http_errors=2"),
+				},
+			},
+		}
+
+		s := webtest.NewServer(t, routing.NewRouteRegister())
+		s.Mux.Use(hs.pluginMetricsEndpoint)
+
+		t.Run("When plugin is registered, wrong basic auth credentials should return 401", func(t *testing.T) {
+			req := s.NewGetRequest("/metrics/plugins/test-plugin")
+			req.SetBasicAuth("user2", "pwd2")
+			resp, err := s.Send(req)
+			require.NoError(t, err)
+			require.NotNil(t, resp)
+
+			require.NoError(t, err)
+			require.NoError(t, resp.Body.Close())
+			require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+		})
+
+		t.Run("When plugin is registered, correct basic auth credentials should return 200", func(t *testing.T) {
+			req := s.NewGetRequest("/metrics/plugins/test-plugin")
+			req.SetBasicAuth("user", "pwd")
+			resp, err := s.Send(req)
+			require.NoError(t, err)
+			require.NotNil(t, resp)
+
+			body, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, "http_errors=2", string(body))
+			require.NoError(t, resp.Body.Close())
+			require.Equal(t, http.StatusOK, resp.StatusCode)
+			require.Equal(t, "text/plain", resp.Header.Get("Content-Type"))
+		})
+	})
+
+	t.Run("Endpoint is disabled", func(t *testing.T) {
+		hs := &HTTPServer{
+			Cfg: &setting.Cfg{
+				MetricsEndpointEnabled: false,
+			},
+			pluginClient: &fakePluginClientMetrics{
+				store: map[string][]byte{
+					"test-plugin": []byte("http_errors=2"),
+				},
+			},
+		}
+
+		s := webtest.NewServer(t, routing.NewRouteRegister())
+		s.Mux.Use(hs.pluginMetricsEndpoint)
+
+		t.Run("When plugin is registered, should return 404", func(t *testing.T) {
+			req := s.NewGetRequest("/metrics/plugins/test-plugin")
+			resp, err := s.Send(req)
+			require.NoError(t, err)
+			require.NotNil(t, resp)
+			require.NoError(t, resp.Body.Close())
+			require.Equal(t, http.StatusNotFound, resp.StatusCode)
+		})
+	})
+}
+
+type fakePluginClientMetrics struct {
+	plugins.Client
+
+	store map[string][]byte
+}
+
+func (c *fakePluginClientMetrics) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) {
+	metrics, exists := c.store[req.PluginContext.PluginID]
+
+	if !exists {
+		return nil, backendplugin.ErrPluginNotRegistered
+	}
+
+	return &backend.CollectMetricsResult{
+		PrometheusMetrics: metrics,
+	}, nil
+}