From 9eb2cd537d330c21ac0dcd2e5fe2dd728361b1a8 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 29 Mar 2022 10:18:26 +0100 Subject: [PATCH] Plugins: Make backend plugin metrics endpoints available with optional authentication (#46467) * add new endpoint without auth+config * add cfg check * fit lint issue * Add basic auth support Co-authored-by: Will Browne * WIP docs * Update docs/sources/administration/view-server/internal-metrics.md Co-authored-by: Dave Henderson * update instructions Co-authored-by: Will Browne Co-authored-by: Will Browne Co-authored-by: Dave Henderson --- conf/defaults.ini | 4 +- conf/sample.ini | 4 +- .../docker/blocks/prometheus/prometheus.yml | 5 + .../view-server/internal-metrics.md | 49 +++++- pkg/api/http_server.go | 1 + pkg/api/http_server_test.go | 3 +- pkg/api/plugin_metrics.go | 45 +++++ pkg/api/plugin_metrics_test.go | 160 ++++++++++++++++++ 8 files changed, 263 insertions(+), 8 deletions(-) create mode 100644 pkg/api/plugin_metrics.go create mode 100644 pkg/api/plugin_metrics_test.go diff --git a/conf/defaults.ini b/conf/defaults.ini index 44f751d7234..8504f1b8735 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -896,14 +896,14 @@ enabled = true enabled = false #################################### Internal Grafana Metrics ############ -# Metrics available at HTTP API Url /metrics +# Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId [metrics] enabled = true interval_seconds = 10 # Disable total stats (stat_totals_*) metrics to be generated disable_total_stats = false -#If both are set, basic auth will be required for the metrics endpoint. +#If both are set, basic auth will be required for the metrics endpoints. basic_auth_username = basic_auth_password = diff --git a/conf/sample.ini b/conf/sample.ini index 55b3ceed354..b83d6ed8824 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -878,7 +878,7 @@ ;enabled = false #################################### Internal Grafana Metrics ########################## -# Metrics available at HTTP API Url /metrics +# Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId [metrics] # Disable / Enable internal metrics ;enabled = true @@ -887,7 +887,7 @@ # Disable total stats (stat_totals_*) metrics to be generated ;disable_total_stats = false -#If both are set, basic auth will be required for the metrics endpoint. +#If both are set, basic auth will be required for the metrics endpoints. ; basic_auth_username = ; basic_auth_password = diff --git a/devenv/docker/blocks/prometheus/prometheus.yml b/devenv/docker/blocks/prometheus/prometheus.yml index 5fe6e9ac988..691767247c3 100644 --- a/devenv/docker/blocks/prometheus/prometheus.yml +++ b/devenv/docker/blocks/prometheus/prometheus.yml @@ -37,3 +37,8 @@ scrape_configs: - job_name: 'prometheus-random-data' static_configs: - targets: ['prometheus-random-data:8080'] + + # - job_name: 'grafana-test-datasource' + # metrics_path: /metrics/plugins/grafana-test-datasource + # static_configs: + # - targets: ['host.docker.internal:3000'] diff --git a/docs/sources/administration/view-server/internal-metrics.md b/docs/sources/administration/view-server/internal-metrics.md index ed73c45492d..0c129c296dc 100644 --- a/docs/sources/administration/view-server/internal-metrics.md +++ b/docs/sources/administration/view-server/internal-metrics.md @@ -30,7 +30,7 @@ These instructions assume you have already added Prometheus as a data source in 1. Enable Prometheus to scrape metrics from Grafana. In your configuration file (`grafana.ini` or `custom.ini` depending on your operating system) remove the semicolon to enable the following configuration options: ``` - # Metrics available at HTTP API Url /metrics + # Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId [metrics] # Disable / Enable internal metrics enabled = true @@ -39,7 +39,7 @@ These instructions assume you have already added Prometheus as a data source in disable_total_stats = false ``` -1. (optional) If you want to require authorization to view the metrics endpoint, then uncomment and set the following options: +1. (optional) If you want to require authorization to view the metrics endpoints, then uncomment and set the following options: ``` basic_auth_username = @@ -92,3 +92,48 @@ These instructions assume you have already added Graphite as a data source in Gr ``` 1. Restart Grafana. Grafana now exposes metrics at http://localhost:3000/metrics and sends them to the Graphite location you specified. + +## Pull metrics from Grafana backend plugin into Prometheus + +Any installed [backend plugin]({{< relref "../../developers/plugins/backend/_index.md" >}}) exposes a metrics endpoint through Grafana that you can configure Prometheus to scrape. + +These instructions assume you have already added Prometheus as a data source in Grafana. + +1. Enable Prometheus to scrape backend plugin metrics from Grafana. In your configuration file (`grafana.ini` or `custom.ini` depending on your operating system) remove the semicolon to enable the following configuration options: + + ``` + # Metrics available at HTTP URL /metrics and /metrics/plugins/:pluginId + [metrics] + # Disable / Enable internal metrics + enabled = true + + # Disable total stats (stat_totals_*) metrics to be generated + disable_total_stats = false + ``` + +1. (optional) If you want to require authorization to view the metrics endpoints, then uncomment and set the following options: + + ``` + basic_auth_username = + basic_auth_password = + ``` + +1. Restart Grafana. Grafana now exposes metrics at `http://localhost:3000/metrics/plugins/`, e.g. http://localhost:3000/metrics/plugins/grafana-github-datasource if you have the [Grafana GitHub datasource](https://grafana.com/grafana/plugins/grafana-github-datasource/) installed. +1. Add the job to your prometheus.yml file. + Example: + + ``` + - job_name: 'grafana_github_datasource' + + scrape_interval: 15s + scrape_timeout: 5s + metrics_path: /metrics/plugins/grafana-test-datasource + + static_configs: + - targets: ['localhost:3000'] + ``` + +1. Restart Prometheus. Your new job should appear on the Targets tab. +1. In Grafana, hover your mouse over the **Configuration** (gear) icon on the left sidebar and then click **Data Sources**. +1. Select the **Prometheus** data source. +1. Import a Golang application metrics dashboard - for example [Go Processes](https://grafana.com/grafana/dashboards/6671). diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index cac6ba7fcb9..e508bcd15b9 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -497,6 +497,7 @@ func (hs *HTTPServer) addMiddlewaresAndStaticRoutes() { m.Use(hs.healthzHandler) m.Use(hs.apiHealthHandler) m.Use(hs.metricsEndpoint) + m.Use(hs.pluginMetricsEndpoint) m.Use(hs.ContextHandler.Middleware) m.Use(middleware.OrgRedirect(hs.Cfg)) diff --git a/pkg/api/http_server_test.go b/pkg/api/http_server_test.go index 5cb99004bef..c0d45258855 100644 --- a/pkg/api/http_server_test.go +++ b/pkg/api/http_server_test.go @@ -3,9 +3,8 @@ package api import ( "testing" - "github.com/stretchr/testify/assert" - "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/assert" ) func TestHTTPServer_MetricsBasicAuth(t *testing.T) { diff --git a/pkg/api/plugin_metrics.go b/pkg/api/plugin_metrics.go new file mode 100644 index 00000000000..85379d05d52 --- /dev/null +++ b/pkg/api/plugin_metrics.go @@ -0,0 +1,45 @@ +package api + +import ( + "errors" + "net/http" + "strings" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/plugins/backendplugin" + "github.com/grafana/grafana/pkg/web" +) + +func (hs *HTTPServer) pluginMetricsEndpoint(ctx *web.Context) { + if !hs.Cfg.MetricsEndpointEnabled { + return + } + + if ctx.Req.Method != http.MethodGet || !strings.HasPrefix(ctx.Req.URL.Path, "/metrics/plugins/") { + return + } + + if hs.metricsEndpointBasicAuthEnabled() && !BasicAuthenticatedRequest(ctx.Req, hs.Cfg.MetricsEndpointBasicAuthUsername, hs.Cfg.MetricsEndpointBasicAuthPassword) { + ctx.Resp.WriteHeader(http.StatusUnauthorized) + return + } + + pathParts := strings.SplitAfter(ctx.Req.URL.Path, "/") + pluginID := pathParts[len(pathParts)-1] + + resp, err := hs.pluginClient.CollectMetrics(ctx.Req.Context(), &backend.CollectMetricsRequest{PluginContext: backend.PluginContext{PluginID: pluginID}}) + if err != nil { + if errors.Is(err, backendplugin.ErrPluginNotRegistered) { + ctx.Resp.WriteHeader(http.StatusNotFound) + return + } + + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + + ctx.Resp.Header().Set("Content-Type", "text/plain") + if _, err := ctx.Resp.Write(resp.PrometheusMetrics); err != nil { + hs.log.Error("Failed to write to response", "err", err) + } +} diff --git a/pkg/api/plugin_metrics_test.go b/pkg/api/plugin_metrics_test.go new file mode 100644 index 00000000000..935434b00ad --- /dev/null +++ b/pkg/api/plugin_metrics_test.go @@ -0,0 +1,160 @@ +package api + +import ( + "context" + "io/ioutil" + "net/http" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/backendplugin" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/web/webtest" + "github.com/stretchr/testify/require" +) + +func TestPluginMetricsEndpoint(t *testing.T) { + t.Run("Endpoint is enabled, basic auth disabled", func(t *testing.T) { + hs := &HTTPServer{ + Cfg: &setting.Cfg{ + MetricsEndpointEnabled: true, + MetricsEndpointBasicAuthUsername: "", + MetricsEndpointBasicAuthPassword: "", + }, + pluginClient: &fakePluginClientMetrics{ + store: map[string][]byte{ + "test-plugin": []byte("http_errors=2"), + }, + }, + } + + s := webtest.NewServer(t, routing.NewRouteRegister()) + s.Mux.Use(hs.pluginMetricsEndpoint) + + t.Run("Endpoint matches and plugin is registered", func(t *testing.T) { + req := s.NewGetRequest("/metrics/plugins/test-plugin") + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "http_errors=2", string(body)) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "text/plain", resp.Header.Get("Content-Type")) + }) + + t.Run("Endpoint matches and plugin is not registered", func(t *testing.T) { + req := s.NewGetRequest("/metrics/plugins/plugin-not-registered") + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Empty(t, string(body)) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("Endpoint does not match", func(t *testing.T) { + req := s.NewGetRequest("/foo") + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + }) + + t.Run("Endpoint and basic auth is enabled", func(t *testing.T) { + hs := &HTTPServer{ + Cfg: &setting.Cfg{ + MetricsEndpointEnabled: true, + MetricsEndpointBasicAuthUsername: "user", + MetricsEndpointBasicAuthPassword: "pwd", + }, + pluginClient: &fakePluginClientMetrics{ + store: map[string][]byte{ + "test-plugin": []byte("http_errors=2"), + }, + }, + } + + s := webtest.NewServer(t, routing.NewRouteRegister()) + s.Mux.Use(hs.pluginMetricsEndpoint) + + t.Run("When plugin is registered, wrong basic auth credentials should return 401", func(t *testing.T) { + req := s.NewGetRequest("/metrics/plugins/test-plugin") + req.SetBasicAuth("user2", "pwd2") + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + }) + + t.Run("When plugin is registered, correct basic auth credentials should return 200", func(t *testing.T) { + req := s.NewGetRequest("/metrics/plugins/test-plugin") + req.SetBasicAuth("user", "pwd") + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "http_errors=2", string(body)) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "text/plain", resp.Header.Get("Content-Type")) + }) + }) + + t.Run("Endpoint is disabled", func(t *testing.T) { + hs := &HTTPServer{ + Cfg: &setting.Cfg{ + MetricsEndpointEnabled: false, + }, + pluginClient: &fakePluginClientMetrics{ + store: map[string][]byte{ + "test-plugin": []byte("http_errors=2"), + }, + }, + } + + s := webtest.NewServer(t, routing.NewRouteRegister()) + s.Mux.Use(hs.pluginMetricsEndpoint) + + t.Run("When plugin is registered, should return 404", func(t *testing.T) { + req := s.NewGetRequest("/metrics/plugins/test-plugin") + resp, err := s.Send(req) + require.NoError(t, err) + require.NotNil(t, resp) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + }) +} + +type fakePluginClientMetrics struct { + plugins.Client + + store map[string][]byte +} + +func (c *fakePluginClientMetrics) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) { + metrics, exists := c.store[req.PluginContext.PluginID] + + if !exists { + return nil, backendplugin.ErrPluginNotRegistered + } + + return &backend.CollectMetricsResult{ + PrometheusMetrics: metrics, + }, nil +}