From 13e67660f575b8436f2687f8a98cbb42acb1e723 Mon Sep 17 00:00:00 2001 From: Emil Hessman <248952+ceh@users.noreply.github.com> Date: Wed, 21 Oct 2020 11:06:19 +0200 Subject: [PATCH] API: Query database from /api/health endpoint (#28349) --- pkg/api/health.go | 21 +++ pkg/api/health_test.go | 190 +++++++++++++++++++++++++++ pkg/api/http_server.go | 6 +- pkg/services/sqlstore/health.go | 3 +- pkg/services/sqlstore/health_test.go | 18 +++ 5 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 pkg/api/health.go create mode 100644 pkg/api/health_test.go create mode 100644 pkg/services/sqlstore/health_test.go diff --git a/pkg/api/health.go b/pkg/api/health.go new file mode 100644 index 00000000000..c923f2a8f8c --- /dev/null +++ b/pkg/api/health.go @@ -0,0 +1,21 @@ +package api + +import ( + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" +) + +func (hs *HTTPServer) databaseHealthy() bool { + const cacheKey = "db-healthy" + + if cached, found := hs.CacheService.Get(cacheKey); found { + return cached.(bool) + } + + healthy := bus.Dispatch(&models.GetDBHealthQuery{}) == nil + + hs.CacheService.Set(cacheKey, healthy, time.Second*5) + return healthy +} diff --git a/pkg/api/health_test.go b/pkg/api/health_test.go new file mode 100644 index 00000000000..0c937dbdf81 --- /dev/null +++ b/pkg/api/health_test.go @@ -0,0 +1,190 @@ +package api + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/localcache" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/require" + macaron "gopkg.in/macaron.v1" +) + +func TestHealthAPI_Version(t *testing.T) { + m, _ := setupHealthAPITestEnvironment(t) + setting.BuildVersion = "7.4.0" + setting.BuildCommit = "59906ab1bf" + + bus.AddHandler("test", func(query *models.GetDBHealthQuery) error { + return nil + }) + + req := httptest.NewRequest(http.MethodGet, "/api/health", nil) + rec := httptest.NewRecorder() + m.ServeHTTP(rec, req) + + require.Equal(t, 200, rec.Code) + expectedBody := ` + { + "database": "ok", + "version": "7.4.0", + "commit": "59906ab1bf" + } + ` + require.JSONEq(t, expectedBody, rec.Body.String()) +} + +func TestHealthAPI_AnonymousHideVersion(t *testing.T) { + m, hs := setupHealthAPITestEnvironment(t) + hs.Cfg.AnonymousHideVersion = true + + bus.AddHandler("test", func(query *models.GetDBHealthQuery) error { + return nil + }) + + req := httptest.NewRequest(http.MethodGet, "/api/health", nil) + rec := httptest.NewRecorder() + m.ServeHTTP(rec, req) + + require.Equal(t, 200, rec.Code) + expectedBody := ` + { + "database": "ok" + } + ` + require.JSONEq(t, expectedBody, rec.Body.String()) +} + +func TestHealthAPI_DatabaseHealthy(t *testing.T) { + const cacheKey = "db-healthy" + + m, hs := setupHealthAPITestEnvironment(t) + hs.Cfg.AnonymousHideVersion = true + + bus.AddHandler("test", func(query *models.GetDBHealthQuery) error { + return nil + }) + + healthy, found := hs.CacheService.Get(cacheKey) + require.False(t, found) + require.Nil(t, healthy) + + req := httptest.NewRequest(http.MethodGet, "/api/health", nil) + rec := httptest.NewRecorder() + m.ServeHTTP(rec, req) + + require.Equal(t, 200, rec.Code) + expectedBody := ` + { + "database": "ok" + } + ` + require.JSONEq(t, expectedBody, rec.Body.String()) + + healthy, found = hs.CacheService.Get(cacheKey) + require.True(t, found) + require.True(t, healthy.(bool)) +} + +func TestHealthAPI_DatabaseUnhealthy(t *testing.T) { + const cacheKey = "db-healthy" + + m, hs := setupHealthAPITestEnvironment(t) + hs.Cfg.AnonymousHideVersion = true + + bus.AddHandler("test", func(query *models.GetDBHealthQuery) error { + return errors.New("bad") + }) + + healthy, found := hs.CacheService.Get(cacheKey) + require.False(t, found) + require.Nil(t, healthy) + + req := httptest.NewRequest(http.MethodGet, "/api/health", nil) + rec := httptest.NewRecorder() + m.ServeHTTP(rec, req) + + require.Equal(t, 503, rec.Code) + expectedBody := ` + { + "database": "failing" + } + ` + require.JSONEq(t, expectedBody, rec.Body.String()) + + healthy, found = hs.CacheService.Get(cacheKey) + require.True(t, found) + require.False(t, healthy.(bool)) +} + +func TestHealthAPI_DatabaseHealthCached(t *testing.T) { + const cacheKey = "db-healthy" + + m, hs := setupHealthAPITestEnvironment(t) + hs.Cfg.AnonymousHideVersion = true + + // Database is healthy. + bus.AddHandler("test", func(query *models.GetDBHealthQuery) error { + return nil + }) + + // Mock unhealthy database in cache. + hs.CacheService.Set(cacheKey, false, 5*time.Minute) + + req := httptest.NewRequest(http.MethodGet, "/api/health", nil) + rec := httptest.NewRecorder() + m.ServeHTTP(rec, req) + + require.Equal(t, 503, rec.Code) + expectedBody := ` + { + "database": "failing" + } + ` + require.JSONEq(t, expectedBody, rec.Body.String()) + + // Purge cache and redo request. + hs.CacheService.Delete(cacheKey) + rec = httptest.NewRecorder() + m.ServeHTTP(rec, req) + + require.Equal(t, 200, rec.Code) + expectedBody = ` + { + "database": "ok" + } + ` + require.JSONEq(t, expectedBody, rec.Body.String()) + + healthy, found := hs.CacheService.Get(cacheKey) + require.True(t, found) + require.True(t, healthy.(bool)) +} + +func setupHealthAPITestEnvironment(t *testing.T) (*macaron.Macaron, *HTTPServer) { + t.Helper() + + oldVersion := setting.BuildVersion + oldCommit := setting.BuildCommit + t.Cleanup(func() { + setting.BuildVersion = oldVersion + setting.BuildCommit = oldCommit + }) + + bus.ClearBusHandlers() + t.Cleanup(bus.ClearBusHandlers) + + m := macaron.New() + hs := &HTTPServer{ + CacheService: localcache.New(5*time.Minute, 10*time.Minute), + Cfg: setting.NewCfg(), + } + + m.Get("/api/health", hs.apiHealthHandler) + return m, hs +} diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index bf996ff4bae..623feb63dfc 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -338,7 +338,7 @@ func (hs *HTTPServer) addMiddlewaresAndStaticRoutes() { })) // These endpoints are used for monitoring the Grafana instance - // and should not be redirect or rejected. + // and should not be redirected or rejected. m.Use(hs.healthzHandler) m.Use(hs.apiHealthHandler) m.Use(hs.metricsEndpoint) @@ -396,7 +396,7 @@ func (hs *HTTPServer) healthzHandler(ctx *macaron.Context) { } // apiHealthHandler will return ok if Grafana's web server is running and it -// can access the database. If the database cannot be access it will return +// can access the database. If the database cannot be accessed it will return // http status code 503. func (hs *HTTPServer) apiHealthHandler(ctx *macaron.Context) { notHeadOrGet := ctx.Req.Method != http.MethodGet && ctx.Req.Method != http.MethodHead @@ -411,7 +411,7 @@ func (hs *HTTPServer) apiHealthHandler(ctx *macaron.Context) { data.Set("commit", setting.BuildCommit) } - if err := bus.Dispatch(&models.GetDBHealthQuery{}); err != nil { + if !hs.databaseHealthy() { data.Set("database", "failing") ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8") ctx.Resp.WriteHeader(503) diff --git a/pkg/services/sqlstore/health.go b/pkg/services/sqlstore/health.go index 436a2a9fd75..791a68428e3 100644 --- a/pkg/services/sqlstore/health.go +++ b/pkg/services/sqlstore/health.go @@ -10,5 +10,6 @@ func init() { } func GetDBHealthQuery(query *models.GetDBHealthQuery) error { - return x.Ping() + _, err := x.Exec("SELECT 1") + return err } diff --git a/pkg/services/sqlstore/health_test.go b/pkg/services/sqlstore/health_test.go new file mode 100644 index 00000000000..a3b6129edcb --- /dev/null +++ b/pkg/services/sqlstore/health_test.go @@ -0,0 +1,18 @@ +// +build integration + +package sqlstore + +import ( + "testing" + + "github.com/grafana/grafana/pkg/models" + "github.com/stretchr/testify/require" +) + +func TestGetDBHealthQuery(t *testing.T) { + InitTestDB(t) + + query := models.GetDBHealthQuery{} + err := GetDBHealthQuery(&query) + require.NoError(t, err) +}