From 7cd3cd6cd43cb0ef2597ce032935e4b2f38904db Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Sat, 2 Feb 2019 12:11:30 +0100 Subject: [PATCH 01/19] auth package refactoring moving middleware/hooks away from package exposing public struct UserToken accessible from other packages fix debug log lines so the same order and naming are used --- pkg/services/auth/auth.go | 6 + pkg/services/auth/auth_token.go | 279 ------------- pkg/services/auth/auth_token_test.go | 378 ----------------- pkg/services/auth/authtoken/auth_token.go | 225 ++++++++++ .../auth/authtoken/auth_token_test.go | 386 ++++++++++++++++++ pkg/services/auth/authtoken/model.go | 76 ++++ .../auth/{ => authtoken}/session_cleanup.go | 2 +- .../{ => authtoken}/session_cleanup_test.go | 2 +- pkg/services/auth/model.go | 25 -- 9 files changed, 695 insertions(+), 684 deletions(-) create mode 100644 pkg/services/auth/auth.go delete mode 100644 pkg/services/auth/auth_token.go delete mode 100644 pkg/services/auth/auth_token_test.go create mode 100644 pkg/services/auth/authtoken/auth_token.go create mode 100644 pkg/services/auth/authtoken/auth_token_test.go create mode 100644 pkg/services/auth/authtoken/model.go rename pkg/services/auth/{ => authtoken}/session_cleanup.go (98%) rename pkg/services/auth/{ => authtoken}/session_cleanup_test.go (98%) delete mode 100644 pkg/services/auth/model.go diff --git a/pkg/services/auth/auth.go b/pkg/services/auth/auth.go new file mode 100644 index 00000000000..31316f473f5 --- /dev/null +++ b/pkg/services/auth/auth.go @@ -0,0 +1,6 @@ +package auth + +type UserToken interface { + GetUserId() int64 + GetToken() string +} diff --git a/pkg/services/auth/auth_token.go b/pkg/services/auth/auth_token.go deleted file mode 100644 index 13b9ef607f5..00000000000 --- a/pkg/services/auth/auth_token.go +++ /dev/null @@ -1,279 +0,0 @@ -package auth - -import ( - "crypto/sha256" - "encoding/hex" - "errors" - "net/http" - "net/url" - "time" - - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/infra/serverlock" - "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/registry" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" -) - -func init() { - registry.RegisterService(&UserAuthTokenServiceImpl{}) -} - -var ( - getTime = time.Now - UrgentRotateTime = 1 * time.Minute - oneYearInSeconds = 31557600 //used as default maxage for session cookies. We validate/rotate them more often. -) - -// UserAuthTokenService are used for generating and validating user auth tokens -type UserAuthTokenService interface { - InitContextWithToken(ctx *models.ReqContext, orgID int64) bool - UserAuthenticatedHook(user *models.User, c *models.ReqContext) error - SignOutUser(c *models.ReqContext) error -} - -type UserAuthTokenServiceImpl struct { - SQLStore *sqlstore.SqlStore `inject:""` - ServerLockService *serverlock.ServerLockService `inject:""` - Cfg *setting.Cfg `inject:""` - log log.Logger -} - -// Init this service -func (s *UserAuthTokenServiceImpl) Init() error { - s.log = log.New("auth") - return nil -} - -func (s *UserAuthTokenServiceImpl) InitContextWithToken(ctx *models.ReqContext, orgID int64) bool { - //auth User - unhashedToken := ctx.GetCookie(s.Cfg.LoginCookieName) - if unhashedToken == "" { - return false - } - - userToken, err := s.LookupToken(unhashedToken) - if err != nil { - ctx.Logger.Info("failed to look up user based on cookie", "error", err) - return false - } - - query := models.GetSignedInUserQuery{UserId: userToken.UserId, OrgId: orgID} - if err := bus.Dispatch(&query); err != nil { - ctx.Logger.Error("Failed to get user with id", "userId", userToken.UserId, "error", err) - return false - } - - ctx.SignedInUser = query.Result - ctx.IsSignedIn = true - - //rotate session token if needed. - rotated, err := s.RefreshToken(userToken, ctx.RemoteAddr(), ctx.Req.UserAgent()) - if err != nil { - ctx.Logger.Error("failed to rotate token", "error", err, "userId", userToken.UserId, "tokenId", userToken.Id) - return true - } - - if rotated { - s.writeSessionCookie(ctx, userToken.UnhashedToken, oneYearInSeconds) - } - - return true -} - -func (s *UserAuthTokenServiceImpl) writeSessionCookie(ctx *models.ReqContext, value string, maxAge int) { - if setting.Env == setting.DEV { - ctx.Logger.Debug("new token", "unhashed token", value) - } - - ctx.Resp.Header().Del("Set-Cookie") - cookie := http.Cookie{ - Name: s.Cfg.LoginCookieName, - Value: url.QueryEscape(value), - HttpOnly: true, - Path: setting.AppSubUrl + "/", - Secure: s.Cfg.SecurityHTTPSCookies, - MaxAge: maxAge, - SameSite: s.Cfg.LoginCookieSameSite, - } - - http.SetCookie(ctx.Resp, &cookie) -} - -func (s *UserAuthTokenServiceImpl) UserAuthenticatedHook(user *models.User, c *models.ReqContext) error { - userToken, err := s.CreateToken(user.Id, c.RemoteAddr(), c.Req.UserAgent()) - if err != nil { - return err - } - - s.writeSessionCookie(c, userToken.UnhashedToken, oneYearInSeconds) - return nil -} - -func (s *UserAuthTokenServiceImpl) SignOutUser(c *models.ReqContext) error { - unhashedToken := c.GetCookie(s.Cfg.LoginCookieName) - if unhashedToken == "" { - return errors.New("cannot logout without session token") - } - - hashedToken := hashToken(unhashedToken) - - sql := `DELETE FROM user_auth_token WHERE auth_token = ?` - _, err := s.SQLStore.NewSession().Exec(sql, hashedToken) - - s.writeSessionCookie(c, "", -1) - return err -} - -func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent string) (*userAuthToken, error) { - clientIP = util.ParseIPAddress(clientIP) - token, err := util.RandomHex(16) - if err != nil { - return nil, err - } - - hashedToken := hashToken(token) - - now := getTime().Unix() - - userToken := userAuthToken{ - UserId: userId, - AuthToken: hashedToken, - PrevAuthToken: hashedToken, - ClientIp: clientIP, - UserAgent: userAgent, - RotatedAt: now, - CreatedAt: now, - UpdatedAt: now, - SeenAt: 0, - AuthTokenSeen: false, - } - _, err = s.SQLStore.NewSession().Insert(&userToken) - if err != nil { - return nil, err - } - - userToken.UnhashedToken = token - - return &userToken, nil -} - -func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*userAuthToken, error) { - hashedToken := hashToken(unhashedToken) - if setting.Env == setting.DEV { - s.log.Debug("looking up token", "unhashed", unhashedToken, "hashed", hashedToken) - } - - expireBefore := getTime().Add(time.Duration(-86400*s.Cfg.LoginCookieMaxDays) * time.Second).Unix() - - var userToken userAuthToken - exists, err := s.SQLStore.NewSession().Where("(auth_token = ? OR prev_auth_token = ?) AND created_at > ?", hashedToken, hashedToken, expireBefore).Get(&userToken) - if err != nil { - return nil, err - } - - if !exists { - return nil, ErrAuthTokenNotFound - } - - if userToken.AuthToken != hashedToken && userToken.PrevAuthToken == hashedToken && userToken.AuthTokenSeen { - userTokenCopy := userToken - userTokenCopy.AuthTokenSeen = false - expireBefore := getTime().Add(-UrgentRotateTime).Unix() - affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND prev_auth_token = ? AND rotated_at < ?", userTokenCopy.Id, userTokenCopy.PrevAuthToken, expireBefore).AllCols().Update(&userTokenCopy) - if err != nil { - return nil, err - } - - if affectedRows == 0 { - s.log.Debug("prev seen token unchanged", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent) - } else { - s.log.Debug("prev seen token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent) - } - } - - if !userToken.AuthTokenSeen && userToken.AuthToken == hashedToken { - userTokenCopy := userToken - userTokenCopy.AuthTokenSeen = true - userTokenCopy.SeenAt = getTime().Unix() - affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND auth_token = ?", userTokenCopy.Id, userTokenCopy.AuthToken).AllCols().Update(&userTokenCopy) - if err != nil { - return nil, err - } - - if affectedRows == 1 { - userToken = userTokenCopy - } - - if affectedRows == 0 { - s.log.Debug("seen wrong token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent) - } else { - s.log.Debug("seen token", "userTokenId", userToken.Id, "userId", userToken.UserId, "authToken", userToken.AuthToken, "clientIP", userToken.ClientIp, "userAgent", userToken.UserAgent) - } - } - - userToken.UnhashedToken = unhashedToken - - return &userToken, nil -} - -func (s *UserAuthTokenServiceImpl) RefreshToken(token *userAuthToken, clientIP, userAgent string) (bool, error) { - if token == nil { - return false, nil - } - - now := getTime() - - needsRotation := false - rotatedAt := time.Unix(token.RotatedAt, 0) - if token.AuthTokenSeen { - needsRotation = rotatedAt.Before(now.Add(-time.Duration(s.Cfg.LoginCookieRotation) * time.Minute)) - } else { - needsRotation = rotatedAt.Before(now.Add(-UrgentRotateTime)) - } - - if !needsRotation { - return false, nil - } - - s.log.Debug("refresh token needs rotation?", "auth_token_seen", token.AuthTokenSeen, "rotated_at", rotatedAt, "token.Id", token.Id) - - clientIP = util.ParseIPAddress(clientIP) - newToken, _ := util.RandomHex(16) - hashedToken := hashToken(newToken) - - // very important that auth_token_seen is set after the prev_auth_token = case when ... for mysql to function correctly - sql := ` - UPDATE user_auth_token - SET - seen_at = 0, - user_agent = ?, - client_ip = ?, - prev_auth_token = case when auth_token_seen = ? then auth_token else prev_auth_token end, - auth_token = ?, - auth_token_seen = ?, - rotated_at = ? - WHERE id = ? AND (auth_token_seen = ? OR rotated_at < ?)` - - res, err := s.SQLStore.NewSession().Exec(sql, userAgent, clientIP, s.SQLStore.Dialect.BooleanStr(true), hashedToken, s.SQLStore.Dialect.BooleanStr(false), now.Unix(), token.Id, s.SQLStore.Dialect.BooleanStr(true), now.Add(-30*time.Second).Unix()) - if err != nil { - return false, err - } - - affected, _ := res.RowsAffected() - s.log.Debug("rotated", "affected", affected, "auth_token_id", token.Id, "userId", token.UserId) - if affected > 0 { - token.UnhashedToken = newToken - return true, nil - } - - return false, nil -} - -func hashToken(token string) string { - hashBytes := sha256.Sum256([]byte(token + setting.SecretKey)) - return hex.EncodeToString(hashBytes[:]) -} diff --git a/pkg/services/auth/auth_token_test.go b/pkg/services/auth/auth_token_test.go deleted file mode 100644 index 312e53a3970..00000000000 --- a/pkg/services/auth/auth_token_test.go +++ /dev/null @@ -1,378 +0,0 @@ -package auth - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" - macaron "gopkg.in/macaron.v1" - - "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/services/sqlstore" - . "github.com/smartystreets/goconvey/convey" -) - -func TestUserAuthToken(t *testing.T) { - Convey("Test user auth token", t, func() { - ctx := createTestContext(t) - userAuthTokenService := ctx.tokenService - userID := int64(10) - - t := time.Date(2018, 12, 13, 13, 45, 0, 0, time.UTC) - getTime = func() time.Time { - return t - } - - Convey("When creating token", func() { - token, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") - So(err, ShouldBeNil) - So(token, ShouldNotBeNil) - So(token.AuthTokenSeen, ShouldBeFalse) - - Convey("When lookup unhashed token should return user auth token", func() { - LookupToken, err := userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - So(LookupToken, ShouldNotBeNil) - So(LookupToken.UserId, ShouldEqual, userID) - So(LookupToken.AuthTokenSeen, ShouldBeTrue) - - storedAuthToken, err := ctx.getAuthTokenByID(LookupToken.Id) - So(err, ShouldBeNil) - So(storedAuthToken, ShouldNotBeNil) - So(storedAuthToken.AuthTokenSeen, ShouldBeTrue) - }) - - Convey("When lookup hashed token should return user auth token not found error", func() { - LookupToken, err := userAuthTokenService.LookupToken(token.AuthToken) - So(err, ShouldEqual, ErrAuthTokenNotFound) - So(LookupToken, ShouldBeNil) - }) - - Convey("signing out should delete token and cookie if present", func() { - httpreq := &http.Request{Header: make(http.Header)} - httpreq.AddCookie(&http.Cookie{Name: userAuthTokenService.Cfg.LoginCookieName, Value: token.UnhashedToken}) - - ctx := &models.ReqContext{Context: &macaron.Context{ - Req: macaron.Request{Request: httpreq}, - Resp: macaron.NewResponseWriter("POST", httptest.NewRecorder()), - }, - Logger: log.New("fakelogger"), - } - - err = userAuthTokenService.SignOutUser(ctx) - So(err, ShouldBeNil) - - // makes sure we tell the browser to overwrite the cookie - cookieHeader := fmt.Sprintf("%s=; Path=/; Max-Age=0; HttpOnly", userAuthTokenService.Cfg.LoginCookieName) - So(ctx.Resp.Header().Get("Set-Cookie"), ShouldEqual, cookieHeader) - }) - - Convey("signing out an none existing session should return an error", func() { - httpreq := &http.Request{Header: make(http.Header)} - httpreq.AddCookie(&http.Cookie{Name: userAuthTokenService.Cfg.LoginCookieName, Value: ""}) - - ctx := &models.ReqContext{Context: &macaron.Context{ - Req: macaron.Request{Request: httpreq}, - Resp: macaron.NewResponseWriter("POST", httptest.NewRecorder()), - }, - Logger: log.New("fakelogger"), - } - - err = userAuthTokenService.SignOutUser(ctx) - So(err, ShouldNotBeNil) - }) - }) - - Convey("expires correctly", func() { - token, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") - So(err, ShouldBeNil) - So(token, ShouldNotBeNil) - - _, err = userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - - token, err = ctx.getAuthTokenByID(token.Id) - So(err, ShouldBeNil) - - getTime = func() time.Time { - return t.Add(time.Hour) - } - - refreshed, err := userAuthTokenService.RefreshToken(token, "192.168.10.11:1234", "some user agent") - So(err, ShouldBeNil) - So(refreshed, ShouldBeTrue) - - _, err = userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - - stillGood, err := userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - So(stillGood, ShouldNotBeNil) - - getTime = func() time.Time { - return t.Add(24 * 7 * time.Hour) - } - notGood, err := userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldEqual, ErrAuthTokenNotFound) - So(notGood, ShouldBeNil) - }) - - Convey("can properly rotate tokens", func() { - token, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") - So(err, ShouldBeNil) - So(token, ShouldNotBeNil) - - prevToken := token.AuthToken - unhashedPrev := token.UnhashedToken - - refreshed, err := userAuthTokenService.RefreshToken(token, "192.168.10.12:1234", "a new user agent") - So(err, ShouldBeNil) - So(refreshed, ShouldBeFalse) - - updated, err := ctx.markAuthTokenAsSeen(token.Id) - So(err, ShouldBeNil) - So(updated, ShouldBeTrue) - - token, err = ctx.getAuthTokenByID(token.Id) - So(err, ShouldBeNil) - - getTime = func() time.Time { - return t.Add(time.Hour) - } - - refreshed, err = userAuthTokenService.RefreshToken(token, "192.168.10.12:1234", "a new user agent") - So(err, ShouldBeNil) - So(refreshed, ShouldBeTrue) - - unhashedToken := token.UnhashedToken - - token, err = ctx.getAuthTokenByID(token.Id) - So(err, ShouldBeNil) - token.UnhashedToken = unhashedToken - - So(token.RotatedAt, ShouldEqual, getTime().Unix()) - So(token.ClientIp, ShouldEqual, "192.168.10.12") - So(token.UserAgent, ShouldEqual, "a new user agent") - So(token.AuthTokenSeen, ShouldBeFalse) - So(token.SeenAt, ShouldEqual, 0) - So(token.PrevAuthToken, ShouldEqual, prevToken) - - // ability to auth using an old token - - lookedUp, err := userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - So(lookedUp.AuthTokenSeen, ShouldBeTrue) - So(lookedUp.SeenAt, ShouldEqual, getTime().Unix()) - - lookedUp, err = userAuthTokenService.LookupToken(unhashedPrev) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - So(lookedUp.Id, ShouldEqual, token.Id) - So(lookedUp.AuthTokenSeen, ShouldBeTrue) - - getTime = func() time.Time { - return t.Add(time.Hour + (2 * time.Minute)) - } - - lookedUp, err = userAuthTokenService.LookupToken(unhashedPrev) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - So(lookedUp.AuthTokenSeen, ShouldBeTrue) - - lookedUp, err = ctx.getAuthTokenByID(lookedUp.Id) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - So(lookedUp.AuthTokenSeen, ShouldBeFalse) - - refreshed, err = userAuthTokenService.RefreshToken(token, "192.168.10.12:1234", "a new user agent") - So(err, ShouldBeNil) - So(refreshed, ShouldBeTrue) - - token, err = ctx.getAuthTokenByID(token.Id) - So(err, ShouldBeNil) - So(token, ShouldNotBeNil) - So(token.SeenAt, ShouldEqual, 0) - }) - - Convey("keeps prev token valid for 1 minute after it is confirmed", func() { - token, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") - So(err, ShouldBeNil) - So(token, ShouldNotBeNil) - - lookedUp, err := userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - - getTime = func() time.Time { - return t.Add(10 * time.Minute) - } - - prevToken := token.UnhashedToken - refreshed, err := userAuthTokenService.RefreshToken(token, "1.1.1.1", "firefox") - So(err, ShouldBeNil) - So(refreshed, ShouldBeTrue) - - getTime = func() time.Time { - return t.Add(20 * time.Minute) - } - - current, err := userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - So(current, ShouldNotBeNil) - - prev, err := userAuthTokenService.LookupToken(prevToken) - So(err, ShouldBeNil) - So(prev, ShouldNotBeNil) - }) - - Convey("will not mark token unseen when prev and current are the same", func() { - token, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") - So(err, ShouldBeNil) - So(token, ShouldNotBeNil) - - lookedUp, err := userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - - lookedUp, err = userAuthTokenService.LookupToken(token.UnhashedToken) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - - lookedUp, err = ctx.getAuthTokenByID(lookedUp.Id) - So(err, ShouldBeNil) - So(lookedUp, ShouldNotBeNil) - So(lookedUp.AuthTokenSeen, ShouldBeTrue) - }) - - Convey("Rotate token", func() { - token, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") - So(err, ShouldBeNil) - So(token, ShouldNotBeNil) - - prevToken := token.AuthToken - - Convey("Should rotate current token and previous token when auth token seen", func() { - updated, err := ctx.markAuthTokenAsSeen(token.Id) - So(err, ShouldBeNil) - So(updated, ShouldBeTrue) - - getTime = func() time.Time { - return t.Add(10 * time.Minute) - } - - refreshed, err := userAuthTokenService.RefreshToken(token, "1.1.1.1", "firefox") - So(err, ShouldBeNil) - So(refreshed, ShouldBeTrue) - - storedToken, err := ctx.getAuthTokenByID(token.Id) - So(err, ShouldBeNil) - So(storedToken, ShouldNotBeNil) - So(storedToken.AuthTokenSeen, ShouldBeFalse) - So(storedToken.PrevAuthToken, ShouldEqual, prevToken) - So(storedToken.AuthToken, ShouldNotEqual, prevToken) - - prevToken = storedToken.AuthToken - - updated, err = ctx.markAuthTokenAsSeen(token.Id) - So(err, ShouldBeNil) - So(updated, ShouldBeTrue) - - getTime = func() time.Time { - return t.Add(20 * time.Minute) - } - - refreshed, err = userAuthTokenService.RefreshToken(token, "1.1.1.1", "firefox") - So(err, ShouldBeNil) - So(refreshed, ShouldBeTrue) - - storedToken, err = ctx.getAuthTokenByID(token.Id) - So(err, ShouldBeNil) - So(storedToken, ShouldNotBeNil) - So(storedToken.AuthTokenSeen, ShouldBeFalse) - So(storedToken.PrevAuthToken, ShouldEqual, prevToken) - So(storedToken.AuthToken, ShouldNotEqual, prevToken) - }) - - Convey("Should rotate current token, but keep previous token when auth token not seen", func() { - token.RotatedAt = getTime().Add(-2 * time.Minute).Unix() - - getTime = func() time.Time { - return t.Add(2 * time.Minute) - } - - refreshed, err := userAuthTokenService.RefreshToken(token, "1.1.1.1", "firefox") - So(err, ShouldBeNil) - So(refreshed, ShouldBeTrue) - - storedToken, err := ctx.getAuthTokenByID(token.Id) - So(err, ShouldBeNil) - So(storedToken, ShouldNotBeNil) - So(storedToken.AuthTokenSeen, ShouldBeFalse) - So(storedToken.PrevAuthToken, ShouldEqual, prevToken) - So(storedToken.AuthToken, ShouldNotEqual, prevToken) - }) - }) - - Reset(func() { - getTime = time.Now - }) - }) -} - -func createTestContext(t *testing.T) *testContext { - t.Helper() - - sqlstore := sqlstore.InitTestDB(t) - tokenService := &UserAuthTokenServiceImpl{ - SQLStore: sqlstore, - Cfg: &setting.Cfg{ - LoginCookieName: "grafana_session", - LoginCookieMaxDays: 7, - LoginDeleteExpiredTokensAfterDays: 30, - LoginCookieRotation: 10, - }, - log: log.New("test-logger"), - } - - UrgentRotateTime = time.Minute - - return &testContext{ - sqlstore: sqlstore, - tokenService: tokenService, - } -} - -type testContext struct { - sqlstore *sqlstore.SqlStore - tokenService *UserAuthTokenServiceImpl -} - -func (c *testContext) getAuthTokenByID(id int64) (*userAuthToken, error) { - sess := c.sqlstore.NewSession() - var t userAuthToken - found, err := sess.ID(id).Get(&t) - if err != nil || !found { - return nil, err - } - - return &t, nil -} - -func (c *testContext) markAuthTokenAsSeen(id int64) (bool, error) { - sess := c.sqlstore.NewSession() - res, err := sess.Exec("UPDATE user_auth_token SET auth_token_seen = ? WHERE id = ?", c.sqlstore.Dialect.BooleanStr(true), id) - if err != nil { - return false, err - } - - rowsAffected, err := res.RowsAffected() - if err != nil { - return false, err - } - return rowsAffected == 1, nil -} diff --git a/pkg/services/auth/authtoken/auth_token.go b/pkg/services/auth/authtoken/auth_token.go new file mode 100644 index 00000000000..4e4bd375501 --- /dev/null +++ b/pkg/services/auth/authtoken/auth_token.go @@ -0,0 +1,225 @@ +package authtoken + +import ( + "crypto/sha256" + "encoding/hex" + "time" + + "github.com/grafana/grafana/pkg/services/auth" + + "github.com/grafana/grafana/pkg/infra/serverlock" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" +) + +func init() { + registry.Register(®istry.Descriptor{ + Name: "AuthTokenService", + Instance: &UserAuthTokenServiceImpl{}, + InitPriority: registry.Low, + }) +} + +var getTime = time.Now + +const urgentRotateTime = 1 * time.Minute + +type UserAuthTokenServiceImpl struct { + SQLStore *sqlstore.SqlStore `inject:""` + ServerLockService *serverlock.ServerLockService `inject:""` + Cfg *setting.Cfg `inject:""` + log log.Logger +} + +func (s *UserAuthTokenServiceImpl) Init() error { + s.log = log.New("auth") + return nil +} + +func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) { + clientIP = util.ParseIPAddress(clientIP) + token, err := util.RandomHex(16) + if err != nil { + return nil, err + } + + hashedToken := hashToken(token) + + now := getTime().Unix() + + userAuthToken := userAuthToken{ + UserId: userId, + AuthToken: hashedToken, + PrevAuthToken: hashedToken, + ClientIp: clientIP, + UserAgent: userAgent, + RotatedAt: now, + CreatedAt: now, + UpdatedAt: now, + SeenAt: 0, + AuthTokenSeen: false, + } + _, err = s.SQLStore.NewSession().Insert(&userAuthToken) + if err != nil { + return nil, err + } + + userAuthToken.UnhashedToken = token + + s.log.Debug("user auth token created", "tokenId", userAuthToken.Id, "userId", userAuthToken.UserId, "clientIP", userAuthToken.ClientIp, "userAgent", userAuthToken.UserAgent, "authToken", userAuthToken.AuthToken) + + return userAuthToken.toUserToken() +} + +func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (auth.UserToken, error) { + hashedToken := hashToken(unhashedToken) + if setting.Env == setting.DEV { + s.log.Debug("looking up token", "unhashed", unhashedToken, "hashed", hashedToken) + } + + expireBefore := getTime().Add(time.Duration(-86400*s.Cfg.LoginCookieMaxDays) * time.Second).Unix() + + var model userAuthToken + exists, err := s.SQLStore.NewSession().Where("(auth_token = ? OR prev_auth_token = ?) AND created_at > ?", hashedToken, hashedToken, expireBefore).Get(&model) + if err != nil { + return nil, err + } + + if !exists { + return nil, ErrAuthTokenNotFound + } + + if model.AuthToken != hashedToken && model.PrevAuthToken == hashedToken && model.AuthTokenSeen { + modelCopy := model + modelCopy.AuthTokenSeen = false + expireBefore := getTime().Add(-urgentRotateTime).Unix() + affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND prev_auth_token = ? AND rotated_at < ?", modelCopy.Id, modelCopy.PrevAuthToken, expireBefore).AllCols().Update(&modelCopy) + if err != nil { + return nil, err + } + + if affectedRows == 0 { + s.log.Debug("prev seen token unchanged", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent, "authToken", model.AuthToken) + } else { + s.log.Debug("prev seen token", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent, "authToken", model.AuthToken) + } + } + + if !model.AuthTokenSeen && model.AuthToken == hashedToken { + modelCopy := model + modelCopy.AuthTokenSeen = true + modelCopy.SeenAt = getTime().Unix() + affectedRows, err := s.SQLStore.NewSession().Where("id = ? AND auth_token = ?", modelCopy.Id, modelCopy.AuthToken).AllCols().Update(&modelCopy) + if err != nil { + return nil, err + } + + if affectedRows == 1 { + model = modelCopy + } + + if affectedRows == 0 { + s.log.Debug("seen wrong token", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent, "authToken", model.AuthToken) + } else { + s.log.Debug("seen token", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent, "authToken", model.AuthToken) + } + } + + model.UnhashedToken = unhashedToken + return model.toUserToken() +} + +func (s *UserAuthTokenServiceImpl) TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) { + if token == nil { + return false, nil + } + + model, err := extractModelFromToken(token) + if err != nil { + return false, err + } + + now := getTime() + + needsRotation := false + rotatedAt := time.Unix(model.RotatedAt, 0) + if model.AuthTokenSeen { + needsRotation = rotatedAt.Before(now.Add(-time.Duration(s.Cfg.LoginCookieRotation) * time.Minute)) + } else { + needsRotation = rotatedAt.Before(now.Add(-urgentRotateTime)) + } + + if !needsRotation { + return false, nil + } + + s.log.Debug("token needs rotation", "tokenId", model.Id, "authTokenSeen", model.AuthTokenSeen, "rotatedAt", rotatedAt) + + clientIP = util.ParseIPAddress(clientIP) + newToken, err := util.RandomHex(16) + if err != nil { + return false, err + } + hashedToken := hashToken(newToken) + + // very important that auth_token_seen is set after the prev_auth_token = case when ... for mysql to function correctly + sql := ` + UPDATE user_auth_token + SET + seen_at = 0, + user_agent = ?, + client_ip = ?, + prev_auth_token = case when auth_token_seen = ? then auth_token else prev_auth_token end, + auth_token = ?, + auth_token_seen = ?, + rotated_at = ? + WHERE id = ? AND (auth_token_seen = ? OR rotated_at < ?)` + + res, err := s.SQLStore.NewSession().Exec(sql, userAgent, clientIP, s.SQLStore.Dialect.BooleanStr(true), hashedToken, s.SQLStore.Dialect.BooleanStr(false), now.Unix(), model.Id, s.SQLStore.Dialect.BooleanStr(true), now.Add(-30*time.Second).Unix()) + if err != nil { + return false, err + } + + affected, _ := res.RowsAffected() + s.log.Debug("auth token rotated", "affected", affected, "auth_token_id", model.Id, "userId", model.UserId) + if affected > 0 { + model.UnhashedToken = newToken + return true, nil + } + + return false, nil +} + +func (s *UserAuthTokenServiceImpl) RevokeToken(token auth.UserToken) error { + if token == nil { + return ErrAuthTokenNotFound + } + + model, err := extractModelFromToken(token) + if err != nil { + return err + } + + rowsAffected, err := s.SQLStore.NewSession().Delete(model) + if err != nil { + return err + } + + if rowsAffected == 0 { + s.log.Debug("user auth token not found/revoked", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent) + return ErrAuthTokenNotFound + } + + s.log.Debug("user auth token revoked", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent) + + return nil +} + +func hashToken(token string) string { + hashBytes := sha256.Sum256([]byte(token + setting.SecretKey)) + return hex.EncodeToString(hashBytes[:]) +} diff --git a/pkg/services/auth/authtoken/auth_token_test.go b/pkg/services/auth/authtoken/auth_token_test.go new file mode 100644 index 00000000000..7809e235f5c --- /dev/null +++ b/pkg/services/auth/authtoken/auth_token_test.go @@ -0,0 +1,386 @@ +package authtoken + +import ( + "testing" + "time" + + "github.com/grafana/grafana/pkg/setting" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/sqlstore" + . "github.com/smartystreets/goconvey/convey" +) + +func TestUserAuthToken(t *testing.T) { + Convey("Test user auth token", t, func() { + ctx := createTestContext(t) + userAuthTokenService := ctx.tokenService + userID := int64(10) + + t := time.Date(2018, 12, 13, 13, 45, 0, 0, time.UTC) + getTime = func() time.Time { + return t + } + + Convey("When creating token", func() { + userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + model, err := extractModelFromToken(userToken) + So(err, ShouldBeNil) + So(model, ShouldNotBeNil) + So(model.AuthTokenSeen, ShouldBeFalse) + + Convey("When lookup unhashed token should return user auth token", func() { + userToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + lookedUpModel, err := extractModelFromToken(userToken) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + So(lookedUpModel.UserId, ShouldEqual, userID) + So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + + storedAuthToken, err := ctx.getAuthTokenByID(lookedUpModel.Id) + So(err, ShouldBeNil) + So(storedAuthToken, ShouldNotBeNil) + So(storedAuthToken.AuthTokenSeen, ShouldBeTrue) + }) + + Convey("When lookup hashed token should return user auth token not found error", func() { + userToken, err := userAuthTokenService.LookupToken(model.AuthToken) + So(err, ShouldEqual, ErrAuthTokenNotFound) + So(userToken, ShouldBeNil) + }) + + Convey("revoking existing token should delete token", func() { + err = userAuthTokenService.RevokeToken(userToken) + So(err, ShouldBeNil) + + model, err := ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + So(model, ShouldBeNil) + }) + + Convey("revoking nil token should return error", func() { + err = userAuthTokenService.RevokeToken(nil) + So(err, ShouldEqual, ErrAuthTokenNotFound) + }) + + Convey("revoking non-existing token should return error", func() { + model.Id = 1000 + nonExistingToken, err := model.toUserToken() + So(err, ShouldBeNil) + err = userAuthTokenService.RevokeToken(nonExistingToken) + So(err, ShouldEqual, ErrAuthTokenNotFound) + }) + }) + + Convey("expires correctly", func() { + userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + model, err := extractModelFromToken(userToken) + So(err, ShouldBeNil) + So(model, ShouldNotBeNil) + + _, err = userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + + model, err = ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + + userToken, err = model.toUserToken() + So(err, ShouldBeNil) + + getTime = func() time.Time { + return t.Add(time.Hour) + } + + rotated, err := userAuthTokenService.TryRotateToken(userToken, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + So(rotated, ShouldBeTrue) + + _, err = userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + + stillGood, err := userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + So(stillGood, ShouldNotBeNil) + + getTime = func() time.Time { + return t.Add(24 * 7 * time.Hour) + } + notGood, err := userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldEqual, ErrAuthTokenNotFound) + So(notGood, ShouldBeNil) + }) + + Convey("can properly rotate tokens", func() { + userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + model, err := extractModelFromToken(userToken) + So(err, ShouldBeNil) + So(model, ShouldNotBeNil) + + prevToken := model.AuthToken + unhashedPrev := model.UnhashedToken + + rotated, err := userAuthTokenService.TryRotateToken(userToken, "192.168.10.12:1234", "a new user agent") + So(err, ShouldBeNil) + So(rotated, ShouldBeFalse) + + updated, err := ctx.markAuthTokenAsSeen(model.Id) + So(err, ShouldBeNil) + So(updated, ShouldBeTrue) + + model, err = ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + tok, err := model.toUserToken() + So(err, ShouldBeNil) + + getTime = func() time.Time { + return t.Add(time.Hour) + } + + rotated, err = userAuthTokenService.TryRotateToken(tok, "192.168.10.12:1234", "a new user agent") + So(err, ShouldBeNil) + So(rotated, ShouldBeTrue) + + unhashedToken := model.UnhashedToken + + model, err = ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + model.UnhashedToken = unhashedToken + + So(model.RotatedAt, ShouldEqual, getTime().Unix()) + So(model.ClientIp, ShouldEqual, "192.168.10.12") + So(model.UserAgent, ShouldEqual, "a new user agent") + So(model.AuthTokenSeen, ShouldBeFalse) + So(model.SeenAt, ShouldEqual, 0) + So(model.PrevAuthToken, ShouldEqual, prevToken) + + // ability to auth using an old token + + lookedUpUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + lookedUpModel, err := extractModelFromToken(lookedUpUserToken) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + So(lookedUpModel.SeenAt, ShouldEqual, getTime().Unix()) + + lookedUpUserToken, err = userAuthTokenService.LookupToken(unhashedPrev) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + So(lookedUpModel.Id, ShouldEqual, model.Id) + So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + + getTime = func() time.Time { + return t.Add(time.Hour + (2 * time.Minute)) + } + + lookedUpUserToken, err = userAuthTokenService.LookupToken(unhashedPrev) + So(err, ShouldBeNil) + lookedUpModel, err = extractModelFromToken(lookedUpUserToken) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + + lookedUpModel, err = ctx.getAuthTokenByID(lookedUpModel.Id) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + So(lookedUpModel.AuthTokenSeen, ShouldBeFalse) + + rotated, err = userAuthTokenService.TryRotateToken(userToken, "192.168.10.12:1234", "a new user agent") + So(err, ShouldBeNil) + So(rotated, ShouldBeTrue) + + model, err = ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + So(model, ShouldNotBeNil) + So(model.SeenAt, ShouldEqual, 0) + }) + + Convey("keeps prev token valid for 1 minute after it is confirmed", func() { + userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + model, err := extractModelFromToken(userToken) + So(err, ShouldBeNil) + So(model, ShouldNotBeNil) + + lookedUpUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + So(lookedUpUserToken, ShouldNotBeNil) + + getTime = func() time.Time { + return t.Add(10 * time.Minute) + } + + prevToken := model.UnhashedToken + rotated, err := userAuthTokenService.TryRotateToken(userToken, "1.1.1.1", "firefox") + So(err, ShouldBeNil) + So(rotated, ShouldBeTrue) + + getTime = func() time.Time { + return t.Add(20 * time.Minute) + } + + currentUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + So(currentUserToken, ShouldNotBeNil) + + prevUserToken, err := userAuthTokenService.LookupToken(prevToken) + So(err, ShouldBeNil) + So(prevUserToken, ShouldNotBeNil) + }) + + Convey("will not mark token unseen when prev and current are the same", func() { + userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + model, err := extractModelFromToken(userToken) + So(err, ShouldBeNil) + So(model, ShouldNotBeNil) + + lookedUpUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + lookedUpModel, err := extractModelFromToken(lookedUpUserToken) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + + lookedUpUserToken, err = userAuthTokenService.LookupToken(model.UnhashedToken) + So(err, ShouldBeNil) + lookedUpModel, err = extractModelFromToken(lookedUpUserToken) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + + lookedUpModel, err = ctx.getAuthTokenByID(lookedUpModel.Id) + So(err, ShouldBeNil) + So(lookedUpModel, ShouldNotBeNil) + So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + }) + + Convey("Rotate token", func() { + userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") + So(err, ShouldBeNil) + model, err := extractModelFromToken(userToken) + So(err, ShouldBeNil) + So(model, ShouldNotBeNil) + + prevToken := model.AuthToken + + Convey("Should rotate current token and previous token when auth token seen", func() { + updated, err := ctx.markAuthTokenAsSeen(model.Id) + So(err, ShouldBeNil) + So(updated, ShouldBeTrue) + + getTime = func() time.Time { + return t.Add(10 * time.Minute) + } + + rotated, err := userAuthTokenService.TryRotateToken(userToken, "1.1.1.1", "firefox") + So(err, ShouldBeNil) + So(rotated, ShouldBeTrue) + + storedToken, err := ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + So(storedToken, ShouldNotBeNil) + So(storedToken.AuthTokenSeen, ShouldBeFalse) + So(storedToken.PrevAuthToken, ShouldEqual, prevToken) + So(storedToken.AuthToken, ShouldNotEqual, prevToken) + + prevToken = storedToken.AuthToken + + updated, err = ctx.markAuthTokenAsSeen(model.Id) + So(err, ShouldBeNil) + So(updated, ShouldBeTrue) + + getTime = func() time.Time { + return t.Add(20 * time.Minute) + } + + rotated, err = userAuthTokenService.TryRotateToken(userToken, "1.1.1.1", "firefox") + So(err, ShouldBeNil) + So(rotated, ShouldBeTrue) + + storedToken, err = ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + So(storedToken, ShouldNotBeNil) + So(storedToken.AuthTokenSeen, ShouldBeFalse) + So(storedToken.PrevAuthToken, ShouldEqual, prevToken) + So(storedToken.AuthToken, ShouldNotEqual, prevToken) + }) + + Convey("Should rotate current token, but keep previous token when auth token not seen", func() { + model.RotatedAt = getTime().Add(-2 * time.Minute).Unix() + + getTime = func() time.Time { + return t.Add(2 * time.Minute) + } + + rotated, err := userAuthTokenService.TryRotateToken(userToken, "1.1.1.1", "firefox") + So(err, ShouldBeNil) + So(rotated, ShouldBeTrue) + + storedToken, err := ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + So(storedToken, ShouldNotBeNil) + So(storedToken.AuthTokenSeen, ShouldBeFalse) + So(storedToken.PrevAuthToken, ShouldEqual, prevToken) + So(storedToken.AuthToken, ShouldNotEqual, prevToken) + }) + }) + + Reset(func() { + getTime = time.Now + }) + }) +} + +func createTestContext(t *testing.T) *testContext { + t.Helper() + + sqlstore := sqlstore.InitTestDB(t) + tokenService := &UserAuthTokenServiceImpl{ + SQLStore: sqlstore, + Cfg: &setting.Cfg{ + LoginCookieName: "grafana_session", + LoginCookieMaxDays: 7, + LoginDeleteExpiredTokensAfterDays: 30, + LoginCookieRotation: 10, + }, + log: log.New("test-logger"), + } + + return &testContext{ + sqlstore: sqlstore, + tokenService: tokenService, + } +} + +type testContext struct { + sqlstore *sqlstore.SqlStore + tokenService *UserAuthTokenServiceImpl +} + +func (c *testContext) getAuthTokenByID(id int64) (*userAuthToken, error) { + sess := c.sqlstore.NewSession() + var t userAuthToken + found, err := sess.ID(id).Get(&t) + if err != nil || !found { + return nil, err + } + + return &t, nil +} + +func (c *testContext) markAuthTokenAsSeen(id int64) (bool, error) { + sess := c.sqlstore.NewSession() + res, err := sess.Exec("UPDATE user_auth_token SET auth_token_seen = ? WHERE id = ?", c.sqlstore.Dialect.BooleanStr(true), id) + if err != nil { + return false, err + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return false, err + } + return rowsAffected == 1, nil +} diff --git a/pkg/services/auth/authtoken/model.go b/pkg/services/auth/authtoken/model.go new file mode 100644 index 00000000000..8bd89c68b04 --- /dev/null +++ b/pkg/services/auth/authtoken/model.go @@ -0,0 +1,76 @@ +package authtoken + +import ( + "errors" + "fmt" + + "github.com/grafana/grafana/pkg/services/auth" +) + +// Typed errors +var ( + ErrAuthTokenNotFound = errors.New("user auth token not found") +) + +type userAuthToken struct { + Id int64 + UserId int64 + AuthToken string + PrevAuthToken string + UserAgent string + ClientIp string + AuthTokenSeen bool + SeenAt int64 + RotatedAt int64 + CreatedAt int64 + UpdatedAt int64 + UnhashedToken string `xorm:"-"` +} + +func (uat *userAuthToken) toUserToken() (auth.UserToken, error) { + if uat == nil { + return nil, fmt.Errorf("needs pointer to userAuthToken struct") + } + + return &userTokenImpl{ + userAuthToken: uat, + }, nil +} + +type userToken interface { + auth.UserToken + GetModel() *userAuthToken +} + +type userTokenImpl struct { + *userAuthToken +} + +func (ut *userTokenImpl) GetUserId() int64 { + return ut.UserId +} + +func (ut *userTokenImpl) GetToken() string { + return ut.UnhashedToken +} + +func (ut *userTokenImpl) GetModel() *userAuthToken { + return ut.userAuthToken +} + +func extractModelFromToken(token auth.UserToken) (*userAuthToken, error) { + ut, ok := token.(userToken) + if !ok { + return nil, fmt.Errorf("failed to cast token") + } + + return ut.GetModel(), nil +} + +// UserAuthTokenService are used for generating and validating user auth tokens +type UserAuthTokenService interface { + CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) + LookupToken(unhashedToken string) (auth.UserToken, error) + TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) + RevokeToken(token auth.UserToken) error +} diff --git a/pkg/services/auth/session_cleanup.go b/pkg/services/auth/authtoken/session_cleanup.go similarity index 98% rename from pkg/services/auth/session_cleanup.go rename to pkg/services/auth/authtoken/session_cleanup.go index 7e523181a7b..cd2b766d6c0 100644 --- a/pkg/services/auth/session_cleanup.go +++ b/pkg/services/auth/authtoken/session_cleanup.go @@ -1,4 +1,4 @@ -package auth +package authtoken import ( "context" diff --git a/pkg/services/auth/session_cleanup_test.go b/pkg/services/auth/authtoken/session_cleanup_test.go similarity index 98% rename from pkg/services/auth/session_cleanup_test.go rename to pkg/services/auth/authtoken/session_cleanup_test.go index eef2cd74d04..101a279c374 100644 --- a/pkg/services/auth/session_cleanup_test.go +++ b/pkg/services/auth/authtoken/session_cleanup_test.go @@ -1,4 +1,4 @@ -package auth +package authtoken import ( "fmt" diff --git a/pkg/services/auth/model.go b/pkg/services/auth/model.go deleted file mode 100644 index 7a0f49539f2..00000000000 --- a/pkg/services/auth/model.go +++ /dev/null @@ -1,25 +0,0 @@ -package auth - -import ( - "errors" -) - -// Typed errors -var ( - ErrAuthTokenNotFound = errors.New("User auth token not found") -) - -type userAuthToken struct { - Id int64 - UserId int64 - AuthToken string - PrevAuthToken string - UserAgent string - ClientIp string - AuthTokenSeen bool - SeenAt int64 - RotatedAt int64 - CreatedAt int64 - UpdatedAt int64 - UnhashedToken string `xorm:"-"` -} From d53e64a32c48fca9149d2e291313a6dc8b04bb53 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 4 Feb 2019 23:44:28 +0100 Subject: [PATCH 02/19] move auth token middleware/hooks to middleware package fix/adds auth token middleware tests --- pkg/api/common_test.go | 63 +++++++++-- pkg/api/http_server.go | 18 +-- pkg/api/login.go | 17 ++- pkg/middleware/middleware.go | 65 ++++++++++- pkg/middleware/middleware_test.go | 165 +++++++++++++++++++++++++--- pkg/middleware/org_redirect_test.go | 31 ++++-- pkg/middleware/quota_test.go | 16 ++- pkg/models/context.go | 2 + 8 files changed, 324 insertions(+), 53 deletions(-) diff --git a/pkg/api/common_test.go b/pkg/api/common_test.go index fe02c94e277..853a04b5c11 100644 --- a/pkg/api/common_test.go +++ b/pkg/api/common_test.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" "gopkg.in/macaron.v1" . "github.com/smartystreets/goconvey/convey" @@ -129,24 +130,70 @@ func setupScenarioContext(url string) *scenarioContext { return sc } +type fakeUserToken interface { + auth.UserToken + SetToken(token string) +} + +type userTokenImpl struct { + userId int64 + token string +} + +func (ut *userTokenImpl) GetUserId() int64 { + return ut.userId +} + +func (ut *userTokenImpl) GetToken() string { + return ut.token +} + +func (ut *userTokenImpl) SetToken(token string) { + ut.token = token +} + type fakeUserAuthTokenService struct { - initContextWithTokenProvider func(ctx *m.ReqContext, orgID int64) bool + createTokenProvider func(userId int64, clientIP, userAgent string) (auth.UserToken, error) + tryRotateTokenProvider func(token auth.UserToken, clientIP, userAgent string) (bool, error) + lookupTokenProvider func(unhashedToken string) (auth.UserToken, error) + revokeTokenProvider func(token auth.UserToken) error } func newFakeUserAuthTokenService() *fakeUserAuthTokenService { return &fakeUserAuthTokenService{ - initContextWithTokenProvider: func(ctx *m.ReqContext, orgID int64) bool { - return false + createTokenProvider: func(userId int64, clientIP, userAgent string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 0, + token: "", + }, nil + }, + tryRotateTokenProvider: func(token auth.UserToken, clientIP, userAgent string) (bool, error) { + return false, nil + }, + lookupTokenProvider: func(unhashedToken string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 0, + token: "", + }, nil + }, + revokeTokenProvider: func(token auth.UserToken) error { + return nil }, } } -func (s *fakeUserAuthTokenService) InitContextWithToken(ctx *m.ReqContext, orgID int64) bool { - return s.initContextWithTokenProvider(ctx, orgID) +func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) { + return s.createTokenProvider(userId, clientIP, userAgent) } -func (s *fakeUserAuthTokenService) UserAuthenticatedHook(user *m.User, c *m.ReqContext) error { - return nil +func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (auth.UserToken, error) { + return s.lookupTokenProvider(unhashedToken) } -func (s *fakeUserAuthTokenService) SignOutUser(c *m.ReqContext) error { return nil } +func (s *fakeUserAuthTokenService) TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) { + return s.tryRotateTokenProvider(token, clientIP, userAgent) +} + +func (s *fakeUserAuthTokenService) RevokeToken(token auth.UserToken) error { + return s.revokeTokenProvider(token) +} diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 7b7c1478a4c..a0a65d73244 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -21,7 +21,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry" - "github.com/grafana/grafana/pkg/services/auth" + "github.com/grafana/grafana/pkg/services/auth/authtoken" "github.com/grafana/grafana/pkg/services/cache" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/hooks" @@ -48,14 +48,14 @@ type HTTPServer struct { streamManager *live.StreamManager httpSrv *http.Server - RouteRegister routing.RouteRegister `inject:""` - Bus bus.Bus `inject:""` - RenderService rendering.Service `inject:""` - Cfg *setting.Cfg `inject:""` - HooksService *hooks.HooksService `inject:""` - CacheService *cache.CacheService `inject:""` - DatasourceCache datasources.CacheService `inject:""` - AuthTokenService auth.UserAuthTokenService `inject:""` + RouteRegister routing.RouteRegister `inject:""` + Bus bus.Bus `inject:""` + RenderService rendering.Service `inject:""` + Cfg *setting.Cfg `inject:""` + HooksService *hooks.HooksService `inject:""` + CacheService *cache.CacheService `inject:""` + DatasourceCache datasources.CacheService `inject:""` + AuthTokenService authtoken.UserAuthTokenService `inject:""` } func (hs *HTTPServer) Init() error { diff --git a/pkg/api/login.go b/pkg/api/login.go index 49da147724e..d25e83d34e8 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -5,11 +5,14 @@ import ( "net/http" "net/url" + "github.com/grafana/grafana/pkg/services/auth/authtoken" + "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/login" "github.com/grafana/grafana/pkg/metrics" + "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -126,17 +129,23 @@ func (hs *HTTPServer) LoginPost(c *m.ReqContext, cmd dtos.LoginCommand) Response func (hs *HTTPServer) loginUserWithUser(user *m.User, c *m.ReqContext) { if user == nil { - hs.log.Error("User login with nil user") + hs.log.Error("user login with nil user") } - err := hs.AuthTokenService.UserAuthenticatedHook(user, c) + userToken, err := hs.AuthTokenService.CreateToken(user.Id, c.RemoteAddr(), c.Req.UserAgent()) if err != nil { - hs.log.Error("User auth hook failed", "error", err) + hs.log.Error("failed to create auth token", "error", err) } + + middleware.WriteSessionCookie(c, userToken.GetToken(), middleware.OneYearInSeconds) } func (hs *HTTPServer) Logout(c *m.ReqContext) { - hs.AuthTokenService.SignOutUser(c) + if err := hs.AuthTokenService.RevokeToken(c.UserToken); err != nil && err != authtoken.ErrAuthTokenNotFound { + hs.log.Error("failed to revoke auth token", "error", err) + } + + middleware.WriteSessionCookie(c, "", -1) if setting.SignoutRedirectUrl != "" { c.Redirect(setting.SignoutRedirectUrl) diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 3722ac3058f..6cf29340b82 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -1,13 +1,15 @@ package middleware import ( + "net/http" + "net/url" "strconv" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/apikeygen" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/auth" + "github.com/grafana/grafana/pkg/services/auth/authtoken" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -21,7 +23,7 @@ var ( ReqOrgAdmin = RoleAuth(m.ROLE_ADMIN) ) -func GetContextHandler(ats auth.UserAuthTokenService) macaron.Handler { +func GetContextHandler(ats authtoken.UserAuthTokenService) macaron.Handler { return func(c *macaron.Context) { ctx := &m.ReqContext{ Context: c, @@ -49,7 +51,7 @@ func GetContextHandler(ats auth.UserAuthTokenService) macaron.Handler { case initContextWithApiKey(ctx): case initContextWithBasicAuth(ctx, orgId): case initContextWithAuthProxy(ctx, orgId): - case ats.InitContextWithToken(ctx, orgId): + case initContextWithToken(ats, ctx, orgId): case initContextWithAnonymousUser(ctx): } @@ -166,6 +168,63 @@ func initContextWithBasicAuth(ctx *m.ReqContext, orgId int64) bool { return true } +const cookieName = "grafana_session" +const OneYearInSeconds = 31557600 //used as default maxage for session cookies. We validate/rotate them more often. + +func initContextWithToken(authTokenService authtoken.UserAuthTokenService, ctx *m.ReqContext, orgID int64) bool { + rawToken := ctx.GetCookie(cookieName) + if rawToken == "" { + return false + } + + token, err := authTokenService.LookupToken(rawToken) + if err != nil { + ctx.Logger.Error("failed to look up user based on cookie", "error", err) + return false + } + + query := m.GetSignedInUserQuery{UserId: token.GetUserId(), OrgId: orgID} + if err := bus.Dispatch(&query); err != nil { + ctx.Logger.Error("failed to get user with id", "userId", token.GetUserId(), "error", err) + return false + } + + ctx.SignedInUser = query.Result + ctx.IsSignedIn = true + ctx.UserToken = token + + rotated, err := authTokenService.TryRotateToken(token, ctx.RemoteAddr(), ctx.Req.UserAgent()) + if err != nil { + ctx.Logger.Error("failed to rotate token", "error", err) + return true + } + + if rotated { + WriteSessionCookie(ctx, token.GetToken(), OneYearInSeconds) + } + + return true +} + +func WriteSessionCookie(ctx *m.ReqContext, value string, maxAge int) { + if setting.Env == setting.DEV { + ctx.Logger.Info("new token", "unhashed token", value) + } + + ctx.Resp.Header().Del("Set-Cookie") + cookie := http.Cookie{ + Name: cookieName, + Value: url.QueryEscape(value), + HttpOnly: true, + Path: setting.AppSubUrl + "/", + Secure: false, // TODO: use setting SecurityHTTPSCookies + MaxAge: maxAge, + SameSite: http.SameSiteLaxMode, // TODO: use setting LoginCookieSameSite + } + + http.SetCookie(ctx.Resp, &cookie) +} + func AddDefaultResponseHeaders() macaron.Handler { return func(ctx *m.ReqContext) { if ctx.IsApiRequest() && ctx.Req.Method == "GET" { diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 4679c449853..4e10ee39201 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -10,6 +10,8 @@ import ( msession "github.com/go-macaron/session" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" + "github.com/grafana/grafana/pkg/services/auth/authtoken" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -146,17 +148,91 @@ func TestMiddlewareContext(t *testing.T) { }) }) - middlewareScenario("Auth token service", func(sc *scenarioContext) { - var wasCalled bool - sc.userAuthTokenService.initContextWithTokenProvider = func(ctx *m.ReqContext, orgId int64) bool { - wasCalled = true - return false + middlewareScenario("Non-expired auth token in cookie which not are being rotated", func(sc *scenarioContext) { + sc.withTokenSessionCookie("token") + + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { + query.Result = &m.SignedInUser{OrgId: 2, UserId: 12} + return nil + }) + + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 12, + token: unhashedToken, + }, nil } sc.fakeReq("GET", "/").exec() - Convey("should call middleware", func() { - So(wasCalled, ShouldBeTrue) + Convey("should init context with user info", func() { + So(sc.context.IsSignedIn, ShouldBeTrue) + So(sc.context.UserId, ShouldEqual, 12) + So(sc.context.UserToken.GetUserId(), ShouldEqual, 12) + So(sc.context.UserToken.GetToken(), ShouldEqual, "token") + }) + + Convey("should not set cookie", func() { + So(sc.resp.Header().Get("Set-Cookie"), ShouldEqual, "") + }) + }) + + middlewareScenario("Non-expired auth token in cookie which are being rotated", func(sc *scenarioContext) { + sc.withTokenSessionCookie("token") + + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { + query.Result = &m.SignedInUser{OrgId: 2, UserId: 12} + return nil + }) + + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 12, + token: unhashedToken, + }, nil + } + + sc.userAuthTokenService.tryRotateTokenProvider = func(userToken auth.UserToken, clientIP, userAgent string) (bool, error) { + userToken.(fakeUserToken).SetToken("rotated") + return true, nil + } + + expectedCookie := &http.Cookie{ + Name: cookieName, + Value: "rotated", + Path: setting.AppSubUrl + "/", + HttpOnly: true, + MaxAge: OneYearInSeconds, + SameSite: http.SameSiteLaxMode, + } + + sc.fakeReq("GET", "/").exec() + + Convey("should init context with user info", func() { + So(sc.context.IsSignedIn, ShouldBeTrue) + So(sc.context.UserId, ShouldEqual, 12) + So(sc.context.UserToken.GetUserId(), ShouldEqual, 12) + So(sc.context.UserToken.GetToken(), ShouldEqual, "rotated") + }) + + Convey("should set cookie", func() { + So(sc.resp.Header().Get("Set-Cookie"), ShouldEqual, expectedCookie.String()) + }) + }) + + middlewareScenario("Invalid/expired auth token in cookie", func(sc *scenarioContext) { + sc.withTokenSessionCookie("token") + + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { + return nil, authtoken.ErrAuthTokenNotFound + } + + sc.fakeReq("GET", "/").exec() + + Convey("should not init context with user info", func() { + So(sc.context.IsSignedIn, ShouldBeFalse) + So(sc.context.UserId, ShouldEqual, 0) + So(sc.context.UserToken, ShouldBeNil) }) }) @@ -508,6 +584,7 @@ type scenarioContext struct { resp *httptest.ResponseRecorder apiKey string authHeader string + tokenSessionCookie string respJson map[string]interface{} handlerFunc handlerFunc defaultHandler macaron.Handler @@ -522,6 +599,11 @@ func (sc *scenarioContext) withValidApiKey() *scenarioContext { return sc } +func (sc *scenarioContext) withTokenSessionCookie(unhashedToken string) *scenarioContext { + sc.tokenSessionCookie = unhashedToken + return sc +} + func (sc *scenarioContext) withAuthorizationHeader(authHeader string) *scenarioContext { sc.authHeader = authHeader return sc @@ -571,6 +653,13 @@ func (sc *scenarioContext) exec() { sc.req.Header.Add("Authorization", sc.authHeader) } + if sc.tokenSessionCookie != "" { + sc.req.AddCookie(&http.Cookie{ + Name: cookieName, + Value: sc.tokenSessionCookie, + }) + } + sc.m.ServeHTTP(sc.resp, sc.req) if sc.resp.Header().Get("Content-Type") == "application/json; charset=UTF-8" { @@ -582,24 +671,70 @@ func (sc *scenarioContext) exec() { type scenarioFunc func(c *scenarioContext) type handlerFunc func(c *m.ReqContext) +type fakeUserToken interface { + auth.UserToken + SetToken(token string) +} + +type userTokenImpl struct { + userId int64 + token string +} + +func (ut *userTokenImpl) GetUserId() int64 { + return ut.userId +} + +func (ut *userTokenImpl) GetToken() string { + return ut.token +} + +func (ut *userTokenImpl) SetToken(token string) { + ut.token = token +} + type fakeUserAuthTokenService struct { - initContextWithTokenProvider func(ctx *m.ReqContext, orgID int64) bool + createTokenProvider func(userId int64, clientIP, userAgent string) (auth.UserToken, error) + tryRotateTokenProvider func(token auth.UserToken, clientIP, userAgent string) (bool, error) + lookupTokenProvider func(unhashedToken string) (auth.UserToken, error) + revokeTokenProvider func(token auth.UserToken) error } func newFakeUserAuthTokenService() *fakeUserAuthTokenService { return &fakeUserAuthTokenService{ - initContextWithTokenProvider: func(ctx *m.ReqContext, orgID int64) bool { - return false + createTokenProvider: func(userId int64, clientIP, userAgent string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 0, + token: "", + }, nil + }, + tryRotateTokenProvider: func(token auth.UserToken, clientIP, userAgent string) (bool, error) { + return false, nil + }, + lookupTokenProvider: func(unhashedToken string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 0, + token: "", + }, nil + }, + revokeTokenProvider: func(token auth.UserToken) error { + return nil }, } } -func (s *fakeUserAuthTokenService) InitContextWithToken(ctx *m.ReqContext, orgID int64) bool { - return s.initContextWithTokenProvider(ctx, orgID) +func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) { + return s.createTokenProvider(userId, clientIP, userAgent) } -func (s *fakeUserAuthTokenService) UserAuthenticatedHook(user *m.User, c *m.ReqContext) error { - return nil +func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (auth.UserToken, error) { + return s.lookupTokenProvider(unhashedToken) } -func (s *fakeUserAuthTokenService) SignOutUser(c *m.ReqContext) error { return nil } +func (s *fakeUserAuthTokenService) TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) { + return s.tryRotateTokenProvider(token, clientIP, userAgent) +} + +func (s *fakeUserAuthTokenService) RevokeToken(token auth.UserToken) error { + return s.revokeTokenProvider(token) +} diff --git a/pkg/middleware/org_redirect_test.go b/pkg/middleware/org_redirect_test.go index 46b8776fdcc..c7479b3e9bc 100644 --- a/pkg/middleware/org_redirect_test.go +++ b/pkg/middleware/org_redirect_test.go @@ -3,6 +3,8 @@ package middleware import ( "testing" + "github.com/grafana/grafana/pkg/services/auth" + "fmt" "github.com/grafana/grafana/pkg/bus" @@ -14,14 +16,21 @@ func TestOrgRedirectMiddleware(t *testing.T) { Convey("Can redirect to correct org", t, func() { middlewareScenario("when setting a correct org for the user", func(sc *scenarioContext) { + sc.withTokenSessionCookie("token") bus.AddHandler("test", func(query *m.SetUsingOrgCommand) error { return nil }) - sc.userAuthTokenService.initContextWithTokenProvider = func(ctx *m.ReqContext, orgId int64) bool { - ctx.SignedInUser = &m.SignedInUser{OrgId: 1, UserId: 12} - ctx.IsSignedIn = true - return true + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { + query.Result = &m.SignedInUser{OrgId: 1, UserId: 12} + return nil + }) + + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 12, + token: "", + }, nil } sc.m.Get("/", sc.defaultHandler) @@ -33,21 +42,23 @@ func TestOrgRedirectMiddleware(t *testing.T) { }) middlewareScenario("when setting an invalid org for user", func(sc *scenarioContext) { + sc.withTokenSessionCookie("token") bus.AddHandler("test", func(query *m.SetUsingOrgCommand) error { return fmt.Errorf("") }) - sc.userAuthTokenService.initContextWithTokenProvider = func(ctx *m.ReqContext, orgId int64) bool { - ctx.SignedInUser = &m.SignedInUser{OrgId: 1, UserId: 12} - ctx.IsSignedIn = true - return true - } - bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { query.Result = &m.SignedInUser{OrgId: 1, UserId: 12} return nil }) + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 12, + token: "", + }, nil + } + sc.m.Get("/", sc.defaultHandler) sc.fakeReq("GET", "/?orgId=3").exec() diff --git a/pkg/middleware/quota_test.go b/pkg/middleware/quota_test.go index 4f2203a5d3d..af22f41deba 100644 --- a/pkg/middleware/quota_test.go +++ b/pkg/middleware/quota_test.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" . "github.com/smartystreets/goconvey/convey" @@ -74,10 +75,17 @@ func TestMiddlewareQuota(t *testing.T) { }) middlewareScenario("with user logged in", func(sc *scenarioContext) { - sc.userAuthTokenService.initContextWithTokenProvider = func(ctx *m.ReqContext, orgId int64) bool { - ctx.SignedInUser = &m.SignedInUser{OrgId: 2, UserId: 12} - ctx.IsSignedIn = true - return true + sc.withTokenSessionCookie("token") + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { + query.Result = &m.SignedInUser{OrgId: 2, UserId: 12} + return nil + }) + + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { + return &userTokenImpl{ + userId: 12, + token: "", + }, nil } bus.AddHandler("globalQuota", func(query *m.GetGlobalQuotaByTargetQuery) error { diff --git a/pkg/models/context.go b/pkg/models/context.go index df970451304..da63db63f45 100644 --- a/pkg/models/context.go +++ b/pkg/models/context.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/prometheus/client_golang/prometheus" @@ -13,6 +14,7 @@ import ( type ReqContext struct { *macaron.Context *SignedInUser + UserToken auth.UserToken // This should only be used by the auth_proxy Session session.SessionStore From 1d1b617cee3cfedeb03586e0db00f5219d187761 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 5 Feb 2019 21:08:55 +0100 Subject: [PATCH 03/19] remove unused code --- pkg/api/common_test.go | 87 ++++-------------------------------------- 1 file changed, 8 insertions(+), 79 deletions(-) diff --git a/pkg/api/common_test.go b/pkg/api/common_test.go index 853a04b5c11..3f3a50aae69 100644 --- a/pkg/api/common_test.go +++ b/pkg/api/common_test.go @@ -8,7 +8,6 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/auth" "gopkg.in/macaron.v1" . "github.com/smartystreets/goconvey/convey" @@ -95,14 +94,13 @@ func (sc *scenarioContext) fakeReqWithParams(method, url string, queryParams map } type scenarioContext struct { - m *macaron.Macaron - context *m.ReqContext - resp *httptest.ResponseRecorder - handlerFunc handlerFunc - defaultHandler macaron.Handler - req *http.Request - url string - userAuthTokenService *fakeUserAuthTokenService + m *macaron.Macaron + context *m.ReqContext + resp *httptest.ResponseRecorder + handlerFunc handlerFunc + defaultHandler macaron.Handler + req *http.Request + url string } func (sc *scenarioContext) exec() { @@ -124,76 +122,7 @@ func setupScenarioContext(url string) *scenarioContext { Delims: macaron.Delims{Left: "[[", Right: "]]"}, })) - sc.userAuthTokenService = newFakeUserAuthTokenService() - sc.m.Use(middleware.GetContextHandler(sc.userAuthTokenService)) + sc.m.Use(middleware.GetContextHandler(nil)) return sc } - -type fakeUserToken interface { - auth.UserToken - SetToken(token string) -} - -type userTokenImpl struct { - userId int64 - token string -} - -func (ut *userTokenImpl) GetUserId() int64 { - return ut.userId -} - -func (ut *userTokenImpl) GetToken() string { - return ut.token -} - -func (ut *userTokenImpl) SetToken(token string) { - ut.token = token -} - -type fakeUserAuthTokenService struct { - createTokenProvider func(userId int64, clientIP, userAgent string) (auth.UserToken, error) - tryRotateTokenProvider func(token auth.UserToken, clientIP, userAgent string) (bool, error) - lookupTokenProvider func(unhashedToken string) (auth.UserToken, error) - revokeTokenProvider func(token auth.UserToken) error -} - -func newFakeUserAuthTokenService() *fakeUserAuthTokenService { - return &fakeUserAuthTokenService{ - createTokenProvider: func(userId int64, clientIP, userAgent string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 0, - token: "", - }, nil - }, - tryRotateTokenProvider: func(token auth.UserToken, clientIP, userAgent string) (bool, error) { - return false, nil - }, - lookupTokenProvider: func(unhashedToken string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 0, - token: "", - }, nil - }, - revokeTokenProvider: func(token auth.UserToken) error { - return nil - }, - } -} - -func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) { - return s.createTokenProvider(userId, clientIP, userAgent) -} - -func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (auth.UserToken, error) { - return s.lookupTokenProvider(unhashedToken) -} - -func (s *fakeUserAuthTokenService) TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) { - return s.tryRotateTokenProvider(token, clientIP, userAgent) -} - -func (s *fakeUserAuthTokenService) RevokeToken(token auth.UserToken) error { - return s.revokeTokenProvider(token) -} From 3c2fd02bc00ac631b1429533e898b848ed22dc47 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 5 Feb 2019 21:09:55 +0100 Subject: [PATCH 04/19] refactor login/auth token configuration settings remove login section and reuse existing sections security and auth --- conf/defaults.ini | 41 +++++++++++---------- conf/sample.ini | 43 +++++++++++----------- pkg/setting/setting.go | 82 ++++++++++++++++++++++++------------------ 3 files changed, 89 insertions(+), 77 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index d021d342fbf..c65cb93d426 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -106,25 +106,6 @@ path = grafana.db # For "sqlite3" only. cache mode setting used for connecting to the database cache_mode = private -#################################### Login ############################### - -[login] - -# Login cookie name -cookie_name = grafana_session - -# Login cookie same site setting. defaults to `lax`. can be set to "lax", "strict" and "none" -cookie_samesite = lax - -# How many days an session can be unused before we inactivate it -login_remember_days = 7 - -# How often should the login token be rotated. default to '10m' -rotate_token_minutes = 10 - -# How long should Grafana keep expired tokens before deleting them -delete_expired_token_after_days = 30 - #################################### Session ############################# [session] # Either "memory", "file", "redis", "mysql", "postgres", "memcache", default is "file" @@ -206,8 +187,11 @@ data_source_proxy_whitelist = # disable protection against brute force login attempts disable_brute_force_login_protection = false -# set cookies as https only. default is false -https_flag_cookies = false +# set to true if you host Grafana behind HTTPS. default is false. +cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none" +cookie_samesite = lax #################################### Snapshots ########################### [snapshots] @@ -260,6 +244,21 @@ external_manage_info = viewers_can_edit = false [auth] +# Login cookie name +login_cookie_name = grafana_session + +# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days. +login_maximum_inactive_lifetime_days = 7 + +# The maximum lifetime (days) an autenticated user can be logged in since login time before being required to login. Default is 30 days. +login_maximum_lifetime_days = 30 + +# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. +token_rotation_interval_minutes = 10 + +# How often should expired auth tokens be deleted from the database. The default is 7 days. +expired_tokens_cleanup_interval_days = 7 + # Set to true to disable (hide) the login form, useful if you use OAuth disable_login_form = false diff --git a/conf/sample.ini b/conf/sample.ini index ef677320686..39feb31441e 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -102,25 +102,6 @@ log_queries = # For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) ;cache_mode = private -#################################### Login ############################### - -[login] - -# Login cookie name -;cookie_name = grafana_session - -# Login cookie same site setting. defaults to `lax`. can be set to "lax", "strict" and "none" -;cookie_samesite = lax - -# How many days an session can be unused before we inactivate it -;login_remember_days = 7 - -# How often should the login token be rotated. default to '10' -;rotate_token_minutes = 10 - -# How long should Grafana keep expired tokens before deleting them -;delete_expired_token_after_days = 30 - #################################### Session #################################### [session] # Either "memory", "file", "redis", "mysql", "postgres", default is "file" @@ -193,8 +174,11 @@ log_queries = # disable protection against brute force login attempts ;disable_brute_force_login_protection = false -# set cookies as https only. default is false -;https_flag_cookies = false +# set to true if you host Grafana behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none" +;cookie_samesite = lax #################################### Snapshots ########################### [snapshots] @@ -240,6 +224,21 @@ log_queries = ;viewers_can_edit = false [auth] +# Login cookie name +;login_cookie_name = grafana_session + +# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days, +;login_maximum_inactive_lifetime_days = 7 + +# The maximum lifetime (days) an autenticated user can be logged in since login time before being required to login. Default is 30 days. +;login_maximum_lifetime_days = 30 + +# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. +;token_rotation_interval_minutes = 10 + +# How often should expired auth tokens be deleted from the database. The default is 7 days. +;expired_tokens_cleanup_interval_days = 7 + # Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false ;disable_login_form = false @@ -253,7 +252,7 @@ log_queries = # This setting is ignored if multiple OAuth providers are configured. ;oauth_auto_login = false -#################################### Anonymous Auth ########################## +#################################### Anonymous Auth ###################### [auth.anonymous] # enable anonymous access ;enabled = false diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index c3c78d10fec..9f7d03bb472 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -89,6 +89,8 @@ var ( EmailCodeValidMinutes int DataProxyWhiteList map[string]bool DisableBruteForceLoginProtection bool + CookieSecure bool + CookieSameSite http.SameSite // Snapshots ExternalSnapshotUrl string @@ -118,8 +120,10 @@ var ( ViewersCanEdit bool // Http auth - AdminUser string - AdminPassword string + AdminUser string + AdminPassword string + LoginCookieName string + LoginMaxLifetimeDays int AnonymousEnabled bool AnonymousOrgName string @@ -215,7 +219,11 @@ type Cfg struct { RendererLimit int RendererLimitAlerting int + // Security DisableBruteForceLoginProtection bool + CookieSecure bool + CookieSameSite http.SameSite + TempDataLifetime time.Duration MetricsEndpointEnabled bool MetricsEndpointBasicAuthUsername string @@ -224,13 +232,12 @@ type Cfg struct { DisableSanitizeHtml bool EnterpriseLicensePath string - LoginCookieName string - LoginCookieMaxDays int - LoginCookieRotation int - LoginDeleteExpiredTokensAfterDays int - LoginCookieSameSite http.SameSite - - SecurityHTTPSCookies bool + // Auth + LoginCookieName string + LoginMaxInactiveLifetimeDays int + LoginMaxLifetimeDays int + TokenRotationIntervalMinutes int + ExpiredTokensCleanupIntervalDays int } type CommandLineArgs struct { @@ -554,30 +561,6 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { ApplicationName = APP_NAME_ENTERPRISE } - //login - login := iniFile.Section("login") - cfg.LoginCookieName = login.Key("cookie_name").MustString("grafana_session") - cfg.LoginCookieMaxDays = login.Key("login_remember_days").MustInt(7) - cfg.LoginDeleteExpiredTokensAfterDays = login.Key("delete_expired_token_after_days").MustInt(30) - - samesiteString := login.Key("cookie_samesite").MustString("lax") - validSameSiteValues := map[string]http.SameSite{ - "lax": http.SameSiteLaxMode, - "strict": http.SameSiteStrictMode, - "none": http.SameSiteDefaultMode, - } - - if samesite, ok := validSameSiteValues[samesiteString]; ok { - cfg.LoginCookieSameSite = samesite - } else { - cfg.LoginCookieSameSite = http.SameSiteLaxMode - } - - cfg.LoginCookieRotation = login.Key("rotate_token_minutes").MustInt(10) - if cfg.LoginCookieRotation < 2 { - cfg.LoginCookieRotation = 2 - } - Env = iniFile.Section("").Key("app_mode").MustString("development") InstanceName = iniFile.Section("").Key("instance_name").MustString("unknown_instance_name") PluginsPath = makeAbsolute(iniFile.Section("paths").Key("plugins").String(), HomePath) @@ -621,9 +604,26 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { SecretKey = security.Key("secret_key").String() DisableGravatar = security.Key("disable_gravatar").MustBool(true) cfg.DisableBruteForceLoginProtection = security.Key("disable_brute_force_login_protection").MustBool(false) - cfg.SecurityHTTPSCookies = security.Key("https_flag_cookies").MustBool(false) DisableBruteForceLoginProtection = cfg.DisableBruteForceLoginProtection + CookieSecure = security.Key("cookie_secure").MustBool(false) + cfg.CookieSecure = CookieSecure + + samesiteString := security.Key("cookie_samesite").MustString("lax") + validSameSiteValues := map[string]http.SameSite{ + "lax": http.SameSiteLaxMode, + "strict": http.SameSiteStrictMode, + "none": http.SameSiteDefaultMode, + } + + if samesite, ok := validSameSiteValues[samesiteString]; ok { + CookieSameSite = samesite + cfg.CookieSameSite = CookieSameSite + } else { + CookieSameSite = http.SameSiteLaxMode + cfg.CookieSameSite = CookieSameSite + } + // read snapshots settings snapshots := iniFile.Section("snapshots") ExternalSnapshotUrl = snapshots.Key("external_snapshot_url").String() @@ -661,6 +661,20 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { // auth auth := iniFile.Section("auth") + + LoginCookieName = auth.Key("login_cookie_name").MustString("grafana_session") + cfg.LoginCookieName = LoginCookieName + cfg.LoginMaxInactiveLifetimeDays = auth.Key("login_maximum_inactive_lifetime_days").MustInt(7) + + LoginMaxLifetimeDays = auth.Key("login_maximum_lifetime_days").MustInt(30) + cfg.LoginMaxLifetimeDays = LoginMaxLifetimeDays + + cfg.TokenRotationIntervalMinutes = auth.Key("token_rotation_interval_minutes").MustInt(10) + if cfg.TokenRotationIntervalMinutes < 2 { + cfg.TokenRotationIntervalMinutes = 2 + } + cfg.ExpiredTokensCleanupIntervalDays = auth.Key("expired_tokens_cleanup_interval_days").MustInt(7) + DisableLoginForm = auth.Key("disable_login_form").MustBool(false) DisableSignoutMenu = auth.Key("disable_signout_menu").MustBool(false) OAuthAutoLogin = auth.Key("oauth_auto_login").MustBool(false) From 80d0943d9d1c0fb20ceb5236dad7ee672b6dc522 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 5 Feb 2019 21:10:56 +0100 Subject: [PATCH 05/19] document login, short-lived tokens and secure cookie configurations --- docs/sources/auth/overview.md | 32 ++++++++++++++++++++++ docs/sources/installation/configuration.md | 8 ++++++ 2 files changed, 40 insertions(+) diff --git a/docs/sources/auth/overview.md b/docs/sources/auth/overview.md index 0480ee88adc..e3d4c08ca5d 100644 --- a/docs/sources/auth/overview.md +++ b/docs/sources/auth/overview.md @@ -36,6 +36,38 @@ Grafana of course has a built in user authentication system with password authen disable authentication by enabling anonymous access. You can also hide login form and only allow login through an auth provider (listed above). There is also options for allowing self sign up. +### Login and short-lived tokens + +> The followung applies when using Grafana's built in user authentication, LDAP (without Auth proxy) or OAuth integration. + +Grafana are using short-lived tokens as a mechanism for verifying authenticated users. +These short-lived tokens are rotated each `token_rotation_interval_minutes` for an active authenticated user. + +An active authenticated user that gets it token rotated will extend the `login_maximum_inactive_lifetime_days` time from "now" that Grafana will remember the user. +This means that a user can close its browser and come back before `now + login_maximum_inactive_lifetime_days` and still being authenticated. + This is true as long as the time since user login is less than `login_maximum_lifetime_days`. + +Example: + +```bash +[auth] + +# Login cookie name +login_cookie_name = grafana_session + +# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days. +login_maximum_inactive_lifetime_days = 7 + +# The maximum lifetime (days) an autenticated user can be logged in since login time before being required to login. Default is 30 days. +login_maximum_lifetime_days = 30 + +# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. +token_rotation_interval_minutes = 10 + +# How often should expired auth tokens be deleted from the database. The default is 7 days. +expired_tokens_cleanup_interval_days = 7 +``` + ### Anonymous authentication You can make Grafana accessible without any login required by enabling anonymous access in the configuration file. diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index 46bab83654e..b4b53d7557b 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -287,6 +287,14 @@ Default is `false`. Define a white list of allowed ips/domains to use in data sources. Format: `ip_or_domain:port` separated by spaces +### cookie_secure + +Set to `true` if you host Grafana behind HTTPS. Default is `false`. + +### cookie_samesite + +Sets the `SameSite` cookie attribute and prevents the browser from sending this cookie along with cross-site requests. The main goal is mitigate the risk of cross-origin information leakage. It also provides some protection against cross-site request forgery attacks (CSRF), [read more here](https://www.owasp.org/index.php/SameSite). Valid values are `lax`, `strict` and `none`. Default is `lax`. +
## [users] From 0915f931ae6cff86ad04d3e531fd969b802d0b4d Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 5 Feb 2019 21:12:30 +0100 Subject: [PATCH 06/19] change configuration settings in auth package --- pkg/services/auth/authtoken/auth_token.go | 4 ++-- pkg/services/auth/authtoken/auth_token_test.go | 8 ++++---- pkg/services/auth/authtoken/session_cleanup.go | 4 ++-- pkg/services/auth/authtoken/session_cleanup_test.go | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/services/auth/authtoken/auth_token.go b/pkg/services/auth/authtoken/auth_token.go index 4e4bd375501..1fdad4dbea5 100644 --- a/pkg/services/auth/authtoken/auth_token.go +++ b/pkg/services/auth/authtoken/auth_token.go @@ -81,7 +81,7 @@ func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (auth.UserT s.log.Debug("looking up token", "unhashed", unhashedToken, "hashed", hashedToken) } - expireBefore := getTime().Add(time.Duration(-86400*s.Cfg.LoginCookieMaxDays) * time.Second).Unix() + expireBefore := getTime().Add(time.Duration(-86400*s.Cfg.LoginMaxInactiveLifetimeDays) * time.Second).Unix() var model userAuthToken exists, err := s.SQLStore.NewSession().Where("(auth_token = ? OR prev_auth_token = ?) AND created_at > ?", hashedToken, hashedToken, expireBefore).Get(&model) @@ -148,7 +148,7 @@ func (s *UserAuthTokenServiceImpl) TryRotateToken(token auth.UserToken, clientIP needsRotation := false rotatedAt := time.Unix(model.RotatedAt, 0) if model.AuthTokenSeen { - needsRotation = rotatedAt.Before(now.Add(-time.Duration(s.Cfg.LoginCookieRotation) * time.Minute)) + needsRotation = rotatedAt.Before(now.Add(-time.Duration(s.Cfg.TokenRotationIntervalMinutes) * time.Minute)) } else { needsRotation = rotatedAt.Before(now.Add(-urgentRotateTime)) } diff --git a/pkg/services/auth/authtoken/auth_token_test.go b/pkg/services/auth/authtoken/auth_token_test.go index 7809e235f5c..51d361a9f4f 100644 --- a/pkg/services/auth/authtoken/auth_token_test.go +++ b/pkg/services/auth/authtoken/auth_token_test.go @@ -341,10 +341,10 @@ func createTestContext(t *testing.T) *testContext { tokenService := &UserAuthTokenServiceImpl{ SQLStore: sqlstore, Cfg: &setting.Cfg{ - LoginCookieName: "grafana_session", - LoginCookieMaxDays: 7, - LoginDeleteExpiredTokensAfterDays: 30, - LoginCookieRotation: 10, + LoginMaxInactiveLifetimeDays: 7, + LoginMaxLifetimeDays: 30, + TokenRotationIntervalMinutes: 10, + ExpiredTokensCleanupIntervalDays: 1, }, log: log.New("test-logger"), } diff --git a/pkg/services/auth/authtoken/session_cleanup.go b/pkg/services/auth/authtoken/session_cleanup.go index cd2b766d6c0..ecee82767e4 100644 --- a/pkg/services/auth/authtoken/session_cleanup.go +++ b/pkg/services/auth/authtoken/session_cleanup.go @@ -7,12 +7,12 @@ import ( func (srv *UserAuthTokenServiceImpl) Run(ctx context.Context) error { ticker := time.NewTicker(time.Hour * 12) - deleteSessionAfter := time.Hour * 24 * time.Duration(srv.Cfg.LoginDeleteExpiredTokensAfterDays) + deleteSessionAfter := time.Hour * 24 * time.Duration(srv.Cfg.ExpiredTokensCleanupIntervalDays) for { select { case <-ticker.C: - srv.ServerLockService.LockAndExecute(ctx, "delete old sessions", time.Hour*12, func() { + srv.ServerLockService.LockAndExecute(ctx, "delete expired auth tokens", time.Hour*12, func() { srv.deleteOldSession(deleteSessionAfter) }) diff --git a/pkg/services/auth/authtoken/session_cleanup_test.go b/pkg/services/auth/authtoken/session_cleanup_test.go index 101a279c374..bca1aa824eb 100644 --- a/pkg/services/auth/authtoken/session_cleanup_test.go +++ b/pkg/services/auth/authtoken/session_cleanup_test.go @@ -14,7 +14,7 @@ func TestUserAuthTokenCleanup(t *testing.T) { ctx := createTestContext(t) insertToken := func(token string, prev string, rotatedAt int64) { - ut := userAuthToken{AuthToken: token, PrevAuthToken: prev, RotatedAt: rotatedAt, UserAgent: "", ClientIp: ""} + ut := userAuthToken{AuthToken: token, PrevAuthToken: prev, CreatedAt: rotatedAt, RotatedAt: rotatedAt, UserAgent: "", ClientIp: ""} _, err := ctx.sqlstore.NewSession().Insert(&ut) So(err, ShouldBeNil) } From 871c84d195417e51839db1f1fb33e47e196e18ef Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 5 Feb 2019 21:14:23 +0100 Subject: [PATCH 07/19] changes needed for api/middleware due to configuration settings --- pkg/api/login.go | 5 +++-- pkg/api/login_oauth.go | 3 ++- pkg/middleware/middleware.go | 24 +++++++++++++++--------- pkg/middleware/middleware_test.go | 16 ++++++++++++---- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/pkg/api/login.go b/pkg/api/login.go index d25e83d34e8..def24f983c1 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -137,7 +137,7 @@ func (hs *HTTPServer) loginUserWithUser(user *m.User, c *m.ReqContext) { hs.log.Error("failed to create auth token", "error", err) } - middleware.WriteSessionCookie(c, userToken.GetToken(), middleware.OneYearInSeconds) + middleware.WriteSessionCookie(c, userToken.GetToken(), hs.Cfg.LoginMaxLifetimeDays) } func (hs *HTTPServer) Logout(c *m.ReqContext) { @@ -185,7 +185,8 @@ func (hs *HTTPServer) trySetEncryptedCookie(ctx *m.ReqContext, cookieName string Value: hex.EncodeToString(encryptedError), HttpOnly: true, Path: setting.AppSubUrl + "/", - Secure: hs.Cfg.SecurityHTTPSCookies, + Secure: hs.Cfg.CookieSecure, + SameSite: hs.Cfg.CookieSameSite, }) return nil diff --git a/pkg/api/login_oauth.go b/pkg/api/login_oauth.go index 4160d48733e..87a8ecc876f 100644 --- a/pkg/api/login_oauth.go +++ b/pkg/api/login_oauth.go @@ -214,7 +214,8 @@ func (hs *HTTPServer) writeCookie(w http.ResponseWriter, name string, value stri Value: value, HttpOnly: true, Path: setting.AppSubUrl + "/", - Secure: hs.Cfg.SecurityHTTPSCookies, + Secure: hs.Cfg.CookieSecure, + SameSite: hs.Cfg.CookieSameSite, }) } diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 6cf29340b82..9a3e5e1e01c 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -4,6 +4,7 @@ import ( "net/http" "net/url" "strconv" + "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/apikeygen" @@ -168,11 +169,8 @@ func initContextWithBasicAuth(ctx *m.ReqContext, orgId int64) bool { return true } -const cookieName = "grafana_session" -const OneYearInSeconds = 31557600 //used as default maxage for session cookies. We validate/rotate them more often. - func initContextWithToken(authTokenService authtoken.UserAuthTokenService, ctx *m.ReqContext, orgID int64) bool { - rawToken := ctx.GetCookie(cookieName) + rawToken := ctx.GetCookie(setting.LoginCookieName) if rawToken == "" { return false } @@ -200,26 +198,34 @@ func initContextWithToken(authTokenService authtoken.UserAuthTokenService, ctx * } if rotated { - WriteSessionCookie(ctx, token.GetToken(), OneYearInSeconds) + WriteSessionCookie(ctx, token.GetToken(), setting.LoginMaxLifetimeDays) } return true } -func WriteSessionCookie(ctx *m.ReqContext, value string, maxAge int) { +func WriteSessionCookie(ctx *m.ReqContext, value string, maxLifetimeDays int) { if setting.Env == setting.DEV { ctx.Logger.Info("new token", "unhashed token", value) } + var maxAge int + if maxLifetimeDays <= 0 { + maxAge = -1 + } else { + maxAgeHours := (time.Duration(setting.LoginMaxLifetimeDays) * 24 * time.Hour) + time.Hour + maxAge = int(maxAgeHours.Seconds()) + } + ctx.Resp.Header().Del("Set-Cookie") cookie := http.Cookie{ - Name: cookieName, + Name: setting.LoginCookieName, Value: url.QueryEscape(value), HttpOnly: true, Path: setting.AppSubUrl + "/", - Secure: false, // TODO: use setting SecurityHTTPSCookies + Secure: setting.CookieSecure, MaxAge: maxAge, - SameSite: http.SameSiteLaxMode, // TODO: use setting LoginCookieSameSite + SameSite: setting.CookieSameSite, } http.SetCookie(ctx.Resp, &cookie) diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 4e10ee39201..fdcc56da3bf 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "path/filepath" "testing" + "time" msession "github.com/go-macaron/session" "github.com/grafana/grafana/pkg/bus" @@ -197,13 +198,17 @@ func TestMiddlewareContext(t *testing.T) { return true, nil } + maxAgeHours := (time.Duration(setting.LoginMaxLifetimeDays) * 24 * time.Hour) + maxAge := (maxAgeHours + time.Hour).Seconds() + expectedCookie := &http.Cookie{ - Name: cookieName, + Name: setting.LoginCookieName, Value: "rotated", Path: setting.AppSubUrl + "/", HttpOnly: true, - MaxAge: OneYearInSeconds, - SameSite: http.SameSiteLaxMode, + MaxAge: int(maxAge), + Secure: setting.CookieSecure, + SameSite: setting.CookieSameSite, } sc.fakeReq("GET", "/").exec() @@ -545,6 +550,9 @@ func middlewareScenario(desc string, fn scenarioFunc) { Convey(desc, func() { defer bus.ClearBusHandlers() + setting.LoginCookieName = "grafana_session" + setting.LoginMaxLifetimeDays = 30 + sc := &scenarioContext{} viewsPath, _ := filepath.Abs("../../public/views") @@ -655,7 +663,7 @@ func (sc *scenarioContext) exec() { if sc.tokenSessionCookie != "" { sc.req.AddCookie(&http.Cookie{ - Name: cookieName, + Name: setting.LoginCookieName, Value: sc.tokenSessionCookie, }) } From 948350659094f140806c829a52b23eac172ec400 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 5 Feb 2019 21:20:11 +0100 Subject: [PATCH 08/19] auth token clean up job now runs on schedule and deletes all expired tokens delete tokens having created_at <= LoginMaxLifetimeDays or rotated_at <= LoginMaxInactiveLifetimeDays --- .../auth/authtoken/session_cleanup.go | 34 ++++++++--- .../auth/authtoken/session_cleanup_test.go | 56 +++++++++++++++---- 2 files changed, 69 insertions(+), 21 deletions(-) diff --git a/pkg/services/auth/authtoken/session_cleanup.go b/pkg/services/auth/authtoken/session_cleanup.go index ecee82767e4..2b8dfb7b4e2 100644 --- a/pkg/services/auth/authtoken/session_cleanup.go +++ b/pkg/services/auth/authtoken/session_cleanup.go @@ -6,14 +6,23 @@ import ( ) func (srv *UserAuthTokenServiceImpl) Run(ctx context.Context) error { - ticker := time.NewTicker(time.Hour * 12) - deleteSessionAfter := time.Hour * 24 * time.Duration(srv.Cfg.ExpiredTokensCleanupIntervalDays) + if srv.Cfg.ExpiredTokensCleanupIntervalDays <= 0 { + srv.log.Debug("cleanup of expired auth tokens are disabled") + return nil + } + + jobInterval := time.Duration(srv.Cfg.ExpiredTokensCleanupIntervalDays) * 24 * time.Hour + srv.log.Debug("cleanup of expired auth tokens are enabled", "intervalDays", srv.Cfg.ExpiredTokensCleanupIntervalDays) + + ticker := time.NewTicker(jobInterval) + maxInactiveLifetime := time.Duration(srv.Cfg.LoginMaxInactiveLifetimeDays) * 24 * time.Hour + maxLifetime := time.Duration(srv.Cfg.LoginMaxLifetimeDays) * 24 * time.Hour for { select { case <-ticker.C: - srv.ServerLockService.LockAndExecute(ctx, "delete expired auth tokens", time.Hour*12, func() { - srv.deleteOldSession(deleteSessionAfter) + srv.ServerLockService.LockAndExecute(ctx, "cleanup expired auth tokens", time.Hour*12, func() { + srv.deleteExpiredTokens(maxInactiveLifetime, maxLifetime) }) case <-ctx.Done(): @@ -22,17 +31,24 @@ func (srv *UserAuthTokenServiceImpl) Run(ctx context.Context) error { } } -func (srv *UserAuthTokenServiceImpl) deleteOldSession(deleteSessionAfter time.Duration) (int64, error) { - sql := `DELETE from user_auth_token WHERE rotated_at < ?` +func (srv *UserAuthTokenServiceImpl) deleteExpiredTokens(maxInactiveLifetime, maxLifetime time.Duration) (int64, error) { + createdBefore := getTime().Add(-maxLifetime) + rotatedBefore := getTime().Add(-maxInactiveLifetime) - deleteBefore := getTime().Add(-deleteSessionAfter) - res, err := srv.SQLStore.NewSession().Exec(sql, deleteBefore.Unix()) + srv.log.Debug("starting cleanup of expired auth tokens", "createdBefore", createdBefore, "rotatedBefore", rotatedBefore) + + sql := `DELETE from user_auth_token WHERE created_at <= ? OR rotated_at <= ?` + res, err := srv.SQLStore.NewSession().Exec(sql, createdBefore.Unix(), rotatedBefore.Unix()) if err != nil { return 0, err } affected, err := res.RowsAffected() - srv.log.Info("deleted old sessions", "count", affected) + if err != nil { + srv.log.Error("failed to cleanup expired auth tokens", "error", err) + return 0, nil + } + srv.log.Info("cleanup of expired auth tokens done", "count", affected) return affected, err } diff --git a/pkg/services/auth/authtoken/session_cleanup_test.go b/pkg/services/auth/authtoken/session_cleanup_test.go index bca1aa824eb..7b611b3263c 100644 --- a/pkg/services/auth/authtoken/session_cleanup_test.go +++ b/pkg/services/auth/authtoken/session_cleanup_test.go @@ -12,25 +12,57 @@ func TestUserAuthTokenCleanup(t *testing.T) { Convey("Test user auth token cleanup", t, func() { ctx := createTestContext(t) + ctx.tokenService.Cfg.LoginMaxInactiveLifetimeDays = 7 + ctx.tokenService.Cfg.LoginMaxLifetimeDays = 30 - insertToken := func(token string, prev string, rotatedAt int64) { - ut := userAuthToken{AuthToken: token, PrevAuthToken: prev, CreatedAt: rotatedAt, RotatedAt: rotatedAt, UserAgent: "", ClientIp: ""} + insertToken := func(token string, prev string, createdAt, rotatedAt int64) { + ut := userAuthToken{AuthToken: token, PrevAuthToken: prev, CreatedAt: createdAt, RotatedAt: rotatedAt, UserAgent: "", ClientIp: ""} _, err := ctx.sqlstore.NewSession().Insert(&ut) So(err, ShouldBeNil) } - // insert three old tokens that should be deleted - for i := 0; i < 3; i++ { - insertToken(fmt.Sprintf("oldA%d", i), fmt.Sprintf("oldB%d", i), int64(i)) + t := time.Date(2018, 12, 13, 13, 45, 0, 0, time.UTC) + getTime = func() time.Time { + return t } - // insert three active tokens that should not be deleted - for i := 0; i < 3; i++ { - insertToken(fmt.Sprintf("newA%d", i), fmt.Sprintf("newB%d", i), getTime().Unix()) - } + Convey("should delete tokens where token rotation age is older than or equal 7 days", func() { + from := t.Add(-7 * 24 * time.Hour) - affected, err := ctx.tokenService.deleteOldSession(time.Hour) - So(err, ShouldBeNil) - So(affected, ShouldEqual, 3) + // insert three old tokens that should be deleted + for i := 0; i < 3; i++ { + insertToken(fmt.Sprintf("oldA%d", i), fmt.Sprintf("oldB%d", i), from.Unix(), from.Unix()) + } + + // insert three active tokens that should not be deleted + for i := 0; i < 3; i++ { + from = from.Add(time.Second) + insertToken(fmt.Sprintf("newA%d", i), fmt.Sprintf("newB%d", i), from.Unix(), from.Unix()) + } + + affected, err := ctx.tokenService.deleteExpiredTokens(7*24*time.Hour, 30*24*time.Hour) + So(err, ShouldBeNil) + So(affected, ShouldEqual, 3) + }) + + Convey("should delete tokens where token age is older than or equal 30 days", func() { + from := t.Add(-30 * 24 * time.Hour) + fromRotate := t.Add(-time.Second) + + // insert three old tokens that should be deleted + for i := 0; i < 3; i++ { + insertToken(fmt.Sprintf("oldA%d", i), fmt.Sprintf("oldB%d", i), from.Unix(), fromRotate.Unix()) + } + + // insert three active tokens that should not be deleted + for i := 0; i < 3; i++ { + from = from.Add(time.Second) + insertToken(fmt.Sprintf("newA%d", i), fmt.Sprintf("newB%d", i), from.Unix(), fromRotate.Unix()) + } + + affected, err := ctx.tokenService.deleteExpiredTokens(7*24*time.Hour, 30*24*time.Hour) + So(err, ShouldBeNil) + So(affected, ShouldEqual, 3) + }) }) } From d8658a765c568d62cfeb3e5bac6d2a55969c9e65 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 08:30:14 +0100 Subject: [PATCH 09/19] enhanced expiration logic for lookup token tokens are not expired if created_at > now - LoginMaxLifetimeDays and rotated_at > now - LoginMaxInactiveLifetimeDays --- pkg/services/auth/authtoken/auth_token.go | 7 +- .../auth/authtoken/auth_token_test.go | 70 +++++++++++++++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/pkg/services/auth/authtoken/auth_token.go b/pkg/services/auth/authtoken/auth_token.go index 1fdad4dbea5..47aa925fd4d 100644 --- a/pkg/services/auth/authtoken/auth_token.go +++ b/pkg/services/auth/authtoken/auth_token.go @@ -81,10 +81,13 @@ func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (auth.UserT s.log.Debug("looking up token", "unhashed", unhashedToken, "hashed", hashedToken) } - expireBefore := getTime().Add(time.Duration(-86400*s.Cfg.LoginMaxInactiveLifetimeDays) * time.Second).Unix() + tokenMaxLifetime := time.Duration(s.Cfg.LoginMaxLifetimeDays) * 24 * time.Hour + tokenMaxInactiveLifetime := time.Duration(s.Cfg.LoginMaxInactiveLifetimeDays) * 24 * time.Hour + createdAfter := getTime().Add(-tokenMaxLifetime).Unix() + rotatedAfter := getTime().Add(-tokenMaxInactiveLifetime).Unix() var model userAuthToken - exists, err := s.SQLStore.NewSession().Where("(auth_token = ? OR prev_auth_token = ?) AND created_at > ?", hashedToken, hashedToken, expireBefore).Get(&model) + exists, err := s.SQLStore.NewSession().Where("(auth_token = ? OR prev_auth_token = ?) AND created_at > ? AND rotated_at > ?", hashedToken, hashedToken, createdAfter, rotatedAfter).Get(&model) if err != nil { return nil, err } diff --git a/pkg/services/auth/authtoken/auth_token_test.go b/pkg/services/auth/authtoken/auth_token_test.go index 51d361a9f4f..7ecb67b2ebf 100644 --- a/pkg/services/auth/authtoken/auth_token_test.go +++ b/pkg/services/auth/authtoken/auth_token_test.go @@ -105,12 +105,56 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) So(stillGood, ShouldNotBeNil) - getTime = func() time.Time { - return t.Add(24 * 7 * time.Hour) - } - notGood, err := userAuthTokenService.LookupToken(model.UnhashedToken) - So(err, ShouldEqual, ErrAuthTokenNotFound) - So(notGood, ShouldBeNil) + model, err = ctx.getAuthTokenByID(model.Id) + So(err, ShouldBeNil) + + Convey("when rotated_at is 6:59:59 ago should find token", func() { + getTime = func() time.Time { + return time.Unix(model.RotatedAt, 0).Add(24 * 7 * time.Hour).Add(-time.Second) + } + + stillGood, err = userAuthTokenService.LookupToken(stillGood.GetToken()) + So(err, ShouldBeNil) + So(stillGood, ShouldNotBeNil) + }) + + Convey("when rotated_at is 7:00:00 ago should not find token", func() { + getTime = func() time.Time { + return time.Unix(model.RotatedAt, 0).Add(24 * 7 * time.Hour) + } + + notGood, err := userAuthTokenService.LookupToken(userToken.GetToken()) + So(err, ShouldEqual, ErrAuthTokenNotFound) + So(notGood, ShouldBeNil) + }) + + Convey("when rotated_at is 5 days ago and created_at is 29 days and 23:59:59 ago should not find token", func() { + updated, err := ctx.updateRotatedAt(model.Id, time.Unix(model.CreatedAt, 0).Add(24*25*time.Hour).Unix()) + So(err, ShouldBeNil) + So(updated, ShouldBeTrue) + + getTime = func() time.Time { + return time.Unix(model.CreatedAt, 0).Add(24 * 30 * time.Hour).Add(-time.Second) + } + + stillGood, err = userAuthTokenService.LookupToken(stillGood.GetToken()) + So(err, ShouldBeNil) + So(stillGood, ShouldNotBeNil) + }) + + Convey("when rotated_at is 5 days ago and created_at is 30 days ago should not find token", func() { + updated, err := ctx.updateRotatedAt(model.Id, time.Unix(model.CreatedAt, 0).Add(24*25*time.Hour).Unix()) + So(err, ShouldBeNil) + So(updated, ShouldBeTrue) + + getTime = func() time.Time { + return time.Unix(model.CreatedAt, 0).Add(24 * 30 * time.Hour) + } + + notGood, err := userAuthTokenService.LookupToken(userToken.GetToken()) + So(err, ShouldEqual, ErrAuthTokenNotFound) + So(notGood, ShouldBeNil) + }) }) Convey("can properly rotate tokens", func() { @@ -384,3 +428,17 @@ func (c *testContext) markAuthTokenAsSeen(id int64) (bool, error) { } return rowsAffected == 1, nil } + +func (c *testContext) updateRotatedAt(id, rotatedAt int64) (bool, error) { + sess := c.sqlstore.NewSession() + res, err := sess.Exec("UPDATE user_auth_token SET rotated_at = ? WHERE id = ?", rotatedAt, id) + if err != nil { + return false, err + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return false, err + } + return rowsAffected == 1, nil +} From 44275d9660feabcb42ca41db2b6866b16314c340 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 08:45:01 +0100 Subject: [PATCH 10/19] middleware fix --- pkg/middleware/middleware.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 9a3e5e1e01c..817372292b9 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -178,6 +178,7 @@ func initContextWithToken(authTokenService authtoken.UserAuthTokenService, ctx * token, err := authTokenService.LookupToken(rawToken) if err != nil { ctx.Logger.Error("failed to look up user based on cookie", "error", err) + WriteSessionCookie(ctx, "", -1) return false } From 85ef2ca738c8e976d0a387728429934637841012 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 09:43:45 +0100 Subject: [PATCH 11/19] fix spelling --- conf/defaults.ini | 2 +- conf/sample.ini | 2 +- docs/sources/auth/overview.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index c65cb93d426..41b948e53af 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -250,7 +250,7 @@ login_cookie_name = grafana_session # The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days. login_maximum_inactive_lifetime_days = 7 -# The maximum lifetime (days) an autenticated user can be logged in since login time before being required to login. Default is 30 days. +# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days. login_maximum_lifetime_days = 30 # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. diff --git a/conf/sample.ini b/conf/sample.ini index 39feb31441e..831fa31253e 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -230,7 +230,7 @@ log_queries = # The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days, ;login_maximum_inactive_lifetime_days = 7 -# The maximum lifetime (days) an autenticated user can be logged in since login time before being required to login. Default is 30 days. +# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days. ;login_maximum_lifetime_days = 30 # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. diff --git a/docs/sources/auth/overview.md b/docs/sources/auth/overview.md index e3d4c08ca5d..fba8da00a5e 100644 --- a/docs/sources/auth/overview.md +++ b/docs/sources/auth/overview.md @@ -58,7 +58,7 @@ login_cookie_name = grafana_session # The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days. login_maximum_inactive_lifetime_days = 7 -# The maximum lifetime (days) an autenticated user can be logged in since login time before being required to login. Default is 30 days. +# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days. login_maximum_lifetime_days = 30 # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. From a60124a88cef42bb301a261a34469eeb930475a2 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 16:21:16 +0100 Subject: [PATCH 12/19] change UserToken from interface to struct --- pkg/api/login.go | 2 +- pkg/middleware/middleware.go | 6 +- pkg/middleware/middleware_test.go | 88 +++----- pkg/middleware/org_redirect_test.go | 16 +- pkg/middleware/quota_test.go | 8 +- pkg/models/context.go | 2 +- pkg/services/auth/auth.go | 16 +- pkg/services/auth/authtoken/auth_token.go | 30 +-- .../auth/authtoken/auth_token_test.go | 213 ++++++++++-------- pkg/services/auth/authtoken/model.go | 78 +++---- 10 files changed, 244 insertions(+), 215 deletions(-) diff --git a/pkg/api/login.go b/pkg/api/login.go index def24f983c1..48f8f237884 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -137,7 +137,7 @@ func (hs *HTTPServer) loginUserWithUser(user *m.User, c *m.ReqContext) { hs.log.Error("failed to create auth token", "error", err) } - middleware.WriteSessionCookie(c, userToken.GetToken(), hs.Cfg.LoginMaxLifetimeDays) + middleware.WriteSessionCookie(c, userToken.UnhashedToken, hs.Cfg.LoginMaxLifetimeDays) } func (hs *HTTPServer) Logout(c *m.ReqContext) { diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 817372292b9..762a12d09d2 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -182,9 +182,9 @@ func initContextWithToken(authTokenService authtoken.UserAuthTokenService, ctx * return false } - query := m.GetSignedInUserQuery{UserId: token.GetUserId(), OrgId: orgID} + query := m.GetSignedInUserQuery{UserId: token.UserId, OrgId: orgID} if err := bus.Dispatch(&query); err != nil { - ctx.Logger.Error("failed to get user with id", "userId", token.GetUserId(), "error", err) + ctx.Logger.Error("failed to get user with id", "userId", token.UserId, "error", err) return false } @@ -199,7 +199,7 @@ func initContextWithToken(authTokenService authtoken.UserAuthTokenService, ctx * } if rotated { - WriteSessionCookie(ctx, token.GetToken(), setting.LoginMaxLifetimeDays) + WriteSessionCookie(ctx, token.UnhashedToken, setting.LoginMaxLifetimeDays) } return true diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index fdcc56da3bf..5852f9d0bcd 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -157,10 +157,10 @@ func TestMiddlewareContext(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 12, - token: unhashedToken, + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { + return &auth.UserToken{ + UserId: 12, + UnhashedToken: unhashedToken, }, nil } @@ -169,8 +169,8 @@ func TestMiddlewareContext(t *testing.T) { Convey("should init context with user info", func() { So(sc.context.IsSignedIn, ShouldBeTrue) So(sc.context.UserId, ShouldEqual, 12) - So(sc.context.UserToken.GetUserId(), ShouldEqual, 12) - So(sc.context.UserToken.GetToken(), ShouldEqual, "token") + So(sc.context.UserToken.UserId, ShouldEqual, 12) + So(sc.context.UserToken.UnhashedToken, ShouldEqual, "token") }) Convey("should not set cookie", func() { @@ -186,15 +186,15 @@ func TestMiddlewareContext(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 12, - token: unhashedToken, + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { + return &auth.UserToken{ + UserId: 12, + UnhashedToken: "", }, nil } - sc.userAuthTokenService.tryRotateTokenProvider = func(userToken auth.UserToken, clientIP, userAgent string) (bool, error) { - userToken.(fakeUserToken).SetToken("rotated") + sc.userAuthTokenService.tryRotateTokenProvider = func(userToken *auth.UserToken, clientIP, userAgent string) (bool, error) { + userToken.UnhashedToken = "rotated" return true, nil } @@ -216,8 +216,8 @@ func TestMiddlewareContext(t *testing.T) { Convey("should init context with user info", func() { So(sc.context.IsSignedIn, ShouldBeTrue) So(sc.context.UserId, ShouldEqual, 12) - So(sc.context.UserToken.GetUserId(), ShouldEqual, 12) - So(sc.context.UserToken.GetToken(), ShouldEqual, "rotated") + So(sc.context.UserToken.UserId, ShouldEqual, 12) + So(sc.context.UserToken.UnhashedToken, ShouldEqual, "rotated") }) Convey("should set cookie", func() { @@ -228,7 +228,7 @@ func TestMiddlewareContext(t *testing.T) { middlewareScenario("Invalid/expired auth token in cookie", func(sc *scenarioContext) { sc.withTokenSessionCookie("token") - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { return nil, authtoken.ErrAuthTokenNotFound } @@ -679,70 +679,48 @@ func (sc *scenarioContext) exec() { type scenarioFunc func(c *scenarioContext) type handlerFunc func(c *m.ReqContext) -type fakeUserToken interface { - auth.UserToken - SetToken(token string) -} - -type userTokenImpl struct { - userId int64 - token string -} - -func (ut *userTokenImpl) GetUserId() int64 { - return ut.userId -} - -func (ut *userTokenImpl) GetToken() string { - return ut.token -} - -func (ut *userTokenImpl) SetToken(token string) { - ut.token = token -} - type fakeUserAuthTokenService struct { - createTokenProvider func(userId int64, clientIP, userAgent string) (auth.UserToken, error) - tryRotateTokenProvider func(token auth.UserToken, clientIP, userAgent string) (bool, error) - lookupTokenProvider func(unhashedToken string) (auth.UserToken, error) - revokeTokenProvider func(token auth.UserToken) error + createTokenProvider func(userId int64, clientIP, userAgent string) (*auth.UserToken, error) + tryRotateTokenProvider func(token *auth.UserToken, clientIP, userAgent string) (bool, error) + lookupTokenProvider func(unhashedToken string) (*auth.UserToken, error) + revokeTokenProvider func(token *auth.UserToken) error } func newFakeUserAuthTokenService() *fakeUserAuthTokenService { return &fakeUserAuthTokenService{ - createTokenProvider: func(userId int64, clientIP, userAgent string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 0, - token: "", + createTokenProvider: func(userId int64, clientIP, userAgent string) (*auth.UserToken, error) { + return &auth.UserToken{ + UserId: 0, + UnhashedToken: "", }, nil }, - tryRotateTokenProvider: func(token auth.UserToken, clientIP, userAgent string) (bool, error) { + tryRotateTokenProvider: func(token *auth.UserToken, clientIP, userAgent string) (bool, error) { return false, nil }, - lookupTokenProvider: func(unhashedToken string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 0, - token: "", + lookupTokenProvider: func(unhashedToken string) (*auth.UserToken, error) { + return &auth.UserToken{ + UserId: 0, + UnhashedToken: "", }, nil }, - revokeTokenProvider: func(token auth.UserToken) error { + revokeTokenProvider: func(token *auth.UserToken) error { return nil }, } } -func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) { +func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*auth.UserToken, error) { return s.createTokenProvider(userId, clientIP, userAgent) } -func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (auth.UserToken, error) { +func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (*auth.UserToken, error) { return s.lookupTokenProvider(unhashedToken) } -func (s *fakeUserAuthTokenService) TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) { +func (s *fakeUserAuthTokenService) TryRotateToken(token *auth.UserToken, clientIP, userAgent string) (bool, error) { return s.tryRotateTokenProvider(token, clientIP, userAgent) } -func (s *fakeUserAuthTokenService) RevokeToken(token auth.UserToken) error { +func (s *fakeUserAuthTokenService) RevokeToken(token *auth.UserToken) error { return s.revokeTokenProvider(token) } diff --git a/pkg/middleware/org_redirect_test.go b/pkg/middleware/org_redirect_test.go index c7479b3e9bc..0cc88016f3a 100644 --- a/pkg/middleware/org_redirect_test.go +++ b/pkg/middleware/org_redirect_test.go @@ -26,10 +26,10 @@ func TestOrgRedirectMiddleware(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 12, - token: "", + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { + return &auth.UserToken{ + UserId: 0, + UnhashedToken: "", }, nil } @@ -52,10 +52,10 @@ func TestOrgRedirectMiddleware(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 12, - token: "", + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { + return &auth.UserToken{ + UserId: 12, + UnhashedToken: "", }, nil } diff --git a/pkg/middleware/quota_test.go b/pkg/middleware/quota_test.go index af22f41deba..df2bf930426 100644 --- a/pkg/middleware/quota_test.go +++ b/pkg/middleware/quota_test.go @@ -81,10 +81,10 @@ func TestMiddlewareQuota(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (auth.UserToken, error) { - return &userTokenImpl{ - userId: 12, - token: "", + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { + return &auth.UserToken{ + UserId: 12, + UnhashedToken: "", }, nil } diff --git a/pkg/models/context.go b/pkg/models/context.go index da63db63f45..bba0024c0f2 100644 --- a/pkg/models/context.go +++ b/pkg/models/context.go @@ -14,7 +14,7 @@ import ( type ReqContext struct { *macaron.Context *SignedInUser - UserToken auth.UserToken + UserToken *auth.UserToken // This should only be used by the auth_proxy Session session.SessionStore diff --git a/pkg/services/auth/auth.go b/pkg/services/auth/auth.go index 31316f473f5..727ceffff79 100644 --- a/pkg/services/auth/auth.go +++ b/pkg/services/auth/auth.go @@ -1,6 +1,16 @@ package auth -type UserToken interface { - GetUserId() int64 - GetToken() string +type UserToken struct { + Id int64 + UserId int64 + AuthToken string + PrevAuthToken string + UserAgent string + ClientIp string + AuthTokenSeen bool + SeenAt int64 + RotatedAt int64 + CreatedAt int64 + UpdatedAt int64 + UnhashedToken string } diff --git a/pkg/services/auth/authtoken/auth_token.go b/pkg/services/auth/authtoken/auth_token.go index 47aa925fd4d..a993a37dbf3 100644 --- a/pkg/services/auth/authtoken/auth_token.go +++ b/pkg/services/auth/authtoken/auth_token.go @@ -40,7 +40,7 @@ func (s *UserAuthTokenServiceImpl) Init() error { return nil } -func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) { +func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent string) (*auth.UserToken, error) { clientIP = util.ParseIPAddress(clientIP) token, err := util.RandomHex(16) if err != nil { @@ -72,10 +72,13 @@ func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent s.log.Debug("user auth token created", "tokenId", userAuthToken.Id, "userId", userAuthToken.UserId, "clientIP", userAuthToken.ClientIp, "userAgent", userAuthToken.UserAgent, "authToken", userAuthToken.AuthToken) - return userAuthToken.toUserToken() + var userToken auth.UserToken + err = userAuthToken.toUserToken(&userToken) + + return &userToken, err } -func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (auth.UserToken, error) { +func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*auth.UserToken, error) { hashedToken := hashToken(unhashedToken) if setting.Env == setting.DEV { s.log.Debug("looking up token", "unhashed", unhashedToken, "hashed", hashedToken) @@ -133,18 +136,19 @@ func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (auth.UserT } model.UnhashedToken = unhashedToken - return model.toUserToken() + + var userToken auth.UserToken + err = model.toUserToken(&userToken) + + return &userToken, err } -func (s *UserAuthTokenServiceImpl) TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) { +func (s *UserAuthTokenServiceImpl) TryRotateToken(token *auth.UserToken, clientIP, userAgent string) (bool, error) { if token == nil { return false, nil } - model, err := extractModelFromToken(token) - if err != nil { - return false, err - } + model := userAuthTokenFromUserToken(token) now := getTime() @@ -191,21 +195,19 @@ func (s *UserAuthTokenServiceImpl) TryRotateToken(token auth.UserToken, clientIP s.log.Debug("auth token rotated", "affected", affected, "auth_token_id", model.Id, "userId", model.UserId) if affected > 0 { model.UnhashedToken = newToken + model.toUserToken(token) return true, nil } return false, nil } -func (s *UserAuthTokenServiceImpl) RevokeToken(token auth.UserToken) error { +func (s *UserAuthTokenServiceImpl) RevokeToken(token *auth.UserToken) error { if token == nil { return ErrAuthTokenNotFound } - model, err := extractModelFromToken(token) - if err != nil { - return err - } + model := userAuthTokenFromUserToken(token) rowsAffected, err := s.SQLStore.NewSession().Delete(model) if err != nil { diff --git a/pkg/services/auth/authtoken/auth_token_test.go b/pkg/services/auth/authtoken/auth_token_test.go index 7ecb67b2ebf..a1b70db60fb 100644 --- a/pkg/services/auth/authtoken/auth_token_test.go +++ b/pkg/services/auth/authtoken/auth_token_test.go @@ -1,12 +1,15 @@ package authtoken import ( + "encoding/json" "testing" "time" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/sqlstore" . "github.com/smartystreets/goconvey/convey" ) @@ -25,28 +28,24 @@ func TestUserAuthToken(t *testing.T) { Convey("When creating token", func() { userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") So(err, ShouldBeNil) - model, err := extractModelFromToken(userToken) - So(err, ShouldBeNil) - So(model, ShouldNotBeNil) - So(model.AuthTokenSeen, ShouldBeFalse) + So(userToken, ShouldNotBeNil) + So(userToken.AuthTokenSeen, ShouldBeFalse) Convey("When lookup unhashed token should return user auth token", func() { - userToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + userToken, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) - lookedUpModel, err := extractModelFromToken(userToken) - So(err, ShouldBeNil) - So(lookedUpModel, ShouldNotBeNil) - So(lookedUpModel.UserId, ShouldEqual, userID) - So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + So(userToken, ShouldNotBeNil) + So(userToken.UserId, ShouldEqual, userID) + So(userToken.AuthTokenSeen, ShouldBeTrue) - storedAuthToken, err := ctx.getAuthTokenByID(lookedUpModel.Id) + storedAuthToken, err := ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) So(storedAuthToken, ShouldNotBeNil) So(storedAuthToken.AuthTokenSeen, ShouldBeTrue) }) Convey("When lookup hashed token should return user auth token not found error", func() { - userToken, err := userAuthTokenService.LookupToken(model.AuthToken) + userToken, err := userAuthTokenService.LookupToken(userToken.AuthToken) So(err, ShouldEqual, ErrAuthTokenNotFound) So(userToken, ShouldBeNil) }) @@ -55,7 +54,7 @@ func TestUserAuthToken(t *testing.T) { err = userAuthTokenService.RevokeToken(userToken) So(err, ShouldBeNil) - model, err := ctx.getAuthTokenByID(model.Id) + model, err := ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) So(model, ShouldBeNil) }) @@ -66,10 +65,8 @@ func TestUserAuthToken(t *testing.T) { }) Convey("revoking non-existing token should return error", func() { - model.Id = 1000 - nonExistingToken, err := model.toUserToken() - So(err, ShouldBeNil) - err = userAuthTokenService.RevokeToken(nonExistingToken) + userToken.Id = 1000 + err = userAuthTokenService.RevokeToken(userToken) So(err, ShouldEqual, ErrAuthTokenNotFound) }) }) @@ -77,17 +74,8 @@ func TestUserAuthToken(t *testing.T) { Convey("expires correctly", func() { userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") So(err, ShouldBeNil) - model, err := extractModelFromToken(userToken) - So(err, ShouldBeNil) - So(model, ShouldNotBeNil) - _, err = userAuthTokenService.LookupToken(model.UnhashedToken) - So(err, ShouldBeNil) - - model, err = ctx.getAuthTokenByID(model.Id) - So(err, ShouldBeNil) - - userToken, err = model.toUserToken() + userToken, err = userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) getTime = func() time.Time { @@ -98,14 +86,14 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) So(rotated, ShouldBeTrue) - _, err = userAuthTokenService.LookupToken(model.UnhashedToken) + userToken, err = userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) - stillGood, err := userAuthTokenService.LookupToken(model.UnhashedToken) + stillGood, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) So(stillGood, ShouldNotBeNil) - model, err = ctx.getAuthTokenByID(model.Id) + model, err := ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) Convey("when rotated_at is 6:59:59 ago should find token", func() { @@ -113,7 +101,7 @@ func TestUserAuthToken(t *testing.T) { return time.Unix(model.RotatedAt, 0).Add(24 * 7 * time.Hour).Add(-time.Second) } - stillGood, err = userAuthTokenService.LookupToken(stillGood.GetToken()) + stillGood, err = userAuthTokenService.LookupToken(stillGood.UnhashedToken) So(err, ShouldBeNil) So(stillGood, ShouldNotBeNil) }) @@ -123,7 +111,7 @@ func TestUserAuthToken(t *testing.T) { return time.Unix(model.RotatedAt, 0).Add(24 * 7 * time.Hour) } - notGood, err := userAuthTokenService.LookupToken(userToken.GetToken()) + notGood, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldEqual, ErrAuthTokenNotFound) So(notGood, ShouldBeNil) }) @@ -137,7 +125,7 @@ func TestUserAuthToken(t *testing.T) { return time.Unix(model.CreatedAt, 0).Add(24 * 30 * time.Hour).Add(-time.Second) } - stillGood, err = userAuthTokenService.LookupToken(stillGood.GetToken()) + stillGood, err = userAuthTokenService.LookupToken(stillGood.UnhashedToken) So(err, ShouldBeNil) So(stillGood, ShouldNotBeNil) }) @@ -151,7 +139,7 @@ func TestUserAuthToken(t *testing.T) { return time.Unix(model.CreatedAt, 0).Add(24 * 30 * time.Hour) } - notGood, err := userAuthTokenService.LookupToken(userToken.GetToken()) + notGood, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldEqual, ErrAuthTokenNotFound) So(notGood, ShouldBeNil) }) @@ -160,37 +148,35 @@ func TestUserAuthToken(t *testing.T) { Convey("can properly rotate tokens", func() { userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") So(err, ShouldBeNil) - model, err := extractModelFromToken(userToken) - So(err, ShouldBeNil) - So(model, ShouldNotBeNil) - prevToken := model.AuthToken - unhashedPrev := model.UnhashedToken + prevToken := userToken.AuthToken + unhashedPrev := userToken.UnhashedToken rotated, err := userAuthTokenService.TryRotateToken(userToken, "192.168.10.12:1234", "a new user agent") So(err, ShouldBeNil) So(rotated, ShouldBeFalse) - updated, err := ctx.markAuthTokenAsSeen(model.Id) + updated, err := ctx.markAuthTokenAsSeen(userToken.Id) So(err, ShouldBeNil) So(updated, ShouldBeTrue) - model, err = ctx.getAuthTokenByID(model.Id) - So(err, ShouldBeNil) - tok, err := model.toUserToken() + model, err := ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) + var tok auth.UserToken + model.toUserToken(&tok) + getTime = func() time.Time { return t.Add(time.Hour) } - rotated, err = userAuthTokenService.TryRotateToken(tok, "192.168.10.12:1234", "a new user agent") + rotated, err = userAuthTokenService.TryRotateToken(&tok, "192.168.10.12:1234", "a new user agent") So(err, ShouldBeNil) So(rotated, ShouldBeTrue) - unhashedToken := model.UnhashedToken + unhashedToken := tok.UnhashedToken - model, err = ctx.getAuthTokenByID(model.Id) + model, err = ctx.getAuthTokenByID(tok.Id) So(err, ShouldBeNil) model.UnhashedToken = unhashedToken @@ -205,17 +191,15 @@ func TestUserAuthToken(t *testing.T) { lookedUpUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) So(err, ShouldBeNil) - lookedUpModel, err := extractModelFromToken(lookedUpUserToken) - So(err, ShouldBeNil) - So(lookedUpModel, ShouldNotBeNil) - So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) - So(lookedUpModel.SeenAt, ShouldEqual, getTime().Unix()) + So(lookedUpUserToken, ShouldNotBeNil) + So(lookedUpUserToken.AuthTokenSeen, ShouldBeTrue) + So(lookedUpUserToken.SeenAt, ShouldEqual, getTime().Unix()) lookedUpUserToken, err = userAuthTokenService.LookupToken(unhashedPrev) So(err, ShouldBeNil) - So(lookedUpModel, ShouldNotBeNil) - So(lookedUpModel.Id, ShouldEqual, model.Id) - So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + So(lookedUpUserToken, ShouldNotBeNil) + So(lookedUpUserToken.Id, ShouldEqual, model.Id) + So(lookedUpUserToken.AuthTokenSeen, ShouldBeTrue) getTime = func() time.Time { return t.Add(time.Hour + (2 * time.Minute)) @@ -223,12 +207,10 @@ func TestUserAuthToken(t *testing.T) { lookedUpUserToken, err = userAuthTokenService.LookupToken(unhashedPrev) So(err, ShouldBeNil) - lookedUpModel, err = extractModelFromToken(lookedUpUserToken) - So(err, ShouldBeNil) - So(lookedUpModel, ShouldNotBeNil) - So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) + So(lookedUpUserToken, ShouldNotBeNil) + So(lookedUpUserToken.AuthTokenSeen, ShouldBeTrue) - lookedUpModel, err = ctx.getAuthTokenByID(lookedUpModel.Id) + lookedUpModel, err := ctx.getAuthTokenByID(lookedUpUserToken.Id) So(err, ShouldBeNil) So(lookedUpModel, ShouldNotBeNil) So(lookedUpModel.AuthTokenSeen, ShouldBeFalse) @@ -237,7 +219,7 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) So(rotated, ShouldBeTrue) - model, err = ctx.getAuthTokenByID(model.Id) + model, err = ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) So(model, ShouldNotBeNil) So(model.SeenAt, ShouldEqual, 0) @@ -246,11 +228,9 @@ func TestUserAuthToken(t *testing.T) { Convey("keeps prev token valid for 1 minute after it is confirmed", func() { userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") So(err, ShouldBeNil) - model, err := extractModelFromToken(userToken) - So(err, ShouldBeNil) - So(model, ShouldNotBeNil) + So(userToken, ShouldNotBeNil) - lookedUpUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + lookedUpUserToken, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) So(lookedUpUserToken, ShouldNotBeNil) @@ -258,7 +238,7 @@ func TestUserAuthToken(t *testing.T) { return t.Add(10 * time.Minute) } - prevToken := model.UnhashedToken + prevToken := userToken.UnhashedToken rotated, err := userAuthTokenService.TryRotateToken(userToken, "1.1.1.1", "firefox") So(err, ShouldBeNil) So(rotated, ShouldBeTrue) @@ -267,7 +247,7 @@ func TestUserAuthToken(t *testing.T) { return t.Add(20 * time.Minute) } - currentUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + currentUserToken, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) So(currentUserToken, ShouldNotBeNil) @@ -279,23 +259,17 @@ func TestUserAuthToken(t *testing.T) { Convey("will not mark token unseen when prev and current are the same", func() { userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") So(err, ShouldBeNil) - model, err := extractModelFromToken(userToken) - So(err, ShouldBeNil) - So(model, ShouldNotBeNil) + So(userToken, ShouldNotBeNil) - lookedUpUserToken, err := userAuthTokenService.LookupToken(model.UnhashedToken) + lookedUpUserToken, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) - lookedUpModel, err := extractModelFromToken(lookedUpUserToken) - So(err, ShouldBeNil) - So(lookedUpModel, ShouldNotBeNil) + So(lookedUpUserToken, ShouldNotBeNil) - lookedUpUserToken, err = userAuthTokenService.LookupToken(model.UnhashedToken) + lookedUpUserToken, err = userAuthTokenService.LookupToken(userToken.UnhashedToken) So(err, ShouldBeNil) - lookedUpModel, err = extractModelFromToken(lookedUpUserToken) - So(err, ShouldBeNil) - So(lookedUpModel, ShouldNotBeNil) + So(lookedUpUserToken, ShouldNotBeNil) - lookedUpModel, err = ctx.getAuthTokenByID(lookedUpModel.Id) + lookedUpModel, err := ctx.getAuthTokenByID(lookedUpUserToken.Id) So(err, ShouldBeNil) So(lookedUpModel, ShouldNotBeNil) So(lookedUpModel.AuthTokenSeen, ShouldBeTrue) @@ -304,14 +278,12 @@ func TestUserAuthToken(t *testing.T) { Convey("Rotate token", func() { userToken, err := userAuthTokenService.CreateToken(userID, "192.168.10.11:1234", "some user agent") So(err, ShouldBeNil) - model, err := extractModelFromToken(userToken) - So(err, ShouldBeNil) - So(model, ShouldNotBeNil) + So(userToken, ShouldNotBeNil) - prevToken := model.AuthToken + prevToken := userToken.AuthToken Convey("Should rotate current token and previous token when auth token seen", func() { - updated, err := ctx.markAuthTokenAsSeen(model.Id) + updated, err := ctx.markAuthTokenAsSeen(userToken.Id) So(err, ShouldBeNil) So(updated, ShouldBeTrue) @@ -323,7 +295,7 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) So(rotated, ShouldBeTrue) - storedToken, err := ctx.getAuthTokenByID(model.Id) + storedToken, err := ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) So(storedToken, ShouldNotBeNil) So(storedToken.AuthTokenSeen, ShouldBeFalse) @@ -332,7 +304,7 @@ func TestUserAuthToken(t *testing.T) { prevToken = storedToken.AuthToken - updated, err = ctx.markAuthTokenAsSeen(model.Id) + updated, err = ctx.markAuthTokenAsSeen(userToken.Id) So(err, ShouldBeNil) So(updated, ShouldBeTrue) @@ -344,7 +316,7 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) So(rotated, ShouldBeTrue) - storedToken, err = ctx.getAuthTokenByID(model.Id) + storedToken, err = ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) So(storedToken, ShouldNotBeNil) So(storedToken.AuthTokenSeen, ShouldBeFalse) @@ -353,7 +325,7 @@ func TestUserAuthToken(t *testing.T) { }) Convey("Should rotate current token, but keep previous token when auth token not seen", func() { - model.RotatedAt = getTime().Add(-2 * time.Minute).Unix() + userToken.RotatedAt = getTime().Add(-2 * time.Minute).Unix() getTime = func() time.Time { return t.Add(2 * time.Minute) @@ -363,7 +335,7 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) So(rotated, ShouldBeTrue) - storedToken, err := ctx.getAuthTokenByID(model.Id) + storedToken, err := ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) So(storedToken, ShouldNotBeNil) So(storedToken.AuthTokenSeen, ShouldBeFalse) @@ -372,6 +344,71 @@ func TestUserAuthToken(t *testing.T) { }) }) + Convey("When populating userAuthToken from UserToken should copy all properties", func() { + ut := auth.UserToken{ + Id: 1, + UserId: 2, + AuthToken: "a", + PrevAuthToken: "b", + UserAgent: "c", + ClientIp: "d", + AuthTokenSeen: true, + SeenAt: 3, + RotatedAt: 4, + CreatedAt: 5, + UpdatedAt: 6, + UnhashedToken: "e", + } + utBytes, err := json.Marshal(ut) + So(err, ShouldBeNil) + utJSON, err := simplejson.NewJson(utBytes) + So(err, ShouldBeNil) + utMap := utJSON.MustMap() + + var uat userAuthToken + uat.fromUserToken(&ut) + uatBytes, err := json.Marshal(uat) + So(err, ShouldBeNil) + uatJSON, err := simplejson.NewJson(uatBytes) + So(err, ShouldBeNil) + uatMap := uatJSON.MustMap() + + So(uatMap, ShouldResemble, utMap) + }) + + Convey("When populating userToken from userAuthToken should copy all properties", func() { + uat := userAuthToken{ + Id: 1, + UserId: 2, + AuthToken: "a", + PrevAuthToken: "b", + UserAgent: "c", + ClientIp: "d", + AuthTokenSeen: true, + SeenAt: 3, + RotatedAt: 4, + CreatedAt: 5, + UpdatedAt: 6, + UnhashedToken: "e", + } + uatBytes, err := json.Marshal(uat) + So(err, ShouldBeNil) + uatJSON, err := simplejson.NewJson(uatBytes) + So(err, ShouldBeNil) + uatMap := uatJSON.MustMap() + + var ut auth.UserToken + err = uat.toUserToken(&ut) + So(err, ShouldBeNil) + utBytes, err := json.Marshal(ut) + So(err, ShouldBeNil) + utJSON, err := simplejson.NewJson(utBytes) + So(err, ShouldBeNil) + utMap := utJSON.MustMap() + + So(utMap, ShouldResemble, uatMap) + }) + Reset(func() { getTime = time.Now }) diff --git a/pkg/services/auth/authtoken/model.go b/pkg/services/auth/authtoken/model.go index 8bd89c68b04..730f615e294 100644 --- a/pkg/services/auth/authtoken/model.go +++ b/pkg/services/auth/authtoken/model.go @@ -27,50 +27,52 @@ type userAuthToken struct { UnhashedToken string `xorm:"-"` } -func (uat *userAuthToken) toUserToken() (auth.UserToken, error) { +func userAuthTokenFromUserToken(ut *auth.UserToken) *userAuthToken { + var uat userAuthToken + uat.fromUserToken(ut) + return &uat +} + +func (uat *userAuthToken) fromUserToken(ut *auth.UserToken) { + uat.Id = ut.Id + uat.UserId = ut.UserId + uat.AuthToken = ut.AuthToken + uat.PrevAuthToken = ut.PrevAuthToken + uat.UserAgent = ut.UserAgent + uat.ClientIp = ut.ClientIp + uat.AuthTokenSeen = ut.AuthTokenSeen + uat.SeenAt = ut.SeenAt + uat.RotatedAt = ut.RotatedAt + uat.CreatedAt = ut.CreatedAt + uat.UpdatedAt = ut.UpdatedAt + uat.UnhashedToken = ut.UnhashedToken +} + +func (uat *userAuthToken) toUserToken(ut *auth.UserToken) error { if uat == nil { - return nil, fmt.Errorf("needs pointer to userAuthToken struct") + return fmt.Errorf("needs pointer to userAuthToken struct") } - return &userTokenImpl{ - userAuthToken: uat, - }, nil -} + ut.Id = uat.Id + ut.UserId = uat.UserId + ut.AuthToken = uat.AuthToken + ut.PrevAuthToken = uat.PrevAuthToken + ut.UserAgent = uat.UserAgent + ut.ClientIp = uat.ClientIp + ut.AuthTokenSeen = uat.AuthTokenSeen + ut.SeenAt = uat.SeenAt + ut.RotatedAt = uat.RotatedAt + ut.CreatedAt = uat.CreatedAt + ut.UpdatedAt = uat.UpdatedAt + ut.UnhashedToken = uat.UnhashedToken -type userToken interface { - auth.UserToken - GetModel() *userAuthToken -} - -type userTokenImpl struct { - *userAuthToken -} - -func (ut *userTokenImpl) GetUserId() int64 { - return ut.UserId -} - -func (ut *userTokenImpl) GetToken() string { - return ut.UnhashedToken -} - -func (ut *userTokenImpl) GetModel() *userAuthToken { - return ut.userAuthToken -} - -func extractModelFromToken(token auth.UserToken) (*userAuthToken, error) { - ut, ok := token.(userToken) - if !ok { - return nil, fmt.Errorf("failed to cast token") - } - - return ut.GetModel(), nil + return nil } // UserAuthTokenService are used for generating and validating user auth tokens type UserAuthTokenService interface { - CreateToken(userId int64, clientIP, userAgent string) (auth.UserToken, error) - LookupToken(unhashedToken string) (auth.UserToken, error) - TryRotateToken(token auth.UserToken, clientIP, userAgent string) (bool, error) - RevokeToken(token auth.UserToken) error + CreateToken(userId int64, clientIP, userAgent string) (*auth.UserToken, error) + LookupToken(unhashedToken string) (*auth.UserToken, error) + TryRotateToken(token *auth.UserToken, clientIP, userAgent string) (bool, error) + RevokeToken(token *auth.UserToken) error } From 8678620730b26dfa8a07bd0f01d9345d6b266ffd Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 16:45:48 +0100 Subject: [PATCH 13/19] move UserToken and UserTokenService to models package --- pkg/api/http_server.go | 17 ++++---- pkg/api/login.go | 4 +- pkg/middleware/middleware.go | 5 +-- pkg/middleware/middleware_test.go | 41 +++++++++---------- pkg/middleware/org_redirect_test.go | 10 ++--- pkg/middleware/quota_test.go | 5 +-- pkg/models/context.go | 3 +- pkg/models/user_token.go | 32 +++++++++++++++ pkg/services/auth/auth.go | 15 ------- pkg/services/auth/authtoken/auth_token.go | 15 ++++--- .../auth/authtoken/auth_token_test.go | 8 ++-- pkg/services/auth/authtoken/model.go | 16 ++------ 12 files changed, 85 insertions(+), 86 deletions(-) create mode 100644 pkg/models/user_token.go diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index a0a65d73244..cadf6896bf4 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -21,7 +21,6 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry" - "github.com/grafana/grafana/pkg/services/auth/authtoken" "github.com/grafana/grafana/pkg/services/cache" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/hooks" @@ -48,14 +47,14 @@ type HTTPServer struct { streamManager *live.StreamManager httpSrv *http.Server - RouteRegister routing.RouteRegister `inject:""` - Bus bus.Bus `inject:""` - RenderService rendering.Service `inject:""` - Cfg *setting.Cfg `inject:""` - HooksService *hooks.HooksService `inject:""` - CacheService *cache.CacheService `inject:""` - DatasourceCache datasources.CacheService `inject:""` - AuthTokenService authtoken.UserAuthTokenService `inject:""` + RouteRegister routing.RouteRegister `inject:""` + Bus bus.Bus `inject:""` + RenderService rendering.Service `inject:""` + Cfg *setting.Cfg `inject:""` + HooksService *hooks.HooksService `inject:""` + CacheService *cache.CacheService `inject:""` + DatasourceCache datasources.CacheService `inject:""` + AuthTokenService models.UserTokenService `inject:""` } func (hs *HTTPServer) Init() error { diff --git a/pkg/api/login.go b/pkg/api/login.go index 48f8f237884..106a48dd6a8 100644 --- a/pkg/api/login.go +++ b/pkg/api/login.go @@ -5,8 +5,6 @@ import ( "net/http" "net/url" - "github.com/grafana/grafana/pkg/services/auth/authtoken" - "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" @@ -141,7 +139,7 @@ func (hs *HTTPServer) loginUserWithUser(user *m.User, c *m.ReqContext) { } func (hs *HTTPServer) Logout(c *m.ReqContext) { - if err := hs.AuthTokenService.RevokeToken(c.UserToken); err != nil && err != authtoken.ErrAuthTokenNotFound { + if err := hs.AuthTokenService.RevokeToken(c.UserToken); err != nil && err != m.ErrUserTokenNotFound { hs.log.Error("failed to revoke auth token", "error", err) } diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 762a12d09d2..fa335eb10d9 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -10,7 +10,6 @@ import ( "github.com/grafana/grafana/pkg/components/apikeygen" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/auth/authtoken" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -24,7 +23,7 @@ var ( ReqOrgAdmin = RoleAuth(m.ROLE_ADMIN) ) -func GetContextHandler(ats authtoken.UserAuthTokenService) macaron.Handler { +func GetContextHandler(ats m.UserTokenService) macaron.Handler { return func(c *macaron.Context) { ctx := &m.ReqContext{ Context: c, @@ -169,7 +168,7 @@ func initContextWithBasicAuth(ctx *m.ReqContext, orgId int64) bool { return true } -func initContextWithToken(authTokenService authtoken.UserAuthTokenService, ctx *m.ReqContext, orgID int64) bool { +func initContextWithToken(authTokenService m.UserTokenService, ctx *m.ReqContext, orgID int64) bool { rawToken := ctx.GetCookie(setting.LoginCookieName) if rawToken == "" { return false diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 5852f9d0bcd..7908be6f225 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -11,7 +11,6 @@ import ( msession "github.com/go-macaron/session" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/auth/authtoken" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" @@ -157,8 +156,8 @@ func TestMiddlewareContext(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { - return &auth.UserToken{ + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + return &m.UserToken{ UserId: 12, UnhashedToken: unhashedToken, }, nil @@ -186,14 +185,14 @@ func TestMiddlewareContext(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { - return &auth.UserToken{ + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + return &m.UserToken{ UserId: 12, UnhashedToken: "", }, nil } - sc.userAuthTokenService.tryRotateTokenProvider = func(userToken *auth.UserToken, clientIP, userAgent string) (bool, error) { + sc.userAuthTokenService.tryRotateTokenProvider = func(userToken *m.UserToken, clientIP, userAgent string) (bool, error) { userToken.UnhashedToken = "rotated" return true, nil } @@ -228,7 +227,7 @@ func TestMiddlewareContext(t *testing.T) { middlewareScenario("Invalid/expired auth token in cookie", func(sc *scenarioContext) { sc.withTokenSessionCookie("token") - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { return nil, authtoken.ErrAuthTokenNotFound } @@ -680,47 +679,47 @@ type scenarioFunc func(c *scenarioContext) type handlerFunc func(c *m.ReqContext) type fakeUserAuthTokenService struct { - createTokenProvider func(userId int64, clientIP, userAgent string) (*auth.UserToken, error) - tryRotateTokenProvider func(token *auth.UserToken, clientIP, userAgent string) (bool, error) - lookupTokenProvider func(unhashedToken string) (*auth.UserToken, error) - revokeTokenProvider func(token *auth.UserToken) error + createTokenProvider func(userId int64, clientIP, userAgent string) (*m.UserToken, error) + tryRotateTokenProvider func(token *m.UserToken, clientIP, userAgent string) (bool, error) + lookupTokenProvider func(unhashedToken string) (*m.UserToken, error) + revokeTokenProvider func(token *m.UserToken) error } func newFakeUserAuthTokenService() *fakeUserAuthTokenService { return &fakeUserAuthTokenService{ - createTokenProvider: func(userId int64, clientIP, userAgent string) (*auth.UserToken, error) { - return &auth.UserToken{ + createTokenProvider: func(userId int64, clientIP, userAgent string) (*m.UserToken, error) { + return &m.UserToken{ UserId: 0, UnhashedToken: "", }, nil }, - tryRotateTokenProvider: func(token *auth.UserToken, clientIP, userAgent string) (bool, error) { + tryRotateTokenProvider: func(token *m.UserToken, clientIP, userAgent string) (bool, error) { return false, nil }, - lookupTokenProvider: func(unhashedToken string) (*auth.UserToken, error) { - return &auth.UserToken{ + lookupTokenProvider: func(unhashedToken string) (*m.UserToken, error) { + return &m.UserToken{ UserId: 0, UnhashedToken: "", }, nil }, - revokeTokenProvider: func(token *auth.UserToken) error { + revokeTokenProvider: func(token *m.UserToken) error { return nil }, } } -func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*auth.UserToken, error) { +func (s *fakeUserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*m.UserToken, error) { return s.createTokenProvider(userId, clientIP, userAgent) } -func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (*auth.UserToken, error) { +func (s *fakeUserAuthTokenService) LookupToken(unhashedToken string) (*m.UserToken, error) { return s.lookupTokenProvider(unhashedToken) } -func (s *fakeUserAuthTokenService) TryRotateToken(token *auth.UserToken, clientIP, userAgent string) (bool, error) { +func (s *fakeUserAuthTokenService) TryRotateToken(token *m.UserToken, clientIP, userAgent string) (bool, error) { return s.tryRotateTokenProvider(token, clientIP, userAgent) } -func (s *fakeUserAuthTokenService) RevokeToken(token *auth.UserToken) error { +func (s *fakeUserAuthTokenService) RevokeToken(token *m.UserToken) error { return s.revokeTokenProvider(token) } diff --git a/pkg/middleware/org_redirect_test.go b/pkg/middleware/org_redirect_test.go index 0cc88016f3a..e01d1a68d21 100644 --- a/pkg/middleware/org_redirect_test.go +++ b/pkg/middleware/org_redirect_test.go @@ -3,8 +3,6 @@ package middleware import ( "testing" - "github.com/grafana/grafana/pkg/services/auth" - "fmt" "github.com/grafana/grafana/pkg/bus" @@ -26,8 +24,8 @@ func TestOrgRedirectMiddleware(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { - return &auth.UserToken{ + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + return &m.UserToken{ UserId: 0, UnhashedToken: "", }, nil @@ -52,8 +50,8 @@ func TestOrgRedirectMiddleware(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { - return &auth.UserToken{ + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + return &m.UserToken{ UserId: 12, UnhashedToken: "", }, nil diff --git a/pkg/middleware/quota_test.go b/pkg/middleware/quota_test.go index df2bf930426..e2a6ef63377 100644 --- a/pkg/middleware/quota_test.go +++ b/pkg/middleware/quota_test.go @@ -5,7 +5,6 @@ import ( "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" . "github.com/smartystreets/goconvey/convey" @@ -81,8 +80,8 @@ func TestMiddlewareQuota(t *testing.T) { return nil }) - sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*auth.UserToken, error) { - return &auth.UserToken{ + sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { + return &m.UserToken{ UserId: 12, UnhashedToken: "", }, nil diff --git a/pkg/models/context.go b/pkg/models/context.go index bba0024c0f2..b0c6ec9226d 100644 --- a/pkg/models/context.go +++ b/pkg/models/context.go @@ -4,7 +4,6 @@ import ( "strings" "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/prometheus/client_golang/prometheus" @@ -14,7 +13,7 @@ import ( type ReqContext struct { *macaron.Context *SignedInUser - UserToken *auth.UserToken + UserToken *UserToken // This should only be used by the auth_proxy Session session.SessionStore diff --git a/pkg/models/user_token.go b/pkg/models/user_token.go new file mode 100644 index 00000000000..c8084cf1eba --- /dev/null +++ b/pkg/models/user_token.go @@ -0,0 +1,32 @@ +package models + +import "errors" + +// Typed errors +var ( + ErrUserTokenNotFound = errors.New("user token not found") +) + +// UserToken represents a user token +type UserToken struct { + Id int64 + UserId int64 + AuthToken string + PrevAuthToken string + UserAgent string + ClientIp string + AuthTokenSeen bool + SeenAt int64 + RotatedAt int64 + CreatedAt int64 + UpdatedAt int64 + UnhashedToken string +} + +// UserTokenService are used for generating and validating user tokens +type UserTokenService interface { + CreateToken(userId int64, clientIP, userAgent string) (*UserToken, error) + LookupToken(unhashedToken string) (*UserToken, error) + TryRotateToken(token *UserToken, clientIP, userAgent string) (bool, error) + RevokeToken(token *UserToken) error +} diff --git a/pkg/services/auth/auth.go b/pkg/services/auth/auth.go index 727ceffff79..8832b06d188 100644 --- a/pkg/services/auth/auth.go +++ b/pkg/services/auth/auth.go @@ -1,16 +1 @@ package auth - -type UserToken struct { - Id int64 - UserId int64 - AuthToken string - PrevAuthToken string - UserAgent string - ClientIp string - AuthTokenSeen bool - SeenAt int64 - RotatedAt int64 - CreatedAt int64 - UpdatedAt int64 - UnhashedToken string -} diff --git a/pkg/services/auth/authtoken/auth_token.go b/pkg/services/auth/authtoken/auth_token.go index a993a37dbf3..eedf38e8141 100644 --- a/pkg/services/auth/authtoken/auth_token.go +++ b/pkg/services/auth/authtoken/auth_token.go @@ -5,11 +5,10 @@ import ( "encoding/hex" "time" - "github.com/grafana/grafana/pkg/services/auth" - "github.com/grafana/grafana/pkg/infra/serverlock" "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" @@ -40,7 +39,7 @@ func (s *UserAuthTokenServiceImpl) Init() error { return nil } -func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent string) (*auth.UserToken, error) { +func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent string) (*models.UserToken, error) { clientIP = util.ParseIPAddress(clientIP) token, err := util.RandomHex(16) if err != nil { @@ -72,13 +71,13 @@ func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent s.log.Debug("user auth token created", "tokenId", userAuthToken.Id, "userId", userAuthToken.UserId, "clientIP", userAuthToken.ClientIp, "userAgent", userAuthToken.UserAgent, "authToken", userAuthToken.AuthToken) - var userToken auth.UserToken + var userToken models.UserToken err = userAuthToken.toUserToken(&userToken) return &userToken, err } -func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*auth.UserToken, error) { +func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*models.UserToken, error) { hashedToken := hashToken(unhashedToken) if setting.Env == setting.DEV { s.log.Debug("looking up token", "unhashed", unhashedToken, "hashed", hashedToken) @@ -137,13 +136,13 @@ func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*auth.User model.UnhashedToken = unhashedToken - var userToken auth.UserToken + var userToken models.UserToken err = model.toUserToken(&userToken) return &userToken, err } -func (s *UserAuthTokenServiceImpl) TryRotateToken(token *auth.UserToken, clientIP, userAgent string) (bool, error) { +func (s *UserAuthTokenServiceImpl) TryRotateToken(token *models.UserToken, clientIP, userAgent string) (bool, error) { if token == nil { return false, nil } @@ -202,7 +201,7 @@ func (s *UserAuthTokenServiceImpl) TryRotateToken(token *auth.UserToken, clientI return false, nil } -func (s *UserAuthTokenServiceImpl) RevokeToken(token *auth.UserToken) error { +func (s *UserAuthTokenServiceImpl) RevokeToken(token *models.UserToken) error { if token == nil { return ErrAuthTokenNotFound } diff --git a/pkg/services/auth/authtoken/auth_token_test.go b/pkg/services/auth/authtoken/auth_token_test.go index a1b70db60fb..3021b0efa4b 100644 --- a/pkg/services/auth/authtoken/auth_token_test.go +++ b/pkg/services/auth/authtoken/auth_token_test.go @@ -9,7 +9,7 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/log" - "github.com/grafana/grafana/pkg/services/auth" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/sqlstore" . "github.com/smartystreets/goconvey/convey" ) @@ -163,7 +163,7 @@ func TestUserAuthToken(t *testing.T) { model, err := ctx.getAuthTokenByID(userToken.Id) So(err, ShouldBeNil) - var tok auth.UserToken + var tok models.UserToken model.toUserToken(&tok) getTime = func() time.Time { @@ -345,7 +345,7 @@ func TestUserAuthToken(t *testing.T) { }) Convey("When populating userAuthToken from UserToken should copy all properties", func() { - ut := auth.UserToken{ + ut := models.UserToken{ Id: 1, UserId: 2, AuthToken: "a", @@ -397,7 +397,7 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) uatMap := uatJSON.MustMap() - var ut auth.UserToken + var ut models.UserToken err = uat.toUserToken(&ut) So(err, ShouldBeNil) utBytes, err := json.Marshal(ut) diff --git a/pkg/services/auth/authtoken/model.go b/pkg/services/auth/authtoken/model.go index 730f615e294..64123f02bc6 100644 --- a/pkg/services/auth/authtoken/model.go +++ b/pkg/services/auth/authtoken/model.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "github.com/grafana/grafana/pkg/services/auth" + "github.com/grafana/grafana/pkg/models" ) // Typed errors @@ -27,13 +27,13 @@ type userAuthToken struct { UnhashedToken string `xorm:"-"` } -func userAuthTokenFromUserToken(ut *auth.UserToken) *userAuthToken { +func userAuthTokenFromUserToken(ut *models.UserToken) *userAuthToken { var uat userAuthToken uat.fromUserToken(ut) return &uat } -func (uat *userAuthToken) fromUserToken(ut *auth.UserToken) { +func (uat *userAuthToken) fromUserToken(ut *models.UserToken) { uat.Id = ut.Id uat.UserId = ut.UserId uat.AuthToken = ut.AuthToken @@ -48,7 +48,7 @@ func (uat *userAuthToken) fromUserToken(ut *auth.UserToken) { uat.UnhashedToken = ut.UnhashedToken } -func (uat *userAuthToken) toUserToken(ut *auth.UserToken) error { +func (uat *userAuthToken) toUserToken(ut *models.UserToken) error { if uat == nil { return fmt.Errorf("needs pointer to userAuthToken struct") } @@ -68,11 +68,3 @@ func (uat *userAuthToken) toUserToken(ut *auth.UserToken) error { return nil } - -// UserAuthTokenService are used for generating and validating user auth tokens -type UserAuthTokenService interface { - CreateToken(userId int64, clientIP, userAgent string) (*auth.UserToken, error) - LookupToken(unhashedToken string) (*auth.UserToken, error) - TryRotateToken(token *auth.UserToken, clientIP, userAgent string) (bool, error) - RevokeToken(token *auth.UserToken) error -} From 8ae066ab5d30e9049b186479fda64b62de50d544 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 17:02:57 +0100 Subject: [PATCH 14/19] move authtoken package into auth package --- pkg/cmd/grafana-server/server.go | 1 + pkg/services/auth/auth.go | 1 - .../auth/{authtoken => }/auth_token.go | 26 ++++++++----------- .../auth/{authtoken => }/auth_token_test.go | 19 +++++++------- pkg/services/auth/{authtoken => }/model.go | 16 ++++++------ .../session_cleanup.go => token_cleanup.go} | 6 ++--- ..._cleanup_test.go => token_cleanup_test.go} | 2 +- 7 files changed, 34 insertions(+), 37 deletions(-) delete mode 100644 pkg/services/auth/auth.go rename pkg/services/auth/{authtoken => }/auth_token.go (89%) rename pkg/services/auth/{authtoken => }/auth_token_test.go (97%) rename pkg/services/auth/{authtoken => }/model.go (88%) rename pkg/services/auth/{authtoken/session_cleanup.go => token_cleanup.go} (87%) rename pkg/services/auth/{authtoken/session_cleanup_test.go => token_cleanup_test.go} (99%) diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go index 4781361b9b9..f663e6be895 100644 --- a/pkg/cmd/grafana-server/server.go +++ b/pkg/cmd/grafana-server/server.go @@ -32,6 +32,7 @@ import ( _ "github.com/grafana/grafana/pkg/metrics" _ "github.com/grafana/grafana/pkg/plugins" _ "github.com/grafana/grafana/pkg/services/alerting" + _ "github.com/grafana/grafana/pkg/services/auth" _ "github.com/grafana/grafana/pkg/services/cleanup" _ "github.com/grafana/grafana/pkg/services/notifications" _ "github.com/grafana/grafana/pkg/services/provisioning" diff --git a/pkg/services/auth/auth.go b/pkg/services/auth/auth.go deleted file mode 100644 index 8832b06d188..00000000000 --- a/pkg/services/auth/auth.go +++ /dev/null @@ -1 +0,0 @@ -package auth diff --git a/pkg/services/auth/authtoken/auth_token.go b/pkg/services/auth/auth_token.go similarity index 89% rename from pkg/services/auth/authtoken/auth_token.go rename to pkg/services/auth/auth_token.go index eedf38e8141..ef5dccd779f 100644 --- a/pkg/services/auth/authtoken/auth_token.go +++ b/pkg/services/auth/auth_token.go @@ -1,4 +1,4 @@ -package authtoken +package auth import ( "crypto/sha256" @@ -16,30 +16,26 @@ import ( ) func init() { - registry.Register(®istry.Descriptor{ - Name: "AuthTokenService", - Instance: &UserAuthTokenServiceImpl{}, - InitPriority: registry.Low, - }) + registry.RegisterService(&UserAuthTokenService{}) } var getTime = time.Now const urgentRotateTime = 1 * time.Minute -type UserAuthTokenServiceImpl struct { +type UserAuthTokenService struct { SQLStore *sqlstore.SqlStore `inject:""` ServerLockService *serverlock.ServerLockService `inject:""` Cfg *setting.Cfg `inject:""` log log.Logger } -func (s *UserAuthTokenServiceImpl) Init() error { +func (s *UserAuthTokenService) Init() error { s.log = log.New("auth") return nil } -func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent string) (*models.UserToken, error) { +func (s *UserAuthTokenService) CreateToken(userId int64, clientIP, userAgent string) (*models.UserToken, error) { clientIP = util.ParseIPAddress(clientIP) token, err := util.RandomHex(16) if err != nil { @@ -77,7 +73,7 @@ func (s *UserAuthTokenServiceImpl) CreateToken(userId int64, clientIP, userAgent return &userToken, err } -func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*models.UserToken, error) { +func (s *UserAuthTokenService) LookupToken(unhashedToken string) (*models.UserToken, error) { hashedToken := hashToken(unhashedToken) if setting.Env == setting.DEV { s.log.Debug("looking up token", "unhashed", unhashedToken, "hashed", hashedToken) @@ -95,7 +91,7 @@ func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*models.Us } if !exists { - return nil, ErrAuthTokenNotFound + return nil, models.ErrUserTokenNotFound } if model.AuthToken != hashedToken && model.PrevAuthToken == hashedToken && model.AuthTokenSeen { @@ -142,7 +138,7 @@ func (s *UserAuthTokenServiceImpl) LookupToken(unhashedToken string) (*models.Us return &userToken, err } -func (s *UserAuthTokenServiceImpl) TryRotateToken(token *models.UserToken, clientIP, userAgent string) (bool, error) { +func (s *UserAuthTokenService) TryRotateToken(token *models.UserToken, clientIP, userAgent string) (bool, error) { if token == nil { return false, nil } @@ -201,9 +197,9 @@ func (s *UserAuthTokenServiceImpl) TryRotateToken(token *models.UserToken, clien return false, nil } -func (s *UserAuthTokenServiceImpl) RevokeToken(token *models.UserToken) error { +func (s *UserAuthTokenService) RevokeToken(token *models.UserToken) error { if token == nil { - return ErrAuthTokenNotFound + return models.ErrUserTokenNotFound } model := userAuthTokenFromUserToken(token) @@ -215,7 +211,7 @@ func (s *UserAuthTokenServiceImpl) RevokeToken(token *models.UserToken) error { if rowsAffected == 0 { s.log.Debug("user auth token not found/revoked", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent) - return ErrAuthTokenNotFound + return models.ErrUserTokenNotFound } s.log.Debug("user auth token revoked", "tokenId", model.Id, "userId", model.UserId, "clientIP", model.ClientIp, "userAgent", model.UserAgent) diff --git a/pkg/services/auth/authtoken/auth_token_test.go b/pkg/services/auth/auth_token_test.go similarity index 97% rename from pkg/services/auth/authtoken/auth_token_test.go rename to pkg/services/auth/auth_token_test.go index 3021b0efa4b..3313af9f87f 100644 --- a/pkg/services/auth/authtoken/auth_token_test.go +++ b/pkg/services/auth/auth_token_test.go @@ -1,4 +1,4 @@ -package authtoken +package auth import ( "encoding/json" @@ -46,7 +46,7 @@ func TestUserAuthToken(t *testing.T) { Convey("When lookup hashed token should return user auth token not found error", func() { userToken, err := userAuthTokenService.LookupToken(userToken.AuthToken) - So(err, ShouldEqual, ErrAuthTokenNotFound) + So(err, ShouldEqual, models.ErrUserTokenNotFound) So(userToken, ShouldBeNil) }) @@ -61,13 +61,13 @@ func TestUserAuthToken(t *testing.T) { Convey("revoking nil token should return error", func() { err = userAuthTokenService.RevokeToken(nil) - So(err, ShouldEqual, ErrAuthTokenNotFound) + So(err, ShouldEqual, models.ErrUserTokenNotFound) }) Convey("revoking non-existing token should return error", func() { userToken.Id = 1000 err = userAuthTokenService.RevokeToken(userToken) - So(err, ShouldEqual, ErrAuthTokenNotFound) + So(err, ShouldEqual, models.ErrUserTokenNotFound) }) }) @@ -112,7 +112,7 @@ func TestUserAuthToken(t *testing.T) { } notGood, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) - So(err, ShouldEqual, ErrAuthTokenNotFound) + So(err, ShouldEqual, models.ErrUserTokenNotFound) So(notGood, ShouldBeNil) }) @@ -140,7 +140,7 @@ func TestUserAuthToken(t *testing.T) { } notGood, err := userAuthTokenService.LookupToken(userToken.UnhashedToken) - So(err, ShouldEqual, ErrAuthTokenNotFound) + So(err, ShouldEqual, models.ErrUserTokenNotFound) So(notGood, ShouldBeNil) }) }) @@ -164,7 +164,8 @@ func TestUserAuthToken(t *testing.T) { So(err, ShouldBeNil) var tok models.UserToken - model.toUserToken(&tok) + err = model.toUserToken(&tok) + So(err, ShouldBeNil) getTime = func() time.Time { return t.Add(time.Hour) @@ -419,7 +420,7 @@ func createTestContext(t *testing.T) *testContext { t.Helper() sqlstore := sqlstore.InitTestDB(t) - tokenService := &UserAuthTokenServiceImpl{ + tokenService := &UserAuthTokenService{ SQLStore: sqlstore, Cfg: &setting.Cfg{ LoginMaxInactiveLifetimeDays: 7, @@ -438,7 +439,7 @@ func createTestContext(t *testing.T) *testContext { type testContext struct { sqlstore *sqlstore.SqlStore - tokenService *UserAuthTokenServiceImpl + tokenService *UserAuthTokenService } func (c *testContext) getAuthTokenByID(id int64) (*userAuthToken, error) { diff --git a/pkg/services/auth/authtoken/model.go b/pkg/services/auth/model.go similarity index 88% rename from pkg/services/auth/authtoken/model.go rename to pkg/services/auth/model.go index 64123f02bc6..36652e70436 100644 --- a/pkg/services/auth/authtoken/model.go +++ b/pkg/services/auth/model.go @@ -1,17 +1,11 @@ -package authtoken +package auth import ( - "errors" "fmt" "github.com/grafana/grafana/pkg/models" ) -// Typed errors -var ( - ErrAuthTokenNotFound = errors.New("user auth token not found") -) - type userAuthToken struct { Id int64 UserId int64 @@ -33,7 +27,11 @@ func userAuthTokenFromUserToken(ut *models.UserToken) *userAuthToken { return &uat } -func (uat *userAuthToken) fromUserToken(ut *models.UserToken) { +func (uat *userAuthToken) fromUserToken(ut *models.UserToken) error { + if uat == nil { + return fmt.Errorf("needs pointer to userAuthToken struct") + } + uat.Id = ut.Id uat.UserId = ut.UserId uat.AuthToken = ut.AuthToken @@ -46,6 +44,8 @@ func (uat *userAuthToken) fromUserToken(ut *models.UserToken) { uat.CreatedAt = ut.CreatedAt uat.UpdatedAt = ut.UpdatedAt uat.UnhashedToken = ut.UnhashedToken + + return nil } func (uat *userAuthToken) toUserToken(ut *models.UserToken) error { diff --git a/pkg/services/auth/authtoken/session_cleanup.go b/pkg/services/auth/token_cleanup.go similarity index 87% rename from pkg/services/auth/authtoken/session_cleanup.go rename to pkg/services/auth/token_cleanup.go index 2b8dfb7b4e2..d0e12c9c0e1 100644 --- a/pkg/services/auth/authtoken/session_cleanup.go +++ b/pkg/services/auth/token_cleanup.go @@ -1,11 +1,11 @@ -package authtoken +package auth import ( "context" "time" ) -func (srv *UserAuthTokenServiceImpl) Run(ctx context.Context) error { +func (srv *UserAuthTokenService) Run(ctx context.Context) error { if srv.Cfg.ExpiredTokensCleanupIntervalDays <= 0 { srv.log.Debug("cleanup of expired auth tokens are disabled") return nil @@ -31,7 +31,7 @@ func (srv *UserAuthTokenServiceImpl) Run(ctx context.Context) error { } } -func (srv *UserAuthTokenServiceImpl) deleteExpiredTokens(maxInactiveLifetime, maxLifetime time.Duration) (int64, error) { +func (srv *UserAuthTokenService) deleteExpiredTokens(maxInactiveLifetime, maxLifetime time.Duration) (int64, error) { createdBefore := getTime().Add(-maxLifetime) rotatedBefore := getTime().Add(-maxInactiveLifetime) diff --git a/pkg/services/auth/authtoken/session_cleanup_test.go b/pkg/services/auth/token_cleanup_test.go similarity index 99% rename from pkg/services/auth/authtoken/session_cleanup_test.go rename to pkg/services/auth/token_cleanup_test.go index 7b611b3263c..410764d3f8d 100644 --- a/pkg/services/auth/authtoken/session_cleanup_test.go +++ b/pkg/services/auth/token_cleanup_test.go @@ -1,4 +1,4 @@ -package authtoken +package auth import ( "fmt" From 836501186f314c78a69f5ab88ca86b1d1925625c Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 17:30:17 +0100 Subject: [PATCH 15/19] fix --- pkg/middleware/middleware_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 7908be6f225..8545c3856c9 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -11,7 +11,6 @@ import ( msession "github.com/go-macaron/session" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/auth/authtoken" "github.com/grafana/grafana/pkg/services/session" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -228,7 +227,7 @@ func TestMiddlewareContext(t *testing.T) { sc.withTokenSessionCookie("token") sc.userAuthTokenService.lookupTokenProvider = func(unhashedToken string) (*m.UserToken, error) { - return nil, authtoken.ErrAuthTokenNotFound + return nil, m.ErrUserTokenNotFound } sc.fakeReq("GET", "/").exec() From 1a140ee199cf7430b67743cc2399ec749a16d94d Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 22:27:08 +0100 Subject: [PATCH 16/19] run token cleanup job when grafana starts, then each hour --- conf/defaults.ini | 4 ++-- conf/sample.ini | 4 ++-- docs/sources/auth/overview.md | 4 ++-- pkg/services/auth/auth_token_test.go | 8 ++++---- pkg/services/auth/token_cleanup.go | 22 +++++++++++++--------- pkg/setting/setting.go | 12 ++++++------ 6 files changed, 29 insertions(+), 25 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 41b948e53af..d0fc1133fac 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -256,8 +256,8 @@ login_maximum_lifetime_days = 30 # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. token_rotation_interval_minutes = 10 -# How often should expired auth tokens be deleted from the database. The default is 7 days. -expired_tokens_cleanup_interval_days = 7 +# How often should expired auth tokens be deleted from the database. The default is each hour. +expired_tokens_cleanup_interval_hours = 1 # Set to true to disable (hide) the login form, useful if you use OAuth disable_login_form = false diff --git a/conf/sample.ini b/conf/sample.ini index 831fa31253e..2ff37239abf 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -236,8 +236,8 @@ log_queries = # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. ;token_rotation_interval_minutes = 10 -# How often should expired auth tokens be deleted from the database. The default is 7 days. -;expired_tokens_cleanup_interval_days = 7 +# How often should expired auth tokens be deleted from the database. The default is each hour. +;expired_tokens_cleanup_interval_hours = 1 # Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false ;disable_login_form = false diff --git a/docs/sources/auth/overview.md b/docs/sources/auth/overview.md index fba8da00a5e..0f563fbe8d5 100644 --- a/docs/sources/auth/overview.md +++ b/docs/sources/auth/overview.md @@ -64,8 +64,8 @@ login_maximum_lifetime_days = 30 # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. token_rotation_interval_minutes = 10 -# How often should expired auth tokens be deleted from the database. The default is 7 days. -expired_tokens_cleanup_interval_days = 7 +# How often should expired auth tokens be deleted from the database. The default is each hour. +expired_tokens_cleanup_interval_hours = 1 ``` ### Anonymous authentication diff --git a/pkg/services/auth/auth_token_test.go b/pkg/services/auth/auth_token_test.go index 3313af9f87f..964bd499a01 100644 --- a/pkg/services/auth/auth_token_test.go +++ b/pkg/services/auth/auth_token_test.go @@ -423,10 +423,10 @@ func createTestContext(t *testing.T) *testContext { tokenService := &UserAuthTokenService{ SQLStore: sqlstore, Cfg: &setting.Cfg{ - LoginMaxInactiveLifetimeDays: 7, - LoginMaxLifetimeDays: 30, - TokenRotationIntervalMinutes: 10, - ExpiredTokensCleanupIntervalDays: 1, + LoginMaxInactiveLifetimeDays: 7, + LoginMaxLifetimeDays: 30, + TokenRotationIntervalMinutes: 10, + ExpiredTokensCleanupIntervalHours: 1, }, log: log.New("test-logger"), } diff --git a/pkg/services/auth/token_cleanup.go b/pkg/services/auth/token_cleanup.go index d0e12c9c0e1..0d5cbdaca10 100644 --- a/pkg/services/auth/token_cleanup.go +++ b/pkg/services/auth/token_cleanup.go @@ -6,25 +6,29 @@ import ( ) func (srv *UserAuthTokenService) Run(ctx context.Context) error { - if srv.Cfg.ExpiredTokensCleanupIntervalDays <= 0 { - srv.log.Debug("cleanup of expired auth tokens are disabled") - return nil - } - - jobInterval := time.Duration(srv.Cfg.ExpiredTokensCleanupIntervalDays) * 24 * time.Hour - srv.log.Debug("cleanup of expired auth tokens are enabled", "intervalDays", srv.Cfg.ExpiredTokensCleanupIntervalDays) - + jobInterval := time.Duration(srv.Cfg.ExpiredTokensCleanupIntervalHours) * time.Hour ticker := time.NewTicker(jobInterval) maxInactiveLifetime := time.Duration(srv.Cfg.LoginMaxInactiveLifetimeDays) * 24 * time.Hour maxLifetime := time.Duration(srv.Cfg.LoginMaxLifetimeDays) * 24 * time.Hour + err := srv.ServerLockService.LockAndExecute(ctx, "cleanup expired auth tokens", time.Hour*12, func() { + srv.deleteExpiredTokens(maxInactiveLifetime, maxLifetime) + }) + if err != nil { + srv.log.Error("failed to lock and execite cleanup of expired auth token", "erro", err) + } + for { select { case <-ticker.C: - srv.ServerLockService.LockAndExecute(ctx, "cleanup expired auth tokens", time.Hour*12, func() { + err := srv.ServerLockService.LockAndExecute(ctx, "cleanup expired auth tokens", time.Hour*12, func() { srv.deleteExpiredTokens(maxInactiveLifetime, maxLifetime) }) + if err != nil { + srv.log.Error("failed to lock and execite cleanup of expired auth token", "erro", err) + } + case <-ctx.Done(): return ctx.Err() } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 9f7d03bb472..43ac46e5ab8 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -233,11 +233,11 @@ type Cfg struct { EnterpriseLicensePath string // Auth - LoginCookieName string - LoginMaxInactiveLifetimeDays int - LoginMaxLifetimeDays int - TokenRotationIntervalMinutes int - ExpiredTokensCleanupIntervalDays int + LoginCookieName string + LoginMaxInactiveLifetimeDays int + LoginMaxLifetimeDays int + TokenRotationIntervalMinutes int + ExpiredTokensCleanupIntervalHours int } type CommandLineArgs struct { @@ -673,7 +673,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { if cfg.TokenRotationIntervalMinutes < 2 { cfg.TokenRotationIntervalMinutes = 2 } - cfg.ExpiredTokensCleanupIntervalDays = auth.Key("expired_tokens_cleanup_interval_days").MustInt(7) + cfg.ExpiredTokensCleanupIntervalHours = auth.Key("expired_tokens_cleanup_interval_hours").MustInt(1) DisableLoginForm = auth.Key("disable_login_form").MustBool(false) DisableSignoutMenu = auth.Key("disable_signout_menu").MustBool(false) From 3555997f98cb82c53032aa9d286ab76dd748a65b Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 6 Feb 2019 22:33:48 +0100 Subject: [PATCH 17/19] devenv: update ha test and load test better ha setup for many mysql connections prometheus now scrapes mysql metrics in ha setup ha setup provisions mysql dashboard adds configurable virtual users for load test run script --- devenv/docker/ha_test/docker-compose.yaml | 21 +- .../{alerts.yaml => dashboards.yaml} | 6 + .../dashboards/mysql/overview.json | 5397 +++++++++++++++++ .../docker/ha_test/prometheus/prometheus.yml | 14 +- devenv/docker/loadtest/README.md | 14 +- devenv/docker/loadtest/auth_token_test.js | 2 +- devenv/docker/loadtest/run.sh | 8 +- 7 files changed, 5443 insertions(+), 19 deletions(-) rename devenv/docker/ha_test/grafana/provisioning/dashboards/{alerts.yaml => dashboards.yaml} (55%) create mode 100644 devenv/docker/ha_test/grafana/provisioning/dashboards/mysql/overview.json diff --git a/devenv/docker/ha_test/docker-compose.yaml b/devenv/docker/ha_test/docker-compose.yaml index 504ee86404d..8087894c56e 100644 --- a/devenv/docker/ha_test/docker-compose.yaml +++ b/devenv/docker/ha_test/docker-compose.yaml @@ -15,6 +15,7 @@ services: MYSQL_DATABASE: grafana MYSQL_USER: grafana MYSQL_PASSWORD: password + command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --innodb_monitor_enable=all, --max-connections=1001] ports: - 3306 healthcheck: @@ -22,6 +23,16 @@ services: timeout: 10s retries: 10 + mysqld-exporter: + image: prom/mysqld-exporter + environment: + - DATA_SOURCE_NAME=root:rootpass@(db:3306)/ + ports: + - 9104 + depends_on: + db: + condition: service_healthy + # db: # image: postgres:9.3 # environment: @@ -47,6 +58,7 @@ services: - GF_DATABASE_PASSWORD=password - GF_DATABASE_TYPE=mysql - GF_DATABASE_HOST=db:3306 + - GF_DATABASE_MAX_OPEN_CONN=300 - GF_SESSION_PROVIDER=mysql - GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(db:3306)/grafana?allowNativePasswords=true # - GF_DATABASE_TYPE=postgres @@ -55,7 +67,7 @@ services: # - GF_SESSION_PROVIDER=postgres # - GF_SESSION_PROVIDER_CONFIG=user=grafana password=password host=db port=5432 dbname=grafana sslmode=disable - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug,auth:debug - - GF_LOGIN_ROTATE_TOKEN_MINUTES=2 + - GF_AUTH_TOKEN_ROTATION_INTERVAL_MINUTES=2 ports: - 3000 depends_on: @@ -70,10 +82,3 @@ services: - VIRTUAL_HOST=prometheus.loc ports: - 9090 - - # mysqld-exporter: - # image: prom/mysqld-exporter - # environment: - # - DATA_SOURCE_NAME=grafana:password@(mysql:3306)/ - # ports: - # - 9104 diff --git a/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts.yaml b/devenv/docker/ha_test/grafana/provisioning/dashboards/dashboards.yaml similarity index 55% rename from devenv/docker/ha_test/grafana/provisioning/dashboards/alerts.yaml rename to devenv/docker/ha_test/grafana/provisioning/dashboards/dashboards.yaml index 60b6cd4bb04..ad85bb7036f 100644 --- a/devenv/docker/ha_test/grafana/provisioning/dashboards/alerts.yaml +++ b/devenv/docker/ha_test/grafana/provisioning/dashboards/dashboards.yaml @@ -6,3 +6,9 @@ providers: type: file options: path: /etc/grafana/provisioning/dashboards/alerts + + - name: 'MySQL' + folder: 'MySQL' + type: file + options: + path: /etc/grafana/provisioning/dashboards/mysql diff --git a/devenv/docker/ha_test/grafana/provisioning/dashboards/mysql/overview.json b/devenv/docker/ha_test/grafana/provisioning/dashboards/mysql/overview.json new file mode 100644 index 00000000000..d072e4c1d28 --- /dev/null +++ b/devenv/docker/ha_test/grafana/provisioning/dashboards/mysql/overview.json @@ -0,0 +1,5397 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": false, + "hide": true, + "iconColor": "#e0752d", + "limit": 100, + "name": "PMM Annotations", + "showIn": 0, + "tags": [ + "pmm_annotation" + ], + "type": "tags" + }, + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": false, + "hide": true, + "iconColor": "#6ed0e0", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "tags": [ + + ], + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1540971751770, + "links": [ + { + "icon": "dashboard", + "includeVars": true, + "keepTime": true, + "tags": [ + "QAN" + ], + "targetBlank": false, + "title": "Query Analytics", + "type": "link", + "url": "/graph/dashboard/db/_pmm-query-analytics" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "OS" + ], + "targetBlank": false, + "title": "OS", + "type": "dashboards" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "MySQL" + ], + "targetBlank": false, + "title": "MySQL", + "type": "dashboards" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "MongoDB" + ], + "targetBlank": false, + "title": "MongoDB", + "type": "dashboards" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "PostgreSQL" + ], + "targetBlank": false, + "title": "PostgreSQL", + "type": "dashboards" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "HA" + ], + "targetBlank": false, + "title": "HA", + "type": "dashboards" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "Cloud" + ], + "targetBlank": false, + "title": "Cloud", + "type": "dashboards" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "Insight" + ], + "targetBlank": false, + "title": "Insight", + "type": "dashboards" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "PMM" + ], + "targetBlank": false, + "title": "PMM", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 382, + "panels": [ + + ], + "repeat": null, + "title": "", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 1, + "description": "**MySQL Uptime**\n\nThe amount of time since the last restart of the MySQL server process.", + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 1 + }, + "height": "125px", + "id": 12, + "interval": "$interval", + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "s", + "postfixFontSize": "80%", + "prefix": "", + "prefixFontSize": "80%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "calculatedInterval": "10m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_uptime{instance=\"$host\"}", + "format": "time_series", + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "300,3600", + "title": "MySQL Uptime", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 2, + "description": "**Current QPS**\n\nBased on the queries reported by MySQL's ``SHOW STATUS`` command, it is the number of statements executed by the server within the last second. This variable includes statements executed within stored programs, unlike the Questions variable. It does not count \n``COM_PING`` or ``COM_STATISTICS`` commands.", + "editable": true, + "error": false, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 1 + }, + "height": "125px", + "id": 13, + "interval": "$interval", + "links": [ + { + "targetBlank": true, + "title": "MySQL Server Status Variables", + "type": "absolute", + "url": "https://dev.mysql.com/doc/refman/5.7/en/server-status-variables.html#statvar_Queries" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "80%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "calculatedInterval": "10m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_queries{instance=\"$host\"}[$interval]) or irate(mysql_global_status_queries{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": "35,75", + "title": "Current QPS", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 0, + "description": "**InnoDB Buffer Pool Size**\n\nInnoDB maintains a storage area called the buffer pool for caching data and indexes in memory. Knowing how the InnoDB buffer pool works, and taking advantage of it to keep frequently accessed data in memory, is one of the most important aspects of MySQL tuning. The goal is to keep the working set in memory. In most cases, this should be between 60%-90% of available memory on a dedicated database host, but depends on many factors.", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 1 + }, + "height": "125px", + "id": 51, + "interval": "$interval", + "links": [ + { + "targetBlank": true, + "title": "Tuning the InnoDB Buffer Pool Size", + "type": "absolute", + "url": "https://www.percona.com/blog/2015/06/02/80-ram-tune-innodb_buffer_pool_size/" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "80%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "calculatedInterval": "10m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_variables_innodb_buffer_pool_size{instance=\"$host\"}", + "format": "time_series", + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "90,95", + "title": "InnoDB Buffer Pool Size", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 0, + "description": "**InnoDB Buffer Pool Size % of Total RAM**\n\nInnoDB maintains a storage area called the buffer pool for caching data and indexes in memory. Knowing how the InnoDB buffer pool works, and taking advantage of it to keep frequently accessed data in memory, is one of the most important aspects of MySQL tuning. The goal is to keep the working set in memory. In most cases, this should be between 60%-90% of available memory on a dedicated database host, but depends on many factors.", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 1 + }, + "height": "125px", + "id": 52, + "interval": "$interval", + "links": [ + { + "targetBlank": true, + "title": "Tuning the InnoDB Buffer Pool Size", + "type": "absolute", + "url": "https://www.percona.com/blog/2015/06/02/80-ram-tune-innodb_buffer_pool_size/" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "80%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "maxValue": 100, + "minValue": 0, + "show": true + }, + "tableColumn": "", + "targets": [ + { + "calculatedInterval": "10m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "(mysql_global_variables_innodb_buffer_pool_size{instance=\"$host\"} * 100) / on (instance) node_memory_MemTotal{instance=\"$host\"}", + "format": "time_series", + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": "40,80", + "title": "Buffer Pool Size of Total RAM", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 383, + "panels": [ + + ], + "repeat": null, + "title": "Connections", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "description": "**Max Connections** \n\nMax Connections is the maximum permitted number of simultaneous client connections. By default, this is 151. Increasing this value increases the number of file descriptors that mysqld requires. If the required number of descriptors are not available, the server reduces the value of Max Connections.\n\nmysqld actually permits Max Connections + 1 clients to connect. The extra connection is reserved for use by accounts that have the SUPER privilege, such as root.\n\nMax Used Connections is the maximum number of connections that have been in use simultaneously since the server started.\n\nConnections is the number of connection attempts (successful or not) to the MySQL server.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "height": "250px", + "id": 92, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "MySQL Server System Variables", + "type": "absolute", + "url": "https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_connections" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max Connections", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "max(max_over_time(mysql_global_status_threads_connected{instance=\"$host\"}[$interval]) or mysql_global_status_threads_connected{instance=\"$host\"} )", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Connections", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_max_used_connections{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Max Used Connections", + "metric": "", + "refId": "C", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_variables_max_connections{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Max Connections", + "metric": "", + "refId": "B", + "step": 20, + "target": "" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Active Threads**\n\nThreads Connected is the number of open connections, while Threads Running is the number of threads not sleeping.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Peak Threads Running", + "color": "#E24D42", + "lines": false, + "pointradius": 1, + "points": true + }, + { + "alias": "Peak Threads Connected", + "color": "#1F78C1" + }, + { + "alias": "Avg Threads Running", + "color": "#EAB839" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "max_over_time(mysql_global_status_threads_connected{instance=\"$host\"}[$interval]) or\nmax_over_time(mysql_global_status_threads_connected{instance=\"$host\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peak Threads Connected", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "max_over_time(mysql_global_status_threads_running{instance=\"$host\"}[$interval]) or\nmax_over_time(mysql_global_status_threads_running{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peak Threads Running", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "expr": "avg_over_time(mysql_global_status_threads_running{instance=\"$host\"}[$interval]) or \navg_over_time(mysql_global_status_threads_running{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Avg Threads Running", + "refId": "C", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Client Thread Activity", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "short", + "label": "Threads", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 384, + "panels": [ + + ], + "repeat": null, + "title": "Table Locks", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Questions**\n\nThe number of statements executed by the server. This includes only statements sent to the server by clients and not statements executed within stored programs, unlike the Queries used in the QPS calculation. \n\nThis variable does not count the following commands:\n* ``COM_PING``\n* ``COM_STATISTICS``\n* ``COM_STMT_PREPARE``\n* ``COM_STMT_CLOSE``\n* ``COM_STMT_RESET``", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 53, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "MySQL Queries and Questions", + "type": "absolute", + "url": "https://www.percona.com/blog/2014/05/29/how-mysql-queries-and-questions-are-measured/" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_questions{instance=\"$host\"}[$interval]) or irate(mysql_global_status_questions{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Questions", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Questions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Thread Cache**\n\nThe thread_cache_size variable sets how many threads the server should cache to reuse. When a client disconnects, the client's threads are put in the cache if the cache is not full. It is autosized in MySQL 5.6.8 and above (capped to 100). Requests for threads are satisfied by reusing threads taken from the cache if possible, and only when the cache is empty is a new thread created.\n\n* *Threads_created*: The number of threads created to handle connections.\n* *Threads_cached*: The number of threads in the thread cache.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "title": "Tuning information", + "type": "absolute", + "url": "https://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_thread_cache_size" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Threads Created", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_variables_thread_cache_size{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Thread Cache Size", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_threads_cached{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Threads Cached", + "metric": "", + "refId": "C", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_threads_created{instance=\"$host\"}[$interval]) or irate(mysql_global_status_threads_created{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Threads Created", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Thread Cache", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 385, + "panels": [ + + ], + "repeat": null, + "title": "Temporary Objects", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_created_tmp_tables{instance=\"$host\"}[$interval]) or irate(mysql_global_status_created_tmp_tables{instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Created Tmp Tables", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_created_tmp_disk_tables{instance=\"$host\"}[$interval]) or irate(mysql_global_status_created_tmp_disk_tables{instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Created Tmp Disk Tables", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_created_tmp_files{instance=\"$host\"}[$interval]) or irate(mysql_global_status_created_tmp_files{instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Created Tmp Files", + "metric": "", + "refId": "C", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Temporary Objects", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Select Types**\n\nAs with most relational databases, selecting based on indexes is more efficient than scanning an entire table's data. Here we see the counters for selects not done with indexes.\n\n* ***Select Scan*** is how many queries caused full table scans, in which all the data in the table had to be read and either discarded or returned.\n* ***Select Range*** is how many queries used a range scan, which means MySQL scanned all rows in a given range.\n* ***Select Full Join*** is the number of joins that are not joined on an index, this is usually a huge performance hit.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 20 + }, + "height": "250px", + "id": 311, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_select_full_join{instance=\"$host\"}[$interval]) or irate(mysql_global_status_select_full_join{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Select Full Join", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_select_full_range_join{instance=\"$host\"}[$interval]) or irate(mysql_global_status_select_full_range_join{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Select Full Range Join", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_select_range{instance=\"$host\"}[$interval]) or irate(mysql_global_status_select_range{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Select Range", + "metric": "", + "refId": "C", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_select_range_check{instance=\"$host\"}[$interval]) or irate(mysql_global_status_select_range_check{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Select Range Check", + "metric": "", + "refId": "D", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_select_scan{instance=\"$host\"}[$interval]) or irate(mysql_global_status_select_scan{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Select Scan", + "metric": "", + "refId": "E", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Select Types", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 386, + "panels": [ + + ], + "repeat": null, + "title": "Sorts", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Sorts**\n\nDue to a query's structure, order, or other requirements, MySQL sorts the rows before returning them. For example, if a table is ordered 1 to 10 but you want the results reversed, MySQL then has to sort the rows to return 10 to 1.\n\nThis graph also shows when sorts had to scan a whole table or a given range of a table in order to return the results and which could not have been sorted via an index.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_sort_rows{instance=\"$host\"}[$interval]) or irate(mysql_global_status_sort_rows{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Sort Rows", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_sort_range{instance=\"$host\"}[$interval]) or irate(mysql_global_status_sort_range{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Sort Range", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_sort_merge_passes{instance=\"$host\"}[$interval]) or irate(mysql_global_status_sort_merge_passes{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Sort Merge Passes", + "metric": "", + "refId": "C", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_sort_scan{instance=\"$host\"}[$interval]) or irate(mysql_global_status_sort_scan{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Sort Scan", + "metric": "", + "refId": "D", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Sorts", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Slow Queries**\n\nSlow queries are defined as queries being slower than the long_query_time setting. For example, if you have long_query_time set to 3, all queries that take longer than 3 seconds to complete will show on this graph.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_slow_queries{instance=\"$host\"}[$interval]) or irate(mysql_global_status_slow_queries{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Slow Queries", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Slow Queries", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 387, + "panels": [ + + ], + "repeat": null, + "title": "Aborted", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**Aborted Connections**\n\nWhen a given host connects to MySQL and the connection is interrupted in the middle (for example due to bad credentials), MySQL keeps that info in a system table (since 5.6 this table is exposed in performance_schema).\n\nIf the amount of failed requests without a successful connection reaches the value of max_connect_errors, mysqld assumes that something is wrong and blocks the host from further connection.\n\nTo allow connections from that host again, you need to issue the ``FLUSH HOSTS`` statement.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_aborted_connects{instance=\"$host\"}[$interval]) or irate(mysql_global_status_aborted_connects{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Aborted Connects (attempts)", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_aborted_clients{instance=\"$host\"}[$interval]) or irate(mysql_global_status_aborted_clients{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Aborted Clients (timeout)", + "metric": "", + "refId": "B", + "step": 20, + "target": "" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Aborted Connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**Table Locks**\n\nMySQL takes a number of different locks for varying reasons. In this graph we see how many Table level locks MySQL has requested from the storage engine. In the case of InnoDB, many times the locks could actually be row locks as it only takes table level locks in a few specific cases.\n\nIt is most useful to compare Locks Immediate and Locks Waited. If Locks waited is rising, it means you have lock contention. Otherwise, Locks Immediate rising and falling is normal activity.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_table_locks_immediate{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_locks_immediate{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Table Locks Immediate", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_table_locks_waited{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_locks_waited{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Table Locks Waited", + "metric": "", + "refId": "B", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Table Locks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 388, + "panels": [ + + ], + "repeat": null, + "title": "Network", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Network Traffic**\n\nHere we can see how much network traffic is generated by MySQL. Outbound is network traffic sent from MySQL and Inbound is network traffic MySQL has received.", + "editable": true, + "error": false, + "fill": 6, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_bytes_received{instance=\"$host\"}[$interval]) or irate(mysql_global_status_bytes_received{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inbound", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_bytes_sent{instance=\"$host\"}[$interval]) or irate(mysql_global_status_bytes_sent{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Outbound", + "metric": "", + "refId": "B", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Network Usage Hourly**\n\nHere we can see how much network traffic is generated by MySQL per hour. You can use the bar graph to compare data sent by MySQL and data received by MySQL.", + "editable": true, + "error": false, + "fill": 6, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 44 + }, + "height": "250px", + "id": 381, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "increase(mysql_global_status_bytes_received{instance=\"$host\"}[1h])", + "format": "time_series", + "interval": "1h", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "", + "refId": "A", + "step": 3600 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "increase(mysql_global_status_bytes_sent{instance=\"$host\"}[1h])", + "format": "time_series", + "interval": "1h", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "", + "refId": "B", + "step": 3600 + } + ], + "thresholds": [ + + ], + "timeFrom": "24h", + "timeShift": null, + "title": "MySQL Network Usage Hourly", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 389, + "panels": [ + + ], + "repeat": null, + "title": "Memory", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool, TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB ‘s internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.", + "editable": true, + "error": false, + "fill": 6, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "title": "Detailed descriptions about metrics", + "type": "absolute", + "url": "https://www.percona.com/doc/percona-monitoring-and-management/dashboard.mysql-overview.html#mysql-internal-memory-overview" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "System Memory", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal{instance=\"$host\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "System Memory", + "refId": "G", + "step": 4 + }, + { + "expr": "mysql_global_status_innodb_page_size{instance=\"$host\"} * on (instance) mysql_global_status_buffer_pool_pages{instance=\"$host\",state=\"data\"}", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "InnoDB Buffer Pool Data", + "refId": "A", + "step": 20 + }, + { + "expr": "mysql_global_variables_innodb_log_buffer_size{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "InnoDB Log Buffer Size", + "refId": "D", + "step": 20 + }, + { + "expr": "mysql_global_variables_innodb_additional_mem_pool_size{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "InnoDB Additional Memory Pool Size", + "refId": "H", + "step": 40 + }, + { + "expr": "mysql_global_status_innodb_mem_dictionary{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "InnoDB Dictionary Size", + "refId": "F", + "step": 20 + }, + { + "expr": "mysql_global_variables_key_buffer_size{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Key Buffer Size", + "refId": "B", + "step": 20 + }, + { + "expr": "mysql_global_variables_query_cache_size{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Query Cache Size", + "refId": "C", + "step": 20 + }, + { + "expr": "mysql_global_status_innodb_mem_adaptive_hash{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Adaptive Hash Index Size", + "refId": "E", + "step": 20 + }, + { + "expr": "mysql_global_variables_tokudb_cache_size{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "TokuDB Cache Size", + "refId": "I", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Internal Memory Overview", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 390, + "panels": [ + + ], + "repeat": null, + "title": "Command, Handlers, Processes", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**Top Command Counters**\n\nThe Com_{{xxx}} statement counter variables indicate the number of times each xxx statement has been executed. There is one status variable for each type of statement. For example, Com_delete and Com_update count [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements, respectively. Com_delete_multi and Com_update_multi are similar but apply to [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements that use multiple-table syntax.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "title": "Server Status Variables (Com_xxx)", + "type": "absolute", + "url": "https://dev.mysql.com/doc/refman/5.7/en/server-status-variables.html#statvar_Com_xxx" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "topk(5, rate(mysql_global_status_commands_total{instance=\"$host\"}[$interval])>0) or topk(5, irate(mysql_global_status_commands_total{instance=\"$host\"}[5m])>0)", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Com_{{ command }}", + "metric": "", + "refId": "B", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Top Command Counters", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**Top Command Counters Hourly**\n\nThe Com_{{xxx}} statement counter variables indicate the number of times each xxx statement has been executed. There is one status variable for each type of statement. For example, Com_delete and Com_update count [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements, respectively. Com_delete_multi and Com_update_multi are similar but apply to [``DELETE``](https://dev.mysql.com/doc/refman/5.7/en/delete.html) and [``UPDATE``](https://dev.mysql.com/doc/refman/5.7/en/update.html) statements that use multiple-table syntax.", + "editable": true, + "error": false, + "fill": 6, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [ + { + "dashboard": "https://dev.mysql.com/doc/refman/5.7/en/server-status-variables.html#statvar_Com_xxx", + "title": "Server Status Variables (Com_xxx)", + "type": "absolute", + "url": "https://dev.mysql.com/doc/refman/5.7/en/server-status-variables.html#statvar_Com_xxx" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "topk(5, increase(mysql_global_status_commands_total{instance=\"$host\"}[1h])>0)", + "format": "time_series", + "interval": "1h", + "intervalFactor": 1, + "legendFormat": "Com_{{ command }}", + "metric": "", + "refId": "A", + "step": 3600 + } + ], + "thresholds": [ + + ], + "timeFrom": "24h", + "timeShift": null, + "title": "Top Command Counters Hourly", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Handlers**\n\nHandler statistics are internal statistics on how MySQL is selecting, updating, inserting, and modifying rows, tables, and indexes.\n\nThis is in fact the layer between the Storage Engine and MySQL.\n\n* `read_rnd_next` is incremented when the server performs a full table scan and this is a counter you don't really want to see with a high value.\n* `read_key` is incremented when a read is done with an index.\n* `read_next` is incremented when the storage engine is asked to 'read the next index entry'. A high value means a lot of index scans are being done.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_handlers_total{instance=\"$host\", handler!~\"commit|rollback|savepoint.*|prepare\"}[$interval]) or irate(mysql_global_status_handlers_total{instance=\"$host\", handler!~\"commit|rollback|savepoint.*|prepare\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ handler }}", + "metric": "", + "refId": "J", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Handlers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_handlers_total{instance=\"$host\", handler=~\"commit|rollback|savepoint.*|prepare\"}[$interval]) or irate(mysql_global_status_handlers_total{instance=\"$host\", handler=~\"commit|rollback|savepoint.*|prepare\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ handler }}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Transaction Handlers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_info_schema_threads{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ state }}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Process States", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 6, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "topk(5, avg_over_time(mysql_info_schema_threads{instance=\"$host\"}[1h]))", + "interval": "1h", + "intervalFactor": 1, + "legendFormat": "{{ state }}", + "metric": "", + "refId": "A", + "step": 3600 + } + ], + "thresholds": [ + + ], + "timeFrom": "24h", + "timeShift": null, + "title": "Top Process States Hourly", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 391, + "panels": [ + + ], + "repeat": null, + "title": "Query Cache", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Query Cache Memory**\n\nThe query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time. This serialization is true not only for SELECTs, but also for INSERT/UPDATE/DELETE.\n\nThis also means that the larger the `query_cache_size` is set to, the slower those operations become. In concurrent environments, the MySQL Query Cache quickly becomes a contention point, decreasing performance. MariaDB and AWS Aurora have done work to try and eliminate the query cache contention in their flavors of MySQL, while MySQL 8.0 has eliminated the query cache feature.\n\nThe recommended settings for most environments is to set:\n ``query_cache_type=0``\n ``query_cache_size=0``\n\nNote that while you can dynamically change these values, to completely remove the contention point you have to restart the database.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 103 + }, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_qcache_free_memory{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Free Memory", + "metric": "", + "refId": "F", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_variables_query_cache_size{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Query Cache Size", + "metric": "", + "refId": "E", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Query Cache Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Query Cache Activity**\n\nThe query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time. This serialization is true not only for SELECTs, but also for INSERT/UPDATE/DELETE.\n\nThis also means that the larger the `query_cache_size` is set to, the slower those operations become. In concurrent environments, the MySQL Query Cache quickly becomes a contention point, decreasing performance. MariaDB and AWS Aurora have done work to try and eliminate the query cache contention in their flavors of MySQL, while MySQL 8.0 has eliminated the query cache feature.\n\nThe recommended settings for most environments is to set:\n``query_cache_type=0``\n``query_cache_size=0``\n\nNote that while you can dynamically change these values, to completely remove the contention point you have to restart the database.", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 103 + }, + "height": "", + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_qcache_hits{instance=\"$host\"}[$interval]) or irate(mysql_global_status_qcache_hits{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Hits", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_qcache_inserts{instance=\"$host\"}[$interval]) or irate(mysql_global_status_qcache_inserts{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inserts", + "metric": "", + "refId": "C", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_qcache_not_cached{instance=\"$host\"}[$interval]) or irate(mysql_global_status_qcache_not_cached{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Not Cached", + "metric": "", + "refId": "D", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_qcache_lowmem_prunes{instance=\"$host\"}[$interval]) or irate(mysql_global_status_qcache_lowmem_prunes{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Prunes", + "metric": "", + "refId": "F", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_qcache_queries_in_cache{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Queries in Cache", + "metric": "", + "refId": "E", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Query Cache Activity", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 110 + }, + "id": 392, + "panels": [ + + ], + "repeat": null, + "title": "Files and Tables", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 111 + }, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_opened_files{instance=\"$host\"}[$interval]) or irate(mysql_global_status_opened_files{instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Openings", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL File Openings", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 111 + }, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_open_files{instance=\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Open Files", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_variables_open_files_limit{instance=\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Open Files Limit", + "metric": "", + "refId": "D", + "step": 20 + }, + { + "expr": "mysql_global_status_innodb_num_open_files{instance=\"$host\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "InnoDB Open Files", + "refId": "B", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Open Files", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 118 + }, + "id": 393, + "panels": [ + + ], + "repeat": null, + "title": "Table Openings", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Table Open Cache Status**\n\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\n\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 119 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "title": "Server Status Variables (table_open_cache)", + "type": "absolute", + "url": "http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_table_open_cache" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Table Open Cache Hit Ratio", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(mysql_global_status_opened_tables{instance=\"$host\"}[$interval]) or irate(mysql_global_status_opened_tables{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Openings", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "expr": "rate(mysql_global_status_table_open_cache_hits{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_open_cache_hits{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Hits", + "refId": "B", + "step": 20 + }, + { + "expr": "rate(mysql_global_status_table_open_cache_misses{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_open_cache_misses{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Misses", + "refId": "C", + "step": 20 + }, + { + "expr": "rate(mysql_global_status_table_open_cache_overflows{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_open_cache_overflows{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Misses due to Overflows", + "refId": "D", + "step": 20 + }, + { + "expr": "(rate(mysql_global_status_table_open_cache_hits{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_open_cache_hits{instance=\"$host\"}[5m]))/((rate(mysql_global_status_table_open_cache_hits{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_open_cache_hits{instance=\"$host\"}[5m]))+(rate(mysql_global_status_table_open_cache_misses{instance=\"$host\"}[$interval]) or irate(mysql_global_status_table_open_cache_misses{instance=\"$host\"}[5m])))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Table Open Cache Hit Ratio", + "refId": "E", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Table Open Cache Status", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Open Tables**\n\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\n\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 119 + }, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "title": "Server Status Variables (table_open_cache)", + "type": "absolute", + "url": "http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_table_open_cache" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_open_tables{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Open Tables", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_variables_table_open_cache{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Table Open Cache", + "metric": "", + "refId": "C", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Open Tables", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 126 + }, + "id": 394, + "panels": [ + + ], + "repeat": null, + "title": "MySQL Table Definition Cache", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "description": "**MySQL Table Definition Cache**\n\nThe recommendation is to set the `table_open_cache_instances` to a loose correlation to virtual CPUs, keeping in mind that more instances means the cache is split more times. If you have a cache set to 500 but it has 10 instances, each cache will only have 50 cached.\n\nThe `table_definition_cache` and `table_open_cache` can be left as default as they are auto-sized MySQL 5.6 and above (ie: do not set them to any value).", + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 127 + }, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "title": "Server Status Variables (table_open_cache)", + "type": "absolute", + "url": "http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_table_open_cache" + } + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Opened Table Definitions", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_status_open_table_definitions{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Open Table Definitions", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "mysql_global_variables_table_definition_cache{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Table Definitions Cache Size", + "metric": "", + "refId": "C", + "step": 20 + }, + { + "expr": "rate(mysql_global_status_opened_table_definitions{instance=\"$host\"}[$interval]) or irate(mysql_global_status_opened_table_definitions{instance=\"$host\"}[5m])", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Opened Table Definitions", + "refId": "A", + "step": 20 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "MySQL Table Definition Cache", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 134 + }, + "id": 395, + "panels": [ + + ], + "repeat": null, + "title": "System Charts", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 135 + }, + "id": 31, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(node_vmstat_pgpgin{instance=\"$host\"}[$interval]) * 1024 or irate(node_vmstat_pgpgin{instance=\"$host\"}[5m]) * 1024", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Page In", + "metric": "", + "refId": "A", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(node_vmstat_pgpgout{instance=\"$host\"}[$interval]) * 1024 or irate(node_vmstat_pgpgout{instance=\"$host\"}[5m]) * 1024", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Page Out", + "metric": "", + "refId": "B", + "step": 20, + "target": "" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "I/O Activity", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 6, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 135 + }, + "height": "250px", + "id": 37, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "max(node_memory_MemTotal{instance=\"$host\"}) without(job) - \n(max(node_memory_MemFree{instance=\"$host\"}) without(job) + \nmax(node_memory_Buffers{instance=\"$host\"}) without(job) + \n(max(node_memory_Cached{instance=\"$host\",job=~\"rds-enhanced|linux\"}) without (job) or \nmax(node_memory_Cached{instance=\"$host\",job=\"rds-basic\"}) without (job)))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Used", + "metric": "", + "refId": "A", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "node_memory_MemFree{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Free", + "metric": "", + "refId": "B", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "node_memory_Buffers{instance=\"$host\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Buffers", + "metric": "", + "refId": "D", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "max(node_memory_Cached{instance=~\"$host\",job=~\"rds-enhanced|linux\"}) without (job) or \nmax(node_memory_Cached{instance=~\"$host\",job=~\"rds-basic\"}) without (job)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Cached", + "metric": "", + "refId": "E", + "step": 20, + "target": "" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Distribution", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Load 1m": "#58140C", + "Max Core Utilization": "#bf1b00", + "iowait": "#e24d42", + "nice": "#1f78c1", + "softirq": "#806eb7", + "system": "#eab839", + "user": "#508642" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 6, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 142 + }, + "height": "", + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max Core Utilization", + "lines": false, + "pointradius": 1, + "points": true, + "stack": false + }, + { + "alias": "Load 1m", + "color": "#58140C", + "fill": 2, + "legend": false, + "stack": false, + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "clamp_max(((avg by (mode) ( (clamp_max(rate(node_cpu{instance=\"$host\",mode!=\"idle\"}[$interval]),1)) or (clamp_max(irate(node_cpu{instance=\"$host\",mode!=\"idle\"}[5m]),1)) ))*100 or (avg_over_time(node_cpu_average{instance=~\"$host\", mode!=\"total\", mode!=\"idle\"}[$interval]) or avg_over_time(node_cpu_average{instance=~\"$host\", mode!=\"total\", mode!=\"idle\"}[5m]))),100)", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ mode }}", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "expr": "clamp_max(max by () (sum by (cpu) ( (clamp_max(rate(node_cpu{instance=\"$host\",mode!=\"idle\",mode!=\"iowait\"}[$interval]),1)) or (clamp_max(irate(node_cpu{instance=\"$host\",mode!=\"idle\",mode!=\"iowait\"}[5m]),1)) ))*100,100)", + "format": "time_series", + "hide": true, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Max Core Utilization", + "refId": "B", + "step": 20 + }, + { + "expr": "node_load1{instance=\"$host\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Load 1m", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage / Load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": "", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "none", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 142 + }, + "height": "250px", + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "sum((rate(node_disk_read_time_ms{device!~\"dm-.+\", instance=\"$host\"}[$interval]) / rate(node_disk_reads_completed{device!~\"dm-.+\", instance=\"$host\"}[$interval])) or (irate(node_disk_read_time_ms{device!~\"dm-.+\", instance=\"$host\"}[5m]) / irate(node_disk_reads_completed{device!~\"dm-.+\", instance=\"$host\"}[5m]))\nor avg_over_time(aws_rds_read_latency_average{instance=\"$host\"}[$interval]) or avg_over_time(aws_rds_read_latency_average{instance=\"$host\"}[5m]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read", + "metric": "", + "refId": "A", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "sum((rate(node_disk_write_time_ms{device!~\"dm-.+\", instance=\"$host\"}[$interval]) / rate(node_disk_writes_completed{device!~\"dm-.+\", instance=\"$host\"}[$interval])) or (irate(node_disk_write_time_ms{device!~\"dm-.+\", instance=\"$host\"}[5m]) / irate(node_disk_writes_completed{device!~\"dm-.+\", instance=\"$host\"}[5m])) or \navg_over_time(aws_rds_write_latency_average{instance=\"$host\"}[$interval]) or avg_over_time(aws_rds_write_latency_average{instance=\"$host\"}[5m]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write", + "metric": "", + "refId": "B", + "step": 20, + "target": "" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": "", + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 149 + }, + "height": "250px", + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Outbound", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "sum(rate(node_network_receive_bytes{instance=\"$host\", device!=\"lo\"}[$interval])) or sum(irate(node_network_receive_bytes{instance=\"$host\", device!=\"lo\"}[5m])) or sum(max_over_time(rdsosmetrics_network_rx{instance=\"$host\"}[$interval])) or sum(max_over_time(rdsosmetrics_network_rx{instance=\"$host\"}[5m])) ", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inbound", + "metric": "", + "refId": "B", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "sum(rate(node_network_transmit_bytes{instance=\"$host\", device!=\"lo\"}[$interval])) or sum(irate(node_network_transmit_bytes{instance=\"$host\", device!=\"lo\"}[5m])) or\nsum(max_over_time(rdsosmetrics_network_tx{instance=\"$host\"}[$interval])) or sum(max_over_time(rdsosmetrics_network_tx{instance=\"$host\"}[5m]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Outbound", + "metric": "", + "refId": "A", + "step": 20, + "target": "" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Outbound (-) / Inbound (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 2, + "grid": { + + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 149 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(node_vmstat_pswpin{instance=\"$host\"}[$interval]) * 4096 or irate(node_vmstat_pswpin{instance=\"$host\"}[5m]) * 4096", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Swap In (Reads)", + "metric": "", + "refId": "A", + "step": 20, + "target": "" + }, + { + "calculatedInterval": "2s", + "datasourceErrors": { + + }, + "errors": { + + }, + "expr": "rate(node_vmstat_pswpout{instance=\"$host\"}[$interval]) * 4096 or irate(node_vmstat_pswpout{instance=\"$host\"}[5m]) * 4096", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Swap Out (Writes)", + "metric": "", + "refId": "B", + "step": 20, + "target": "" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Swap Activity", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "1m", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "Percona", + "MySQL" + ], + "templating": { + "list": [ + { + "allFormat": "glob", + "auto": true, + "auto_count": 200, + "auto_min": "1s", + "current": { + "text": "auto", + "value": "$__auto_interval_interval" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "multiFormat": "glob", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,5s,1m,5m,1h,6h,1d", + "refresh": 2, + "type": "interval" + }, + { + "allFormat": "glob", + "allValue": null, + "current": { + + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "multiFormat": "regex values", + "name": "host", + "options": [ + + ], + "query": "label_values(mysql_up, instance)", + "refresh": 1, + "refresh_on_load": false, + "regex": "", + "sort": 1, + "tagValuesQuery": null, + "tags": [ + + ], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "hidden": false, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "MySQL Overview", + "uid": "MQWgroiiz", + "version": 1 +} \ No newline at end of file diff --git a/devenv/docker/ha_test/prometheus/prometheus.yml b/devenv/docker/ha_test/prometheus/prometheus.yml index ea97ba8ba05..0950bad6f7d 100644 --- a/devenv/docker/ha_test/prometheus/prometheus.yml +++ b/devenv/docker/ha_test/prometheus/prometheus.yml @@ -30,10 +30,10 @@ scrape_configs: port: 3000 refresh_interval: 10s - # - job_name: 'mysql' - # dns_sd_configs: - # - names: - # - 'mysqld-exporter' - # type: 'A' - # port: 9104 - # refresh_interval: 10s \ No newline at end of file + - job_name: 'mysql' + dns_sd_configs: + - names: + - 'mysqld-exporter' + type: 'A' + port: 9104 + refresh_interval: 10s \ No newline at end of file diff --git a/devenv/docker/loadtest/README.md b/devenv/docker/loadtest/README.md index 8e724637acb..ca70a77dc74 100644 --- a/devenv/docker/loadtest/README.md +++ b/devenv/docker/loadtest/README.md @@ -8,7 +8,7 @@ Docker ## Run -Run load test for 15 minutes: +Run load test for 15 minutes using 2 virtual users and targeting http://localhost:3000. ```bash $ ./run.sh @@ -20,6 +20,18 @@ Run load test for custom duration: $ ./run.sh -d 10s ``` +Run load test for custom target url: + +```bash +$ ./run.sh -u http://grafana.loc +``` + +Run load test for 10 virtual users: + +```bash +$ ./run.sh -v 10 +``` + Example output: ```bash diff --git a/devenv/docker/loadtest/auth_token_test.js b/devenv/docker/loadtest/auth_token_test.js index e1356fb6f9a..2742f24e0e1 100644 --- a/devenv/docker/loadtest/auth_token_test.js +++ b/devenv/docker/loadtest/auth_token_test.js @@ -65,7 +65,7 @@ export default (data) => { } }); - sleep(1) + sleep(5) } export const teardown = (data) => {} diff --git a/devenv/docker/loadtest/run.sh b/devenv/docker/loadtest/run.sh index 474d75383b6..9517edf5d74 100755 --- a/devenv/docker/loadtest/run.sh +++ b/devenv/docker/loadtest/run.sh @@ -5,8 +5,9 @@ PWD=$(pwd) run() { duration='15m' url='http://localhost:3000' + vus='2' - while getopts ":d:u:" o; do + while getopts ":d:u:v:" o; do case "${o}" in d) duration=${OPTARG} @@ -14,11 +15,14 @@ run() { u) url=${OPTARG} ;; + v) + vus=${OPTARG} + ;; esac done shift $((OPTIND-1)) - docker run -t --network=host -v $PWD:/src -e URL=$url --rm -i loadimpact/k6:master run --vus 2 --duration $duration src/auth_token_test.js + docker run -t --network=host -v $PWD:/src -e URL=$url --rm -i loadimpact/k6:master run --vus $vus --duration $duration src/auth_token_test.js } run "$@" From 170783c292cafd53730f8f365f661f7fd83d124c Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Feb 2019 10:51:35 +0100 Subject: [PATCH 18/19] make hourly cleanup the default behavior --- conf/defaults.ini | 3 --- conf/sample.ini | 3 --- pkg/services/auth/auth_token_test.go | 7 +++---- pkg/services/auth/token_cleanup.go | 3 +-- pkg/setting/setting.go | 10 ++++------ 5 files changed, 8 insertions(+), 18 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index d0fc1133fac..a87aba10adb 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -256,9 +256,6 @@ login_maximum_lifetime_days = 30 # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. token_rotation_interval_minutes = 10 -# How often should expired auth tokens be deleted from the database. The default is each hour. -expired_tokens_cleanup_interval_hours = 1 - # Set to true to disable (hide) the login form, useful if you use OAuth disable_login_form = false diff --git a/conf/sample.ini b/conf/sample.ini index 2ff37239abf..dbbb3593f0f 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -236,9 +236,6 @@ log_queries = # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. ;token_rotation_interval_minutes = 10 -# How often should expired auth tokens be deleted from the database. The default is each hour. -;expired_tokens_cleanup_interval_hours = 1 - # Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false ;disable_login_form = false diff --git a/pkg/services/auth/auth_token_test.go b/pkg/services/auth/auth_token_test.go index 964bd499a01..26dcbc5c868 100644 --- a/pkg/services/auth/auth_token_test.go +++ b/pkg/services/auth/auth_token_test.go @@ -423,10 +423,9 @@ func createTestContext(t *testing.T) *testContext { tokenService := &UserAuthTokenService{ SQLStore: sqlstore, Cfg: &setting.Cfg{ - LoginMaxInactiveLifetimeDays: 7, - LoginMaxLifetimeDays: 30, - TokenRotationIntervalMinutes: 10, - ExpiredTokensCleanupIntervalHours: 1, + LoginMaxInactiveLifetimeDays: 7, + LoginMaxLifetimeDays: 30, + TokenRotationIntervalMinutes: 10, }, log: log.New("test-logger"), } diff --git a/pkg/services/auth/token_cleanup.go b/pkg/services/auth/token_cleanup.go index 0d5cbdaca10..aa5bc4856ab 100644 --- a/pkg/services/auth/token_cleanup.go +++ b/pkg/services/auth/token_cleanup.go @@ -6,8 +6,7 @@ import ( ) func (srv *UserAuthTokenService) Run(ctx context.Context) error { - jobInterval := time.Duration(srv.Cfg.ExpiredTokensCleanupIntervalHours) * time.Hour - ticker := time.NewTicker(jobInterval) + ticker := time.NewTicker(time.Hour) maxInactiveLifetime := time.Duration(srv.Cfg.LoginMaxInactiveLifetimeDays) * 24 * time.Hour maxLifetime := time.Duration(srv.Cfg.LoginMaxLifetimeDays) * 24 * time.Hour diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 43ac46e5ab8..21899482529 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -233,11 +233,10 @@ type Cfg struct { EnterpriseLicensePath string // Auth - LoginCookieName string - LoginMaxInactiveLifetimeDays int - LoginMaxLifetimeDays int - TokenRotationIntervalMinutes int - ExpiredTokensCleanupIntervalHours int + LoginCookieName string + LoginMaxInactiveLifetimeDays int + LoginMaxLifetimeDays int + TokenRotationIntervalMinutes int } type CommandLineArgs struct { @@ -673,7 +672,6 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { if cfg.TokenRotationIntervalMinutes < 2 { cfg.TokenRotationIntervalMinutes = 2 } - cfg.ExpiredTokensCleanupIntervalHours = auth.Key("expired_tokens_cleanup_interval_hours").MustInt(1) DisableLoginForm = auth.Key("disable_login_form").MustBool(false) DisableSignoutMenu = auth.Key("disable_signout_menu").MustBool(false) From 487e7b5ea699e37408890793c1a9975f77d7a4b4 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Feb 2019 11:07:55 +0100 Subject: [PATCH 19/19] removes cleanup setting from docs --- docs/sources/auth/overview.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/sources/auth/overview.md b/docs/sources/auth/overview.md index 0f563fbe8d5..1d0b0d89b3f 100644 --- a/docs/sources/auth/overview.md +++ b/docs/sources/auth/overview.md @@ -63,9 +63,6 @@ login_maximum_lifetime_days = 30 # How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes. token_rotation_interval_minutes = 10 - -# How often should expired auth tokens be deleted from the database. The default is each hour. -expired_tokens_cleanup_interval_hours = 1 ``` ### Anonymous authentication