Auth: Update oauthtoken service to use remote cache and server lock (#90572)

* update oauthtoken service to use remote cache and server lock

* remove token cache

* retry is lock is held by an in-flight refresh

* refactor token renewal to avoid race condition

* re-add refresh token expiry cache, but in SyncOauthTokenHook

* Add delta to the cache ttl

* Fix merge

* Change lockTimeConfig

* Always set the token from within the server lock

* Improvements

* early return when user is not authed by OAuth or refresh is disabled

* Allow more time for token refresh, tracing

* Retry on Mysql Deadlock error 1213

* Update pkg/services/authn/authnimpl/sync/oauth_token_sync.go

Co-authored-by: Dan Cech <dcech@grafana.com>

* Update pkg/services/authn/authnimpl/sync/oauth_token_sync.go

Co-authored-by: Dan Cech <dcech@grafana.com>

* Add settings for configuring min wait time between retries

* Add docs for the new setting

* Clean up

* Update docs/sources/setup-grafana/configure-grafana/_index.md

Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>

---------

Co-authored-by: Mihaly Gyongyosi <mgyongyosi@users.noreply.github.com>
Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>

pkg/setting/setting.go

@@ -258,9 +258,10 @@ type Cfg struct {
 	AuthProxy AuthProxySettings
 
 	// OAuth
-	OAuthAutoLogin                bool
-	OAuthCookieMaxAge             int
-	OAuthAllowInsecureEmailLookup bool
+	OAuthAutoLogin                       bool
+	OAuthCookieMaxAge                    int
+	OAuthAllowInsecureEmailLookup        bool
+	OAuthRefreshTokenServerLockMinWaitMs int64
 
 	JWTAuth    AuthJWTSettings
 	ExtJWTAuth ExtJWTSettings
@@ -1603,6 +1604,7 @@ func readAuthSettings(iniFile *ini.File, cfg *Cfg) (err error) {
 	}
 
 	cfg.OAuthCookieMaxAge = auth.Key("oauth_state_cookie_max_age").MustInt(600)
+	cfg.OAuthRefreshTokenServerLockMinWaitMs = auth.Key("oauth_refresh_token_server_lock_min_wait_ms").MustInt64(1000)
 	cfg.SignoutRedirectUrl = valueAsString(auth, "signout_redirect_url", "")
 
 	// Deprecated