SQLStore: Optionally retry queries if sqlite returns database is locked (#56096)

* SQLStore: Retry queries if sqlite returns database is locked

* Configurable retries

* Apply suggestions from code review

Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>

pkg/services/sqlstore/session.go

@@ -2,11 +2,14 @@ package sqlstore
 
 import (
 	"context"
+	"errors"
 	"reflect"
+	"time"
 
 	"xorm.io/xorm"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/mattn/go-sqlite3"
 )
 
 var sessionLogger = log.New("sqlstore.session")
@@ -55,18 +58,37 @@ func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTr
 // WithDbSession calls the callback with the session in the context (if exists).
 // Otherwise it creates a new one that is closed upon completion.
 // A session is stored in the context if sqlstore.InTransaction() has been been previously called with the same context (and it's not committed/rolledback yet).
+// In case of sqlite3.ErrLocked or sqlite3.ErrBusy failure it will be retried at most five times before giving up.
 func (ss *SQLStore) WithDbSession(ctx context.Context, callback DBTransactionFunc) error {
-	return withDbSession(ctx, ss.engine, callback)
+	return ss.withDbSession(ctx, ss.engine, callback)
 }
 
 // WithNewDbSession calls the callback with a new session that is closed upon completion.
+// In case of sqlite3.ErrLocked or sqlite3.ErrBusy failure it will be retried at most five times before giving up.
 func (ss *SQLStore) WithNewDbSession(ctx context.Context, callback DBTransactionFunc) error {
 	sess := &DBSession{Session: ss.engine.NewSession(), transactionOpen: false}
 	defer sess.Close()
-	return callback(sess)
+	return ss.withRetry(ctx, callback, 0)(sess)
 }
 
-func withDbSession(ctx context.Context, engine *xorm.Engine, callback DBTransactionFunc) error {
+func (ss *SQLStore) withRetry(ctx context.Context, callback DBTransactionFunc, retry int) DBTransactionFunc {
+	return func(sess *DBSession) error {
+		err := callback(sess)
+
+		ctxLogger := tsclogger.FromContext(ctx)
+
+		var sqlError sqlite3.Error
+		if errors.As(err, &sqlError) && retry < ss.dbCfg.QueryRetries && (sqlError.Code == sqlite3.ErrLocked || sqlError.Code == sqlite3.ErrBusy) {
+			time.Sleep(time.Millisecond * time.Duration(10))
+			ctxLogger.Info("Database locked, sleeping then retrying", "error", err, "retry", retry, "code", sqlError.Code)
+			return ss.withRetry(ctx, callback, retry+1)(sess)
+		}
+
+		return err
+	}
+}
+
+func (ss *SQLStore) withDbSession(ctx context.Context, engine *xorm.Engine, callback DBTransactionFunc) error {
 	sess, isNew, err := startSessionOrUseExisting(ctx, engine, false)
 	if err != nil {
 		return err
@@ -74,7 +96,7 @@ func withDbSession(ctx context.Context, engine *xorm.Engine, callback DBTransact
 	if isNew {
 		defer sess.Close()
 	}
-	return callback(sess)
+	return ss.withRetry(ctx, callback, 0)(sess)
 }
 
 func (sess *DBSession) InsertId(bean interface{}) (int64, error) {