From d3875212303055087ac2d3167a7eb6f8ec031af4 Mon Sep 17 00:00:00 2001 From: Owen Smallwood Date: Fri, 12 Dec 2025 11:08:28 -0600 Subject: [PATCH] simplify error checking --- pkg/storage/unified/sql/db/dbimpl/db.go | 62 ++++++------------------- 1 file changed, 15 insertions(+), 47 deletions(-) diff --git a/pkg/storage/unified/sql/db/dbimpl/db.go b/pkg/storage/unified/sql/db/dbimpl/db.go index bb934222357..859eb38da3a 100644 --- a/pkg/storage/unified/sql/db/dbimpl/db.go +++ b/pkg/storage/unified/sql/db/dbimpl/db.go @@ -3,7 +3,7 @@ package dbimpl import ( "context" "database/sql" - "strings" + "errors" "time" "github.com/grafana/grafana/pkg/infra/log" @@ -58,27 +58,29 @@ func (d sqlDB) BeginTx(ctx context.Context, opts *sql.TxOptions) (db.Tx, error) conn, err = d.DB.Conn(connCtx) cancel() - if ctx.Err() != nil { - return nil, ctx.Err() + if err == nil { + break } - if !isTransientConnError(err) { + // retry if connection deadline exceeded and attempts remain + if errors.Is(err, context.DeadlineExceeded) && attempt < defaultConnMaxRetries { + d.log.Warn("Timeout when acquiring database connection, retrying", "attempt", attempt, "max_retries", defaultConnMaxRetries) + time.Sleep(defaultConnRetryBackoff) + continue + } + + if errors.Is(err, context.DeadlineExceeded) { + d.log.Error("Timeout exceeded while trying to acquire database connection", "attempt", attempt, "max_retries", defaultConnMaxRetries) return nil, err } - d.log.Warn("Transient database connection error, retrying", "attempt", attempt, "max_retries", defaultConnMaxRetries, "error", err) - - if attempt < defaultConnMaxRetries { - time.Sleep(defaultConnRetryBackoff) - } + return nil, err } // once we have a connection, begin the transaction + tx, err = conn.BeginTx(ctx, opts) if err == nil { - tx, err = conn.BeginTx(ctx, opts) - if err == nil { - return sqlTx{tx}, nil - } + return sqlTx{tx}, nil } return nil, err @@ -104,37 +106,3 @@ func (tx sqlTx) QueryRowContext(ctx context.Context, query string, args ...any) // and the parameters are passed as arguments." return tx.Tx.QueryRowContext(ctx, query, args...) } - -// isTransientConnError checks if an error is a transient connection error that should be retried. -// This includes TCP timeouts, connection refused, and other network-related errors that can occur -// during Cloud SQL proxy pod churn or similar infrastructure events. -func isTransientConnError(err error) bool { - if err == nil { - return false - } - - errStr := err.Error() - - // TCP/network errors - if strings.Contains(errStr, "i/o timeout") || - strings.Contains(errStr, "connection refused") || - strings.Contains(errStr, "connection reset") || - strings.Contains(errStr, "broken pipe") || - strings.Contains(errStr, "no such host") || - strings.Contains(errStr, "network is unreachable") { - return true - } - - // Context deadline exceeded from our attempt timeout - if strings.Contains(errStr, "context deadline exceeded") { - return true - } - - // MySQL specific errors - if strings.Contains(errStr, "bad connection") || - strings.Contains(errStr, "invalid connection") { - return true - } - - return false -}