AuthN: Support HA setups with External Service Account management (#78425)

* Lock when creating external service

* Add local lock back

* Improve function signature

* Define lockName separately to make it more explicit

* Update pkg/infra/serverlock/serverlock.go

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>

* Update pkg/infra/serverlock/serverlock.go

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>

---------

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>

pkg/infra/serverlock/serverlock_integration_test.go

@@ -2,6 +2,7 @@ package serverlock
 
 import (
 	"context"
+	"sync"
 	"testing"
 	"time"
 
@@ -19,11 +20,11 @@ func TestIntegrationServerLock_LockAndExecute(t *testing.T) {
 	atInterval := time.Hour
 	ctx := context.Background()
 
-	//this time `fn` should be executed
+	// this time `fn` should be executed
 	require.Nil(t, sl.LockAndExecute(ctx, "test-operation", atInterval, fn))
 	require.Equal(t, 1, counter)
 
-	//this should not execute `fn`
+	// this should not execute `fn`
 	require.Nil(t, sl.LockAndExecute(ctx, "test-operation", atInterval, fn))
 	require.Nil(t, sl.LockAndExecute(ctx, "test-operation", atInterval, fn))
 	require.Equal(t, 1, counter)
@@ -62,3 +63,65 @@ func TestIntegrationServerLock_LockExecuteAndRelease(t *testing.T) {
 
 	require.Equal(t, 4, counter)
 }
+
+func TestIntegrationServerLock_LockExecuteAndReleaseWithRetries(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	sl := createTestableServerLock(t)
+
+	retries := 0
+	expectedRetries := 10
+	funcRuns := 0
+	fn := func(context.Context) {
+		funcRuns++
+	}
+	lockTimeConfig := LockTimeConfig{
+		MaxInterval: time.Hour,
+		MinWait:     0 * time.Millisecond,
+		MaxWait:     1 * time.Millisecond,
+	}
+	ctx := context.Background()
+	actionName := "test-operation"
+
+	// Acquire lock so that when `LockExecuteAndReleaseWithRetries` runs, it is forced
+	// to retry
+	err := sl.acquireForRelease(ctx, actionName, lockTimeConfig.MaxInterval)
+	require.NoError(t, err)
+
+	wgRetries := sync.WaitGroup{}
+	wgRetries.Add(expectedRetries)
+	wgRelease := sync.WaitGroup{}
+	wgRelease.Add(1)
+	wgCompleted := sync.WaitGroup{}
+	wgCompleted.Add(1)
+
+	onRetryFn := func(int) error {
+		retries++
+		wgRetries.Done()
+		if retries == expectedRetries {
+			// When we reach `expectedRetries`, wait for the lock to be released
+			// to guarantee that next try will succeed
+			wgRelease.Wait()
+		}
+		return nil
+	}
+
+	go func() {
+		err := sl.LockExecuteAndReleaseWithRetries(ctx, actionName, lockTimeConfig, fn, onRetryFn)
+		require.NoError(t, err)
+		wgCompleted.Done()
+	}()
+
+	// Wait to release the lock until `LockExecuteAndReleaseWithRetries` has retried `expectedRetries` times.
+	wgRetries.Wait()
+	err = sl.releaseLock(ctx, actionName)
+	require.NoError(t, err)
+	wgRelease.Done()
+
+	// `LockExecuteAndReleaseWithRetries` has run completely.
+	// Check that it had to retry because the lock was already taken.
+	wgCompleted.Wait()
+	require.Equal(t, expectedRetries, retries)
+	require.Equal(t, 1, funcRuns)
+}