Provisioning: Fix race condition causing unhealthy repository message to be lost (#115150)
* Fix race condition causing unhealthy repository message to be lost This commit fixes a race condition in the provisioning repository controller where the "Repository is unhealthy" message in the sync status could be lost due to status updates being based on stale repository objects. ## Problem The issue occurred in the `process` function when: 1. Repository object was fetched from cache with old status 2. `RefreshHealth` immediately patched the health status to "unhealthy" 3. `determineSyncStatusOps` used the stale object to check if unhealthy message was already set 4. A second patch operation based on stale data would overwrite the health status update ## Solution Introduced `RefreshHealthWithPatchOps` method that returns patch operations instead of immediately applying them. This allows batching all status updates (health + sync) into a single atomic patch operation, eliminating the race condition. ## Changes - Added `HealthCheckerInterface` for better testability - Added `RefreshHealthWithPatchOps` method to return patch ops without applying - Updated `process` function to batch health and sync status updates - Added comprehensive unit tests for the fix Fixes the issue where unhealthy repositories don't show the "Repository is unhealthy" message in their sync status. * Fix staticcheck lint error: remove unnecessary nil check for slice
This commit is contained in:
Roberto Jiménez Sánchez
@@ -350,6 +350,161 @@ type mockJobsQueueStore struct {
|
||||
*jobs.MockStore
|
||||
}
|
||||
|
||||
func TestRepositoryController_process_UnhealthyRepositoryStatusUpdate(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
repo *provisioning.Repository
|
||||
healthStatus provisioning.HealthStatus
|
||||
hasHealthStatusChanged bool
|
||||
expectedUnhealthyMessage bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "unhealthy repository should set unhealthy message in sync status",
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "default",
|
||||
Generation: 1,
|
||||
},
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Sync: provisioning.SyncOptions{
|
||||
Enabled: true,
|
||||
IntervalSeconds: 300,
|
||||
},
|
||||
},
|
||||
Status: provisioning.RepositoryStatus{
|
||||
ObservedGeneration: 1,
|
||||
Health: provisioning.HealthStatus{
|
||||
Healthy: true,
|
||||
Checked: time.Now().Add(-10 * time.Minute).UnixMilli(),
|
||||
},
|
||||
Sync: provisioning.SyncStatus{
|
||||
State: provisioning.JobStateSuccess,
|
||||
Finished: time.Now().Add(-1 * time.Minute).UnixMilli(),
|
||||
Message: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
healthStatus: provisioning.HealthStatus{
|
||||
Healthy: false,
|
||||
Error: provisioning.HealthFailureHealth,
|
||||
Checked: time.Now().UnixMilli(),
|
||||
Message: []string{"connection failed"},
|
||||
},
|
||||
hasHealthStatusChanged: true,
|
||||
expectedUnhealthyMessage: true,
|
||||
description: "should set unhealthy message when repository becomes unhealthy",
|
||||
},
|
||||
{
|
||||
name: "unhealthy repository should not duplicate unhealthy message",
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "default",
|
||||
Generation: 1,
|
||||
},
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Sync: provisioning.SyncOptions{
|
||||
Enabled: true,
|
||||
IntervalSeconds: 300,
|
||||
},
|
||||
},
|
||||
Status: provisioning.RepositoryStatus{
|
||||
ObservedGeneration: 1,
|
||||
Health: provisioning.HealthStatus{
|
||||
Healthy: false,
|
||||
Checked: time.Now().Add(-2 * time.Minute).UnixMilli(),
|
||||
},
|
||||
Sync: provisioning.SyncStatus{
|
||||
State: provisioning.JobStateError,
|
||||
Finished: time.Now().Add(-1 * time.Minute).UnixMilli(),
|
||||
Message: []string{"Repository is unhealthy"},
|
||||
},
|
||||
},
|
||||
},
|
||||
healthStatus: provisioning.HealthStatus{
|
||||
Healthy: false,
|
||||
Error: provisioning.HealthFailureHealth,
|
||||
Checked: time.Now().UnixMilli(),
|
||||
Message: []string{"connection failed"},
|
||||
},
|
||||
hasHealthStatusChanged: false,
|
||||
expectedUnhealthyMessage: false,
|
||||
description: "should not set unhealthy message when it already exists",
|
||||
},
|
||||
{
|
||||
name: "healthy repository should clear unhealthy message",
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "default",
|
||||
Generation: 1,
|
||||
},
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Sync: provisioning.SyncOptions{
|
||||
Enabled: true,
|
||||
IntervalSeconds: 300,
|
||||
},
|
||||
},
|
||||
Status: provisioning.RepositoryStatus{
|
||||
ObservedGeneration: 1,
|
||||
Health: provisioning.HealthStatus{
|
||||
Healthy: false,
|
||||
Checked: time.Now().Add(-2 * time.Minute).UnixMilli(),
|
||||
},
|
||||
Sync: provisioning.SyncStatus{
|
||||
State: provisioning.JobStateError,
|
||||
Finished: time.Now().Add(-1 * time.Minute).UnixMilli(),
|
||||
Message: []string{"Repository is unhealthy"},
|
||||
},
|
||||
},
|
||||
},
|
||||
healthStatus: provisioning.HealthStatus{
|
||||
Healthy: true,
|
||||
Checked: time.Now().UnixMilli(),
|
||||
Message: []string{},
|
||||
},
|
||||
hasHealthStatusChanged: true,
|
||||
expectedUnhealthyMessage: false,
|
||||
description: "should clear unhealthy message when repository becomes healthy",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Create controller
|
||||
rc := &RepositoryController{}
|
||||
|
||||
// Determine sync status ops (this is a pure function, no mocks needed)
|
||||
syncOps := rc.determineSyncStatusOps(tc.repo, nil, tc.healthStatus)
|
||||
|
||||
// Verify expectations
|
||||
hasUnhealthyOp := false
|
||||
hasClearUnhealthyOp := false
|
||||
for _, op := range syncOps {
|
||||
if path, ok := op["path"].(string); ok {
|
||||
if path == "/status/sync/message" {
|
||||
if messages, ok := op["value"].([]string); ok {
|
||||
if len(messages) > 0 && messages[0] == "Repository is unhealthy" {
|
||||
hasUnhealthyOp = true
|
||||
} else if len(messages) == 0 {
|
||||
hasClearUnhealthyOp = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if tc.expectedUnhealthyMessage {
|
||||
assert.True(t, hasUnhealthyOp, tc.description+": expected unhealthy message operation")
|
||||
} else if len(tc.repo.Status.Sync.Message) > 0 && tc.healthStatus.Healthy {
|
||||
assert.True(t, hasClearUnhealthyOp, tc.description+": expected clear unhealthy message operation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryController_shouldResync_StaleSyncStatus(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
|
||||
Reference in New Issue
Block a user