Provisioning: update unhealthy status only when not recent (#112014)

This commit is contained in:
Daniele Stefano Ferru
2025-10-03 20:49:44 +02:00
committed by GitHub
parent 2424b1cee9
commit 0c0c66fda1
2 changed files with 55 additions and 4 deletions
@@ -12,6 +12,13 @@ import (
"github.com/prometheus/client_golang/prometheus"
)
const (
// recentHealthyDuration defines how recent a health check must be to be considered "recent" when healthy
recentHealthyDuration = 5 * time.Minute
// recentHealthyDuration defines how recent a health check must be to be considered "recent" when unhealthy
recentUnhealthyDuration = 1 * time.Minute
)
// StatusPatcher defines the interface for updating repository status
//
//go:generate mockery --name=StatusPatcher
@@ -60,9 +67,9 @@ func (hc *HealthChecker) hasRecentHealthCheck(healthStatus provisioning.HealthSt
age := time.Since(time.UnixMilli(healthStatus.Checked))
if healthStatus.Healthy {
return age <= time.Minute*5 // Recent if checked within 5 minutes when healthy
return age <= recentHealthyDuration
}
return age <= time.Minute // Recent if checked within 1 minute when unhealthy
return age <= recentUnhealthyDuration // Recent if checked within 1 minute when unhealthy
}
// HasRecentFailure checks if there's a recent failure of a specific type
@@ -72,7 +79,7 @@ func (hc *HealthChecker) HasRecentFailure(healthStatus provisioning.HealthStatus
}
age := time.Since(time.UnixMilli(healthStatus.Checked))
return age <= time.Minute // Recent if within 1 minute
return age <= recentUnhealthyDuration
}
// RecordFailureAndUpdate records a failure and updates the repository status
@@ -111,7 +118,11 @@ func (hc *HealthChecker) hasHealthStatusChanged(old, new provisioning.HealthStat
return true
}
if old.Checked != new.Checked {
recent := recentUnhealthyDuration
if new.Healthy {
recent = recentHealthyDuration
}
if time.UnixMilli(new.Checked).Sub(time.UnixMilli(old.Checked)) > recent {
return true
}
@@ -400,6 +400,26 @@ func TestRefreshHealth(t *testing.T) {
expectedHealth: true,
expectPatch: false,
},
{
name: "no status change - no patch needed for unhealthy repo (recent check)",
testResult: &provisioning.TestResults{
Success: false,
Code: 500,
Errors: []provisioning.ErrorDetails{
{Detail: "connection failed"},
{Detail: "timeout"},
},
},
testError: nil,
existingStatus: provisioning.HealthStatus{
Healthy: false,
Checked: time.Now().Add(-15 * time.Second).UnixMilli(),
Message: []string{"connection failed", "timeout"},
},
expectError: false,
expectedHealth: false,
expectPatch: false,
},
{
name: "status unchanged but timestamp needs update (old check)",
testResult: &provisioning.TestResults{
@@ -415,6 +435,26 @@ func TestRefreshHealth(t *testing.T) {
expectedHealth: true,
expectPatch: true,
},
{
name: "status unchanged but timestamp needs update (old unhealthy check)",
testResult: &provisioning.TestResults{
Success: false,
Code: 500,
Errors: []provisioning.ErrorDetails{
{Detail: "connection failed"},
{Detail: "timeout"},
},
},
testError: nil,
existingStatus: provisioning.HealthStatus{
Healthy: false,
Checked: time.Now().Add(-2 * time.Minute).UnixMilli(),
Message: []string{"connection failed", "timeout"},
},
expectError: false,
expectedHealth: false,
expectPatch: true,
},
{
name: "patch error",
testResult: &provisioning.TestResults{