Provisioning: update unhealthy status only when not recent (#112014)
This commit is contained in:
@@ -12,6 +12,13 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
const (
|
||||
// recentHealthyDuration defines how recent a health check must be to be considered "recent" when healthy
|
||||
recentHealthyDuration = 5 * time.Minute
|
||||
// recentHealthyDuration defines how recent a health check must be to be considered "recent" when unhealthy
|
||||
recentUnhealthyDuration = 1 * time.Minute
|
||||
)
|
||||
|
||||
// StatusPatcher defines the interface for updating repository status
|
||||
//
|
||||
//go:generate mockery --name=StatusPatcher
|
||||
@@ -60,9 +67,9 @@ func (hc *HealthChecker) hasRecentHealthCheck(healthStatus provisioning.HealthSt
|
||||
|
||||
age := time.Since(time.UnixMilli(healthStatus.Checked))
|
||||
if healthStatus.Healthy {
|
||||
return age <= time.Minute*5 // Recent if checked within 5 minutes when healthy
|
||||
return age <= recentHealthyDuration
|
||||
}
|
||||
return age <= time.Minute // Recent if checked within 1 minute when unhealthy
|
||||
return age <= recentUnhealthyDuration // Recent if checked within 1 minute when unhealthy
|
||||
}
|
||||
|
||||
// HasRecentFailure checks if there's a recent failure of a specific type
|
||||
@@ -72,7 +79,7 @@ func (hc *HealthChecker) HasRecentFailure(healthStatus provisioning.HealthStatus
|
||||
}
|
||||
|
||||
age := time.Since(time.UnixMilli(healthStatus.Checked))
|
||||
return age <= time.Minute // Recent if within 1 minute
|
||||
return age <= recentUnhealthyDuration
|
||||
}
|
||||
|
||||
// RecordFailureAndUpdate records a failure and updates the repository status
|
||||
@@ -111,7 +118,11 @@ func (hc *HealthChecker) hasHealthStatusChanged(old, new provisioning.HealthStat
|
||||
return true
|
||||
}
|
||||
|
||||
if old.Checked != new.Checked {
|
||||
recent := recentUnhealthyDuration
|
||||
if new.Healthy {
|
||||
recent = recentHealthyDuration
|
||||
}
|
||||
if time.UnixMilli(new.Checked).Sub(time.UnixMilli(old.Checked)) > recent {
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -400,6 +400,26 @@ func TestRefreshHealth(t *testing.T) {
|
||||
expectedHealth: true,
|
||||
expectPatch: false,
|
||||
},
|
||||
{
|
||||
name: "no status change - no patch needed for unhealthy repo (recent check)",
|
||||
testResult: &provisioning.TestResults{
|
||||
Success: false,
|
||||
Code: 500,
|
||||
Errors: []provisioning.ErrorDetails{
|
||||
{Detail: "connection failed"},
|
||||
{Detail: "timeout"},
|
||||
},
|
||||
},
|
||||
testError: nil,
|
||||
existingStatus: provisioning.HealthStatus{
|
||||
Healthy: false,
|
||||
Checked: time.Now().Add(-15 * time.Second).UnixMilli(),
|
||||
Message: []string{"connection failed", "timeout"},
|
||||
},
|
||||
expectError: false,
|
||||
expectedHealth: false,
|
||||
expectPatch: false,
|
||||
},
|
||||
{
|
||||
name: "status unchanged but timestamp needs update (old check)",
|
||||
testResult: &provisioning.TestResults{
|
||||
@@ -415,6 +435,26 @@ func TestRefreshHealth(t *testing.T) {
|
||||
expectedHealth: true,
|
||||
expectPatch: true,
|
||||
},
|
||||
{
|
||||
name: "status unchanged but timestamp needs update (old unhealthy check)",
|
||||
testResult: &provisioning.TestResults{
|
||||
Success: false,
|
||||
Code: 500,
|
||||
Errors: []provisioning.ErrorDetails{
|
||||
{Detail: "connection failed"},
|
||||
{Detail: "timeout"},
|
||||
},
|
||||
},
|
||||
testError: nil,
|
||||
existingStatus: provisioning.HealthStatus{
|
||||
Healthy: false,
|
||||
Checked: time.Now().Add(-2 * time.Minute).UnixMilli(),
|
||||
Message: []string{"connection failed", "timeout"},
|
||||
},
|
||||
expectError: false,
|
||||
expectedHealth: false,
|
||||
expectPatch: true,
|
||||
},
|
||||
{
|
||||
name: "patch error",
|
||||
testResult: &provisioning.TestResults{
|
||||
|
||||
Reference in New Issue
Block a user