Advisor: Reduce and spread load (#105012)

apps/advisor/pkg/app/checkscheduler/checkscheduler.go

@@ -3,6 +3,7 @@ package checkscheduler
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"sort"
 	"strconv"
 	"time"
@@ -18,7 +19,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const defaultEvaluationInterval = 24 * time.Hour
+const defaultEvaluationInterval = 7 * 24 * time.Hour // 7 days
 const defaultMaxHistory = 10
 
 // Runner is a "runnable" app used to be able to expose and API endpoint
@@ -90,11 +91,7 @@ func (r *Runner) Run(ctx context.Context) error {
 		}
 	}
 
-	nextSendInterval := time.Until(lastCreated.Add(r.evaluationInterval))
-	if nextSendInterval < time.Minute {
-		nextSendInterval = 1 * time.Minute
-	}
-
+	nextSendInterval := getNextSendInterval(lastCreated, r.evaluationInterval)
 	ticker := time.NewTicker(nextSendInterval)
 	defer ticker.Stop()
 
@@ -217,6 +214,17 @@ func getEvaluationInterval(pluginConfig map[string]string) (time.Duration, error
 	return evaluationInterval, nil
 }
 
+func getNextSendInterval(lastCreated time.Time, evaluationInterval time.Duration) time.Duration {
+	nextSendInterval := time.Until(lastCreated.Add(evaluationInterval))
+	// Add random variation of one hour
+	randomVariation := time.Duration(rand.Int63n(time.Hour.Nanoseconds()))
+	nextSendInterval += randomVariation
+	if nextSendInterval < time.Minute {
+		nextSendInterval = 1 * time.Minute
+	}
+	return nextSendInterval
+}
+
 func getMaxHistory(pluginConfig map[string]string) (int, error) {
 	maxHistory := defaultMaxHistory
 	configMaxHistory, ok := pluginConfig["max_history"]

apps/advisor/pkg/app/checkscheduler/checkscheduler_test.go

@@ -214,7 +214,7 @@ func Test_getEvaluationInterval(t *testing.T) {
 	t.Run("default", func(t *testing.T) {
 		interval, err := getEvaluationInterval(map[string]string{})
 		assert.NoError(t, err)
-		assert.Equal(t, 24*time.Hour, interval)
+		assert.Equal(t, 7*24*time.Hour, interval)
 	})
 
 	t.Run("invalid", func(t *testing.T) {
@@ -285,6 +285,17 @@ func Test_markUnprocessedChecksAsErrored(t *testing.T) {
 	assert.Equal(t, expectedAnnotations, patchOperation.Value)
 }
 
+func Test_getNextSendInterval(t *testing.T) {
+	lastCreated := time.Now().Add(-7 * 24 * time.Hour)
+	evaluationInterval := 7 * 24 * time.Hour
+	nextSendInterval := getNextSendInterval(lastCreated, evaluationInterval)
+	// The next send interval should be in < 1 hour
+	assert.True(t, nextSendInterval < time.Hour)
+	// Calculate the next send interval again and it should be different
+	nextSendInterval2 := getNextSendInterval(lastCreated, evaluationInterval)
+	assert.NotEqual(t, nextSendInterval, nextSendInterval2)
+}
+
 type MockCheckService struct {
 	checks []checks.Check
 }