Merge pull request #32 from oviceinc/feat/wait-initial-scaledown

Delay scaling down after rollout
oviceinc · Jul 4, 2024 · b1071f2 · b1071f2
2 parents 61bf3aa + 937a0ec
commit b1071f2
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 2 deletions.
diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go
@@ -186,30 +186,42 @@ var _ = Describe("E2E", func() {
 
 		time.Sleep(10 * time.Second)
 
+		// The old replica set should no be scaled down
+		err = k8sClient.Get(ctx, types.NamespacedName{Name: oldRS.Name, Namespace: namespace}, oldRS)
+		Expect(err).ShouldNot(HaveOccurred())
+		Expect(oldRS.Status.Replicas).Should(Equal(int32(replicas)))
+		klog.Infof("The old ReplicaSet replicas: %d", oldRS.Status.Replicas)
+
+		// Wait 120 seconds
+		time.Sleep(120 * time.Second)
 		// The old replica set should be scaled down
 		err = k8sClient.Get(ctx, types.NamespacedName{Name: oldRS.Name, Namespace: namespace}, oldRS)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(oldRS.Status.Replicas).Should(Equal(int32(replicas - perOnce)))
+		klog.Infof("The old ReplicaSet replicas: %d", oldRS.Status.Replicas)
 
 		// Wait 120 seconds
 		time.Sleep(120 * time.Second)
 		// The old replica set should be scaled down
 		err = k8sClient.Get(ctx, types.NamespacedName{Name: oldRS.Name, Namespace: namespace}, oldRS)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(oldRS.Status.Replicas).Should(Equal(int32(replicas - perOnce*2)))
+		klog.Infof("The old ReplicaSet replicas: %d", oldRS.Status.Replicas)
 
 		// Wait 120 seconds
 		time.Sleep(120 * time.Second)
 		// The old replica set should be scaled down
 		err = k8sClient.Get(ctx, types.NamespacedName{Name: oldRS.Name, Namespace: namespace}, oldRS)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(oldRS.Status.Replicas).Should(Equal(int32(replicas - perOnce*3)))
+		klog.Infof("The old ReplicaSet replicas: %d", oldRS.Status.Replicas)
 
 		// Wait 120 seconds
 		time.Sleep(120 * time.Second)
 		// The old replica set should be scaled down
 		err = k8sClient.Get(ctx, types.NamespacedName{Name: oldRS.Name, Namespace: namespace}, oldRS)
 		Expect(err).ShouldNot(HaveOccurred())
 		Expect(oldRS.Status.Replicas).Should(Equal(int32(replicas - perOnce*4)))
+		klog.Infof("The old ReplicaSet replicas: %d", oldRS.Status.Replicas)
 	})
 })
diff --git a/internal/controller/argorollout_controller.go b/internal/controller/argorollout_controller.go
@@ -94,7 +94,7 @@ func (r *ArgoRolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			}
 			klog.V(1).Infof("Old ReplicaSet: %s/%s", rs.Namespace, rs.Name)
 
-			retry, err := r.scaleDown(ctx, rs, targetScaleDown)
+			retry, err := r.scaleDown(ctx, rs, targetScaleDown, argoRollout)
 			if err != nil {
 				klog.Errorf("Failed to scale down ReplicaSet: %v", err)
 				return ctrl.Result{}, err
@@ -132,7 +132,17 @@ func isCompleted(status *argorolloutsapiv1alpha1.RolloutStatus) bool {
 	return false
 }
 
-func (r *ArgoRolloutReconciler) scaleDown(ctx context.Context, rs *appsv1.ReplicaSet, scaleDown *optimizerv1alpha1.RolloutScaleDown) (time.Duration, error) {
+func (r *ArgoRolloutReconciler) scaleDown(ctx context.Context, rs *appsv1.ReplicaSet, scaleDown *optimizerv1alpha1.RolloutScaleDown, rollout *argorolloutsapiv1alpha1.Rollout) (time.Duration, error) {
+	rolloutUpdated := time.Now()
+	for _, cond := range rollout.Status.Conditions {
+		if cond.Type == argorolloutsapiv1alpha1.RolloutCompleted {
+			rolloutUpdated = cond.LastUpdateTime.Time
+		}
+	}
+	if rolloutUpdated.Add(time.Duration(scaleDown.Spec.CoolTimeSeconds) * time.Second).After(time.Now()) {
+		return 30 * time.Second, nil
+	}
+
 	lastUpdated := scaleDown.Status.LastScaleDownTime
 	if lastUpdated.Add(time.Duration(scaleDown.Spec.CoolTimeSeconds) * time.Second).After(time.Now()) {
 		return 30 * time.Second, nil