diff --git a/.github/workflows/e2e-advanced-deployment-1.19.yaml b/.github/workflows/e2e-advanced-deployment-1.19.yaml index 1b04da6e..0473a57f 100644 --- a/.github/workflows/e2e-advanced-deployment-1.19.yaml +++ b/.github/workflows/e2e-advanced-deployment-1.19.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" @@ -119,6 +120,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-advanced-deployment-1.23.yaml b/.github/workflows/e2e-advanced-deployment-1.23.yaml index 10edb25e..459395f0 100644 --- a/.github/workflows/e2e-advanced-deployment-1.23.yaml +++ b/.github/workflows/e2e-advanced-deployment-1.23.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" @@ -119,6 +120,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-cloneset-1.19.yaml b/.github/workflows/e2e-cloneset-1.19.yaml index 04e0a8eb..aaf1325d 100644 --- a/.github/workflows/e2e-cloneset-1.19.yaml +++ b/.github/workflows/e2e-cloneset-1.19.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-cloneset-1.23.yaml b/.github/workflows/e2e-cloneset-1.23.yaml index 2bc0b0c2..0926666d 100644 --- a/.github/workflows/e2e-cloneset-1.23.yaml +++ b/.github/workflows/e2e-cloneset-1.23.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-custom.yaml b/.github/workflows/e2e-custom.yaml index 4803b098..97848588 100644 --- a/.github/workflows/e2e-custom.yaml +++ b/.github/workflows/e2e-custom.yaml @@ -76,6 +76,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-daemonset-1.19.yaml b/.github/workflows/e2e-daemonset-1.19.yaml index 15c868d3..7656fa6c 100644 --- a/.github/workflows/e2e-daemonset-1.19.yaml +++ b/.github/workflows/e2e-daemonset-1.19.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-daemonset-1.23.yaml b/.github/workflows/e2e-daemonset-1.23.yaml index 75ee88f2..07645914 100644 --- a/.github/workflows/e2e-daemonset-1.23.yaml +++ b/.github/workflows/e2e-daemonset-1.23.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-deployment-1.19.yaml b/.github/workflows/e2e-deployment-1.19.yaml index c67f45bd..b293aec2 100644 --- a/.github/workflows/e2e-deployment-1.19.yaml +++ b/.github/workflows/e2e-deployment-1.19.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-deployment-1.23.yaml b/.github/workflows/e2e-deployment-1.23.yaml index 59ffe444..f39cd51a 100644 --- a/.github/workflows/e2e-deployment-1.23.yaml +++ b/.github/workflows/e2e-deployment-1.23.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-gateway.yaml b/.github/workflows/e2e-gateway.yaml index 457ff5f3..2a324f71 100644 --- a/.github/workflows/e2e-gateway.yaml +++ b/.github/workflows/e2e-gateway.yaml @@ -74,6 +74,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-others-1.19.yaml b/.github/workflows/e2e-others-1.19.yaml index 8caa8c9f..b91fd62a 100644 --- a/.github/workflows/e2e-others-1.19.yaml +++ b/.github/workflows/e2e-others-1.19.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-others-1.23.yaml b/.github/workflows/e2e-others-1.23.yaml index 6fbfdab5..590ac334 100644 --- a/.github/workflows/e2e-others-1.23.yaml +++ b/.github/workflows/e2e-others-1.23.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-statefulset-1.19.yaml b/.github/workflows/e2e-statefulset-1.19.yaml index 571c82e0..857cf786 100644 --- a/.github/workflows/e2e-statefulset-1.19.yaml +++ b/.github/workflows/e2e-statefulset-1.19.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-statefulset-1.23.yaml b/.github/workflows/e2e-statefulset-1.23.yaml index 7045b962..3594962c 100644 --- a/.github/workflows/e2e-statefulset-1.23.yaml +++ b/.github/workflows/e2e-statefulset-1.23.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-v1beta1-1.19.yaml b/.github/workflows/e2e-v1beta1-1.19.yaml index 6ec0e39d..18973265 100644 --- a/.github/workflows/e2e-v1beta1-1.19.yaml +++ b/.github/workflows/e2e-v1beta1-1.19.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/.github/workflows/e2e-v1beta1-1.23.yaml b/.github/workflows/e2e-v1beta1-1.23.yaml index 8dad2a8b..7c11575a 100644 --- a/.github/workflows/e2e-v1beta1-1.23.yaml +++ b/.github/workflows/e2e-v1beta1-1.23.yaml @@ -101,6 +101,7 @@ jobs: restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') if [ "${restartCount}" -eq "0" ];then echo "Kruise-rollout has not restarted" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout else kubectl get pod -n kruise-rollout --no-headers echo "Kruise-rollout has restarted, abort!!!" diff --git a/test/e2e/deployment_test.go b/test/e2e/deployment_test.go index b857f929..96365c25 100644 --- a/test/e2e/deployment_test.go +++ b/test/e2e/deployment_test.go @@ -26,6 +26,16 @@ import ( var _ = SIGDescribe("Advanced Deployment", func() { var namespace string + + DumpAllResources := func() { + deploy := &apps.DeploymentList{} + k8sClient.List(context.TODO(), deploy, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(deploy)) + rs := &apps.ReplicaSetList{} + k8sClient.List(context.TODO(), rs, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rs)) + } + defaultRetry := wait.Backoff{ Steps: 10, Duration: 10 * time.Millisecond, @@ -132,7 +142,12 @@ var _ = SIGDescribe("Advanced Deployment", func() { CheckReplicas := func(deployment *apps.Deployment, replicas, available, updated int32) { var clone *apps.Deployment + start := time.Now() Eventually(func() bool { + if start.Add(time.Minute * 3).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } clone = &apps.Deployment{} err := GetObject(deployment.Namespace, deployment.Name, clone) Expect(err).NotTo(HaveOccurred()) @@ -239,6 +254,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { deployment.Namespace = namespace Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithCheck(deployment, intstr.FromInt(1)) @@ -255,6 +271,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) deployment.Spec.Replicas = pointer.Int32(10) CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromString("0%")) UpdatePartitionWithCheck(deployment, intstr.FromString("40%")) @@ -287,6 +304,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { `{"rollingStyle":"Partition","rollingUpdate":{"maxUnavailable":1,"maxSurge":0}}` deployment.Spec.MinReadySeconds = 10 CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithoutCheck(deployment, intstr.FromInt(3)) @@ -303,6 +321,7 @@ var _ = SIGDescribe("Advanced Deployment", func() { deployment.Namespace = namespace Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithCheck(deployment, intstr.FromInt(2)) @@ -317,7 +336,9 @@ var _ = SIGDescribe("Advanced Deployment", func() { deployment.Namespace = namespace Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2") + // error: stuck in (0,0,0) UpdatePartitionWithCheck(deployment, intstr.FromInt(0)) UpdatePartitionWithCheck(deployment, intstr.FromInt(2)) UpdateDeployment(deployment, "version3") @@ -335,8 +356,10 @@ var _ = SIGDescribe("Advanced Deployment", func() { Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred()) deployment.Annotations["rollouts.kruise.io/deployment-strategy"] = `{"rollingUpdate":{"maxUnavailable":0,"maxSurge":1}}` CreateObject(deployment) + CheckReplicas(deployment, 5, 5, 5) UpdateDeployment(deployment, "version2", "busybox:not-exists") UpdatePartitionWithoutCheck(deployment, intstr.FromInt(1)) + // error: stuck in (5,0,5) CheckReplicas(deployment, 6, 5, 1) UpdateDeployment(deployment, "version3", "busybox:1.32") CheckReplicas(deployment, 5, 5, 0) diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index c8a7cae2..0f0a3593 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -215,7 +215,7 @@ var _ = SIGDescribe("Rollout", func() { body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, v1alpha1.CanaryStepStateReady) Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) return false - }, 10*time.Second, time.Second).Should(BeTrue()) + }, 10*time.Second, 100*time.Microsecond).Should(BeTrue()) } WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) { @@ -4850,7 +4850,12 @@ var _ = SIGDescribe("Rollout", func() { By("wait step(1) pause") WaitRolloutCanaryStepPaused(rollout.Name, 1) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) - + /* + STEP: wait step(2) pause + resume rollout success, and CurrentStepState {"observedGeneration":1,"canaryStatus":{"observedWorkloadGeneration":4,"observedRolloutID":"1","rolloutHash":"9bdxfz594524d9dcddvwv54bdxdv7dw278c657f7fb44wzczx4xb8bw97b4646zx","stableRevision":"774c56dc98","canaryRevision":"774c56dc98","podTemplateHash":"774c56dc98","canaryReplicas":5,"canaryReadyReplicas":5,"nextStepIndex":4,"currentStepIndex":3,"currentStepState":"StepUpgrade","message":"BatchRelease is at state Upgrading, rollout-id 1, step 3","lastUpdateTime":"2024-05-31T03:03:14Z","finalisingStep":""},"conditions":[{"type":"Progressing","status":"True","lastUpdateTime":"2024-05-31T03:03:06Z","lastTransitionTime":"2024-05-31T03:03:06Z","reason":"InRolling","message":"Rollout is in Progressing"}],"phase":"Progressing","message":"Rollout is in step(3/5), and upgrade workload to new version"} + I0531 03:03:15.927607 27074 rollout_test.go:288] current step:3 target step:2 current step state StepUpgrade + I0531 03:03:17.015278 27074 rollout_test.go:288] current step:3 target step:2 current step state StepPaused + */ By("wait step(2) pause") ResumeRolloutCanary(rollout.Name) WaitRolloutCanaryStepPaused(rollout.Name, 2) @@ -5598,7 +5603,12 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) return object.Status.CanaryStatus.CanaryReadyReplicas == 1 }, 5*time.Minute, time.Second).Should(BeTrue()) - + /* + STEP: rolling deployment to be completed + resume rollout success, and CurrentStepState {"observedGeneration":1,"canaryStatus":{"observedWorkloadGeneration":13,"rolloutHash":"bbw699f54445f9b4zv825d8wwvw992z649vv2fz7946f8d52694zvzd4dvf4v4x5","stableRevision":"567694c97d","canaryRevision":"bcb64648f","podTemplateHash":"bcb64648f","canaryReplicas":1,"canaryReadyReplicas":1,"nextStepIndex":3,"currentStepIndex":2,"currentStepState":"StepReady","message":"BatchRelease is at state Verifying, rollout-id , step 1","lastUpdateTime":"2024-06-04T03:15:42Z","finalisingStep":""},"conditions":[{"type":"Progressing","status":"True","lastUpdateTime":"2024-06-04T03:15:26Z","lastTransitionTime":"2024-06-04T03:15:26Z","reason":"InRolling","message":"Rollout is in Progressing"}],"phase":"Progressing","message":"Rollout is in step(1/3), and you need manually confirm to enter the next step"} + I0604 03:15:43.498181 27389 rollout_test.go:288] current step:2 target step:2 current step state StepUpgrade + I0604 03:15:44.501250 27389 rollout_test.go:288] current step:3 target step:2 current step state StepUpgrade + */ By("rolling deployment to be completed") ResumeRolloutCanary(rollout.Name) WaitRolloutCanaryStepPaused(rollout.Name, 2)