fix e2e bugs

Signed-off-by: yunbo <[email protected]>
openkruise · Jun 4, 2024 · b6eaa02 · b6eaa02
1 parent 9836ca6
commit b6eaa02
Show file tree

Hide file tree

Showing 18 changed files with 54 additions and 3 deletions.
diff --git a/.github/workflows/e2e-advanced-deployment-1.19.yaml b/.github/workflows/e2e-advanced-deployment-1.19.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"
@@ -119,6 +120,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-advanced-deployment-1.23.yaml b/.github/workflows/e2e-advanced-deployment-1.23.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"
@@ -119,6 +120,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-cloneset-1.19.yaml b/.github/workflows/e2e-cloneset-1.19.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-cloneset-1.23.yaml b/.github/workflows/e2e-cloneset-1.23.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-custom.yaml b/.github/workflows/e2e-custom.yaml
@@ -76,6 +76,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-daemonset-1.19.yaml b/.github/workflows/e2e-daemonset-1.19.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-daemonset-1.23.yaml b/.github/workflows/e2e-daemonset-1.23.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-deployment-1.19.yaml b/.github/workflows/e2e-deployment-1.19.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-deployment-1.23.yaml b/.github/workflows/e2e-deployment-1.23.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-gateway.yaml b/.github/workflows/e2e-gateway.yaml
@@ -74,6 +74,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-others-1.19.yaml b/.github/workflows/e2e-others-1.19.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-others-1.23.yaml b/.github/workflows/e2e-others-1.23.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-statefulset-1.19.yaml b/.github/workflows/e2e-statefulset-1.19.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-statefulset-1.23.yaml b/.github/workflows/e2e-statefulset-1.23.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-v1beta1-1.19.yaml b/.github/workflows/e2e-v1beta1-1.19.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/.github/workflows/e2e-v1beta1-1.23.yaml b/.github/workflows/e2e-v1beta1-1.23.yaml
@@ -101,6 +101,7 @@ jobs:
           restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
               echo "Kruise-rollout has not restarted"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
           else
               kubectl get pod -n kruise-rollout --no-headers
               echo "Kruise-rollout has restarted, abort!!!"

diff --git a/test/e2e/deployment_test.go b/test/e2e/deployment_test.go
@@ -26,6 +26,16 @@ import (
 
 var _ = SIGDescribe("Advanced Deployment", func() {
 	var namespace string
+
+	DumpAllResources := func() {
+		deploy := &apps.DeploymentList{}
+		k8sClient.List(context.TODO(), deploy, client.InNamespace(namespace))
+		fmt.Println(util.DumpJSON(deploy))
+		rs := &apps.ReplicaSetList{}
+		k8sClient.List(context.TODO(), rs, client.InNamespace(namespace))
+		fmt.Println(util.DumpJSON(rs))
+	}
+
 	defaultRetry := wait.Backoff{
 		Steps:    10,
 		Duration: 10 * time.Millisecond,
@@ -132,7 +142,12 @@ var _ = SIGDescribe("Advanced Deployment", func() {
 
 	CheckReplicas := func(deployment *apps.Deployment, replicas, available, updated int32) {
 		var clone *apps.Deployment
+		start := time.Now()
 		Eventually(func() bool {
+			if start.Add(time.Minute * 3).Before(time.Now()) {
+				DumpAllResources()
+				Expect(true).Should(BeFalse())
+			}
 			clone = &apps.Deployment{}
 			err := GetObject(deployment.Namespace, deployment.Name, clone)
 			Expect(err).NotTo(HaveOccurred())
@@ -239,6 +254,7 @@ var _ = SIGDescribe("Advanced Deployment", func() {
 			deployment.Namespace = namespace
 			Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred())
 			CreateObject(deployment)
+			CheckReplicas(deployment, 5, 5, 5)
 			UpdateDeployment(deployment, "version2")
 			UpdatePartitionWithCheck(deployment, intstr.FromInt(0))
 			UpdatePartitionWithCheck(deployment, intstr.FromInt(1))
@@ -255,6 +271,7 @@ var _ = SIGDescribe("Advanced Deployment", func() {
 			Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred())
 			deployment.Spec.Replicas = pointer.Int32(10)
 			CreateObject(deployment)
+			CheckReplicas(deployment, 5, 5, 5)
 			UpdateDeployment(deployment, "version2")
 			UpdatePartitionWithCheck(deployment, intstr.FromString("0%"))
 			UpdatePartitionWithCheck(deployment, intstr.FromString("40%"))
@@ -287,6 +304,7 @@ var _ = SIGDescribe("Advanced Deployment", func() {
 				`{"rollingStyle":"Partition","rollingUpdate":{"maxUnavailable":1,"maxSurge":0}}`
 			deployment.Spec.MinReadySeconds = 10
 			CreateObject(deployment)
+			CheckReplicas(deployment, 5, 5, 5)
 			UpdateDeployment(deployment, "version2")
 			UpdatePartitionWithCheck(deployment, intstr.FromInt(0))
 			UpdatePartitionWithoutCheck(deployment, intstr.FromInt(3))
@@ -303,6 +321,7 @@ var _ = SIGDescribe("Advanced Deployment", func() {
 			deployment.Namespace = namespace
 			Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred())
 			CreateObject(deployment)
+			CheckReplicas(deployment, 5, 5, 5)
 			UpdateDeployment(deployment, "version2")
 			UpdatePartitionWithCheck(deployment, intstr.FromInt(0))
 			UpdatePartitionWithCheck(deployment, intstr.FromInt(2))
@@ -317,7 +336,9 @@ var _ = SIGDescribe("Advanced Deployment", func() {
 			deployment.Namespace = namespace
 			Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred())
 			CreateObject(deployment)
+			CheckReplicas(deployment, 5, 5, 5)
 			UpdateDeployment(deployment, "version2")
+			// error: stuck in (0,0,0)
 			UpdatePartitionWithCheck(deployment, intstr.FromInt(0))
 			UpdatePartitionWithCheck(deployment, intstr.FromInt(2))
 			UpdateDeployment(deployment, "version3")
@@ -335,8 +356,10 @@ var _ = SIGDescribe("Advanced Deployment", func() {
 			Expect(ReadYamlToObject("./test_data/deployment/deployment.yaml", deployment)).ToNot(HaveOccurred())
 			deployment.Annotations["rollouts.kruise.io/deployment-strategy"] = `{"rollingUpdate":{"maxUnavailable":0,"maxSurge":1}}`
 			CreateObject(deployment)
+			CheckReplicas(deployment, 5, 5, 5)
 			UpdateDeployment(deployment, "version2", "busybox:not-exists")
 			UpdatePartitionWithoutCheck(deployment, intstr.FromInt(1))
+			// error: stuck in (5,0,5)
 			CheckReplicas(deployment, 6, 5, 1)
 			UpdateDeployment(deployment, "version3", "busybox:1.32")
 			CheckReplicas(deployment, 5, 5, 0)

diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go
@@ -215,7 +215,7 @@ var _ = SIGDescribe("Rollout", func() {
 			body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, v1alpha1.CanaryStepStateReady)
 			Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred())
 			return false
-		}, 10*time.Second, time.Second).Should(BeTrue())
+		}, 10*time.Second, 100*time.Microsecond).Should(BeTrue())
 	}
 
 	WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) {
@@ -4850,7 +4850,12 @@ var _ = SIGDescribe("Rollout", func() {
 			By("wait step(1) pause")
 			WaitRolloutCanaryStepPaused(rollout.Name, 1)
 			CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1)
-
+			/*
+				STEP: wait step(2) pause
+				resume rollout success, and CurrentStepState {"observedGeneration":1,"canaryStatus":{"observedWorkloadGeneration":4,"observedRolloutID":"1","rolloutHash":"9bdxfz594524d9dcddvwv54bdxdv7dw278c657f7fb44wzczx4xb8bw97b4646zx","stableRevision":"774c56dc98","canaryRevision":"774c56dc98","podTemplateHash":"774c56dc98","canaryReplicas":5,"canaryReadyReplicas":5,"nextStepIndex":4,"currentStepIndex":3,"currentStepState":"StepUpgrade","message":"BatchRelease is at state Upgrading, rollout-id 1, step 3","lastUpdateTime":"2024-05-31T03:03:14Z","finalisingStep":""},"conditions":[{"type":"Progressing","status":"True","lastUpdateTime":"2024-05-31T03:03:06Z","lastTransitionTime":"2024-05-31T03:03:06Z","reason":"InRolling","message":"Rollout is in Progressing"}],"phase":"Progressing","message":"Rollout is in step(3/5), and upgrade workload to new version"}
+				I0531 03:03:15.927607   27074 rollout_test.go:288] current step:3 target step:2 current step state StepUpgrade
+				I0531 03:03:17.015278   27074 rollout_test.go:288] current step:3 target step:2 current step state StepPaused
+			*/
 			By("wait step(2) pause")
 			ResumeRolloutCanary(rollout.Name)
 			WaitRolloutCanaryStepPaused(rollout.Name, 2)
@@ -5598,7 +5603,12 @@ var _ = SIGDescribe("Rollout", func() {
 				Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred())
 				return object.Status.CanaryStatus.CanaryReadyReplicas == 1
 			}, 5*time.Minute, time.Second).Should(BeTrue())
-
+			/*
+				STEP: rolling deployment to be completed
+				resume rollout success, and CurrentStepState {"observedGeneration":1,"canaryStatus":{"observedWorkloadGeneration":13,"rolloutHash":"bbw699f54445f9b4zv825d8wwvw992z649vv2fz7946f8d52694zvzd4dvf4v4x5","stableRevision":"567694c97d","canaryRevision":"bcb64648f","podTemplateHash":"bcb64648f","canaryReplicas":1,"canaryReadyReplicas":1,"nextStepIndex":3,"currentStepIndex":2,"currentStepState":"StepReady","message":"BatchRelease is at state Verifying, rollout-id , step 1","lastUpdateTime":"2024-06-04T03:15:42Z","finalisingStep":""},"conditions":[{"type":"Progressing","status":"True","lastUpdateTime":"2024-06-04T03:15:26Z","lastTransitionTime":"2024-06-04T03:15:26Z","reason":"InRolling","message":"Rollout is in Progressing"}],"phase":"Progressing","message":"Rollout is in step(1/3), and you need manually confirm to enter the next step"}
+				I0604 03:15:43.498181   27389 rollout_test.go:288] current step:2 target step:2 current step state StepUpgrade
+				I0604 03:15:44.501250   27389 rollout_test.go:288] current step:3 target step:2 current step state StepUpgrade
+			*/
 			By("rolling deployment to be completed")
 			ResumeRolloutCanary(rollout.Name)
 			WaitRolloutCanaryStepPaused(rollout.Name, 2)