Resolve Windows agent crash issue caused by egressSNATRandomFully (an…

…trea-io#6668) * Resolve Windows agent crash issue caused by egressSNATRandomFully o.config.Egress.SNATFullyRandomPorts should be checked for nil regardless of whether the Egress feature is enabled. * Add docker credentials to windows CI to avoid rate limit issue Add credentials in ctr commands to avoid rate limit issue. --------- Signed-off-by: Shuyang Xin <[email protected]>
hangyan · Sep 13, 2024 · 45109f8 · 45109f8
1 parent 6210c2d
commit 45109f8
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 13 deletions.
diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml
@@ -131,7 +131,7 @@
           set -ex
           DOCKER_REGISTRY="$(head -n1 ci/docker-registry)"
           [ "$DOCKER_REGISTRY" != "docker.io" ] || ./ci/jenkins/docker_login.sh --docker-user ${{DOCKER_USERNAME}} --docker-password ${{DOCKER_PASSWORD}}
-          ./ci/jenkins/test.sh --testcase '{e2e_type}' --registry ${{DOCKER_REGISTRY}}
+          ./ci/jenkins/test.sh --testcase '{e2e_type}' --registry ${{DOCKER_REGISTRY}} --docker-user ${{DOCKER_USERNAME}} --docker-password ${{DOCKER_PASSWORD}}
 
 - builder:
     name: builder-conformance-win
@@ -141,7 +141,7 @@
           set -ex
           DOCKER_REGISTRY="$(head -n1 ci/docker-registry)"
           [ "$DOCKER_REGISTRY" != "docker.io" ] || ./ci/jenkins/docker_login.sh --docker-user ${{DOCKER_USERNAME}} --docker-password ${{DOCKER_PASSWORD}}
-          ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}}
+          ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --docker-user ${{DOCKER_USERNAME}} --docker-password ${{DOCKER_PASSWORD}}
 
 - builder:
     name: builder-flexible-ipam-e2e

diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -72,7 +72,9 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o
         --win-image-node         Name of the windows image node in cluster. Images are built by docker on this node.
         --kind-cluster-name      Name of the kind Cluster.
         --proxyall               Enable proxyAll to test AntreaProxy.
-        --build-tag              Custom build tag for images."
+        --build-tag              Custom build tag for images.
+        --docker-user             Username for Docker account.
+        --docker-password         Password for Docker account."
 
 function print_usage {
     echoerr "$_usage"
@@ -127,6 +129,14 @@ case $key in
     BUILD_TAG="$2"
     shift 2
     ;;
+    --docker-user)
+    DOCKER_USERNAME="$2"
+    shift 2
+    ;;
+    --docker-password)
+    DOCKER_PASSWORD="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -423,7 +433,8 @@ function deliver_antrea_linux {
     antrea_images=("e2eteam/agnhost:2.13" "docker.io/library/nginx:1.15-alpine")
     common_images=("registry.k8s.io/e2e-test-images/agnhost:2.29")
     k8s_images=("registry.k8s.io/e2e-test-images/agnhost:2.45" "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.5" "registry.k8s.io/e2e-test-images/nginx:1.14-2")
-    e2e_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.45" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2")
+    conformance_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.45" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2")
+    e2e_images=("toolbox:1.3-0" "nginx:1.21.6-alpine")
 
     echo "===== Deliver Antrea YAML to Controller nodes ====="
     IP=$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 ~ role {print $6}')
@@ -446,15 +457,18 @@ function deliver_antrea_linux {
         ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${WORKDIR}/antrea-ubuntu.tar" || true
 
         for i in "${!harbor_images[@]}"; do
-            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && ctr -n=k8s.io images tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
+            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && ctr -n=k8s.io images tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
         done
         # Pull necessary images in advance to avoid transient error
         for image in "${common_images[@]}"; do
-            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull ${image}" || true
+            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${image}" || true
         done
         # Pull necessary images for Kubernetes conformance test
         for i in "${!k8s_images[@]}"; do
-            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull ${k8s_images[i]} && ctr -n=k8s.io images tag ${k8s_images[i]} ${e2e_images[i]}" || true
+            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${k8s_images[i]} && ctr -n=k8s.io images tag ${k8s_images[i]} ${conformance_images[i]}" || true
+        done
+        for image in "${e2e_images[@]}"; do
+            ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${DOCKER_REGISTRY}/antrea/${image}" || true
         done
     done
     echo "==== Finish building and delivering Linux images ===="
@@ -479,13 +493,17 @@ function deliver_antrea_windows {
         harbor_images=("agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2")
         antrea_images=("e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "registry.k8s.io/e2e-test-images/agnhost:2.29" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2")
         k8s_images=("registry.k8s.io/e2e-test-images/agnhost:2.45" "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.5" "registry.k8s.io/e2e-test-images/nginx:1.14-2" "registry.k8s.io/pause:3.8")
-        e2e_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.45" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2" "k8sprow.azurecr.io/kubernetes-e2e-test-images/pause:3.8")
+        conformance_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.45" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2" "k8sprow.azurecr.io/kubernetes-e2e-test-images/pause:3.8")
+        e2e_images=("toolbox:1.3-0")
         # Pull necessary images in advance to avoid transient error
         for i in "${!harbor_images[@]}"; do
-            ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && ctr -n k8s.io images tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
+            ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && ctr -n k8s.io images tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
         done
         for i in "${!k8s_images[@]}"; do
-            ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull ${k8s_images[i]} && ctr -n k8s.io images tag ${k8s_images[i]} ${e2e_images[i]}" || true
+            ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${k8s_images[i]} && ctr -n k8s.io images tag ${k8s_images[i]} ${conformance_images[i]}" || true
+        done
+        for image in "${e2e_images[@]}"; do
+            ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${DOCKER_REGISTRY}/antrea/${image}" || true
         done
 
         for i in `seq 2`; do

diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -493,9 +493,11 @@ func (o *Options) setK8sNodeDefaultOptions() {
 		if o.config.Egress.MaxEgressIPsPerNode == 0 {
 			o.config.Egress.MaxEgressIPsPerNode = defaultMaxEgressIPsPerNode
 		}
-		if o.config.Egress.SNATFullyRandomPorts == nil {
-			o.config.Egress.SNATFullyRandomPorts = ptr.To(o.config.SNATFullyRandomPorts)
-		}
+	}
+
+	// Regardless of whether the egress feature is enabled, o.config.Egress.SNATFullyRandomPorts should not be nil to prevent a crash in NewClient().
+	if o.config.Egress.SNATFullyRandomPorts == nil {
+		o.config.Egress.SNATFullyRandomPorts = ptr.To(o.config.SNATFullyRandomPorts)
 	}
 }