To support running Antrea Flexible IPAM tests on KIND (#5974)

Signed-off-by: KMAnju-2021 <[email protected]>
antrea-io · Sep 19, 2024 · b180c65 · b180c65
1 parent 84f60d2
commit b180c65
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 58 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -192,6 +192,7 @@ Here are the trigger phrases for individual checks:
 * `/test-kind-networkpolicy`: Linux IPv4 networkpolicy tests on Kind cluster.
 * `/test-kind-ipv6-only-networkpolicy`: Linux IPv6 only networkpolicy tests on Kind cluster.
 * `/test-kind-ipv6-networkpolicy`: Linux dual stack networkpolicy tests on Kind cluster.
+* `/test-kind-flexible-ipam-e2e`: Flexible IPAM e2e tests on Kind clusters.
 
 Here are the trigger phrases for groups of checks:
 

diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml
@@ -252,7 +252,29 @@
           return_code=$?
           set -ex
           ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"
-          exit $return_code 
+          exit $return_code
+                   
+- builder:
+    name: builder-kind-flexible-ipam-e2e
+    builders:
+      - shell: |-
+          #!/bin/bash
+          set -ex
+          DOCKER_REGISTRY="$(head -n1 ci/docker-registry)"
+          KIND_TIMEOUT=135
+          FIRST_VLAN_SUBNET='11=192.168.241.1/24'
+          SECOND_VLAN_SUBNET='12=192.168.242.1/24'
+          FULL_CLUSTER_NAME='{kind_cluster_name}'-"${{BUILD_NUMBER}}"
+          ./ci/kind/kind-setup.sh destroy --all --until ${{KIND_TIMEOUT}}
+          ./ci/kind/kind-install.sh 
+          ./ci/kind/kind-setup.sh --flexible-ipam create "${{FULL_CLUSTER_NAME}}" --vlan-subnets $FIRST_VLAN_SUBNET --vlan-subnets $SECOND_VLAN_SUBNET
+          kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config
+          set +e
+          ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind-flexible-ipam" --kind-cluster-name "${{FULL_CLUSTER_NAME}}"
+          return_code=$?
+          set -e
+          ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"
+          exit $return_code  
 
 - builder:
     name: builder-rancher-e2e

diff --git a/ci/jenkins/jobs/projects-lab.yaml b/ci/jenkins/jobs/projects-lab.yaml
@@ -1582,3 +1582,40 @@
                 default-excludes: true
                 fingerprint: false
                 only-if-success: false
+      - '{name}-{test_name}-for-pull-request':
+            test_name: kind-flexible-ipam-e2e
+            node: 'antrea-kind-flexible-ipam-testbed'
+            description: 'This is the {test_name} test for {name}.'
+            branches:
+            - ${{sha1}}      
+            builders:
+              - builder-kind-flexible-ipam-e2e:
+                  kind_cluster_name: '{test_name}'
+            trigger_phrase: ^(?!Thanks for your PR).*/test-kind-flexible-ipam-e2e.*
+            white_list_target_branches: []
+            allow_whitelist_orgs_as_admins: true
+            admin_list: '{antrea_admin_list}'
+            org_list: '{antrea_org_list}'
+            white_list: '{antrea_white_list}'
+            only_trigger_phrase: true
+            trigger_permit_all: true
+            status_context: jenkins-kind-flexible-ipam-e2e
+            status_url: --none--
+            success_status: Build finished.
+            failure_status: Failed. Add comment /test-kind-flexible-ipam-e2e to re-trigger.
+            error_status: Failed. Add comment /test-kind-flexible-ipam-e2e to re-trigger.
+            triggered_status: null
+            started_status: null
+            wrappers:
+            - timeout:
+                fail: true
+                timeout: 135
+                type: absolute
+            publishers:
+            - archive:
+                allow-empty: true
+                artifacts: 'antrea-test-logs.tar.gz'
+                case-sensitive: true
+                default-excludes: true
+                fingerprint: false
+                only-if-success: false
diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -201,7 +201,7 @@ function clean_antrea {
     for antrea_yml in ${WORKDIR}/*.yml; do
         kubectl delete -f $antrea_yml --ignore-not-found=true || true
     done
-    docker images --format "{{.Repository}}:{{.Tag}}" | grep 'antrea'| xargs -r docker rmi || true
+    docker images --format "{{.Repository}}:{{.Tag}}" | grep 'antrea'| xargs -r docker rmi -f || true
     docker images | grep '<none>' | awk '{print $3}' | xargs -r docker rmi || true
     check_and_cleanup_docker_build_cache
 }
@@ -528,7 +528,6 @@ function deliver_antrea {
     if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then
         redeploy_k8s_if_ip_mode_changes
     fi
-
     echo "====== Building Antrea for the Following Commit ======"
     export GO111MODULE=on
     export GOPATH=${WORKDIR}/go
@@ -605,12 +604,12 @@ function deliver_antrea {
             scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" flow-aggregator.tar jenkins@[${IP}]:${DEFAULT_WORKDIR}/flow-aggregator.tar
             ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/antrea-ubuntu.tar; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/flow-aggregator.tar" || true
         done
-    elif [[ $TESTBED_TYPE == "kind" ]]; then
+    elif [[ $TESTBED_TYPE == "kind" || $TESTBED_TYPE == "kind-flexible-ipam" ]]; then
             kind load docker-image antrea/antrea-agent-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER}
             kind load docker-image antrea/antrea-controller-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER}
             kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} 
             kubectl config use-context kind-${KIND_CLUSTER}
-            docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml 
+            docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml
     elif [[ $TESTBED_TYPE == "jumper" ]]; then
         kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do
             scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/.ssh/id_rsa" antrea-ubuntu.tar jenkins@[${IP}]:${DEFAULT_WORKDIR}/antrea-ubuntu.tar
@@ -663,7 +662,7 @@ function run_e2e {
 
     mkdir -p "${WORKDIR}/.kube"
     mkdir -p "${WORKDIR}/.ssh"
-    if [[ $TESTBED_TYPE != "kind" ]]; then 
+    if [[ $TESTBED_TYPE != "kind" && $TESTBED_TYPE != "kind-flexible-ipam" ]]; then 
         cp -f "${WORKDIR}/kube.conf" "${WORKDIR}/.kube/config"
     fi
     generate_ssh_config
@@ -676,6 +675,8 @@ function run_e2e {
         go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus --antrea-ipam
     elif [[ $TESTBED_TYPE == "kind" ]]; then
         go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus
+    elif [[ $TESTBED_TYPE == "kind-flexible-ipam" ]]; then
+        go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus --antrea-ipam
     else
         go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus
     fi
@@ -999,7 +1000,7 @@ EOF
 }
 
 export KUBECONFIG=${KUBECONFIG_PATH}
-if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then
+if [[ $TESTBED_TYPE == "flexible-ipam" || $TESTBED_TYPE == "kind-flexible-ipam" ]]; then
     MANIFEST_ARGS="$MANIFEST_ARGS --flexible-ipam --multicast --verbose-log"
 fi
 

diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh
@@ -29,9 +29,8 @@ SERVICE_CIDR=""
 IP_FAMILY="ipv4"
 NUM_WORKERS=2
 SUBNETS=""
+VLAN_SUBNETS=()
 EXTRA_NETWORKS=""
-VLAN_SUBNETS=""
-VLAN_ID=""
 ENCAP_MODE=""
 PROXY=true
 KUBE_PROXY_MODE="iptables"
@@ -40,6 +39,7 @@ K8S_VERSION=""
 KUBE_NODE_IPAM=true
 DEPLOY_EXTERNAL_AGNHOST=false
 DEPLOY_EXTERNAL_FRR=false
+FLEXIBLE_IPAM=false
 positional_args=()
 options=()
 
@@ -70,11 +70,10 @@ where:
   --subnets: a subnet creates a separate Docker bridge network (named 'antrea-<idx>') with the assigned subnet. A worker
     Node will be connected to one of those network. Default is empty: all worker Nodes connected to the default Docker
     bridge network created by kind.
-  --vlan-subnets: specify the subnets of the VLAN to which all Nodes will be connected, in addition to the primary network.
-    The IP expression of the subnet will be used as the gateway IP. For example, '--vlan-subnets 10.100.100.1/24' means
-    that a VLAN sub-interface will be created on the primary Docker bridge, and it will be assigned the 10.100.100.1/24 address.
-  --vlan-id: specify the ID of the VLAN to which all Nodes will be connected, in addition to the primary network. Note,
-    '--vlan-subnets' and '--vlan-id' must be specified together.
+  --vlan-subnets: specify the id and subnets of the VLAN to which all Nodes will be connected, in addition to the primary network.
+    The IP expression of the subnet will be used as the gateway IP. For example, '--vlan-subnets 10=172.100.10.1/24,fd00:172:100:10::1/96,' means
+    that a VLAN sub-interface will be created on the primary Docker bridge, and it will be assigned the 10.100.100.1/24 and fd00:172:100:10::1/96 addresses
+    and vlan-id 10. This option can be specified multiple times.
   --extra-networks: an extra network creates a separate Docker bridge network (named 'antrea-<idx>') with the assigned
     subnet. All worker Nodes will be connected to all the extra networks, in addition to the default Docker bridge
     network. Note, '--extra-networks' and '--subnets' cannot be specified together.
@@ -244,25 +243,52 @@ function configure_extra_networks {
   done
 }
 
+# update_kind_ipam_routes add and del routes for non-ipam test-pods.
+function update_kind_ipam_routes {
+  local operation="$1"
+  if [[ "$operation" == "del" ]]; then
+    echo "Deleting routes"
+  else  
+    echo "Adding routes"
+  fi
+
+  node_data=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.podCIDR}{" "}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' 2>/dev/null || true)
+  if [[ -z $node_data ]]; then
+    return
+  fi
+  echo "$node_data"| while read pod_cidr node_ip; do
+    docker_run_with_host_net ip route "$operation" "$pod_cidr" via "$node_ip" >/dev/null 2>&1 || true
+  done
+}
+
 function configure_vlan_subnets {
-  if [[ -z $VLAN_SUBNETS || -z $VLAN_ID ]]; then
+  if [[ ${#VLAN_SUBNETS[@]} -eq 0 ]]; then
     return
   fi
   echo "Configuring VLAN subnets"
 
   bridge_id=$(docker network inspect kind -f {{.ID}})
   bridge_interface="br-${bridge_id:0:12}"
-  vlan_interface="br-${bridge_id:0:7}.$VLAN_ID"
-
-  docker_run_with_host_net ip link add link $bridge_interface name $vlan_interface type vlan id $VLAN_ID
-  docker_run_with_host_net ip link set $vlan_interface up
-  IFS=',' read -r -a vlan_subnets <<< "$VLAN_SUBNETS"
-  for s in "${vlan_subnets[@]}" ; do
-    echo "Configuring extra IP $s to vlan interface $vlan_interface"
-    docker_run_with_host_net ip addr add dev $vlan_interface $s
+
+  for vlan_subnet in "${VLAN_SUBNETS[@]}"; do
+    # Extract VLAN ID and subnets
+    vlan_id=$(echo $vlan_subnet | cut -d= -f1)
+    subnets=$(echo $vlan_subnet | cut -d= -f2)
+
+    vlan_interface="br-${bridge_id:0:7}.$vlan_id"
+
+    docker_run_with_host_net ip link add link $bridge_interface name $vlan_interface type vlan id $vlan_id
+    docker_run_with_host_net ip link set $vlan_interface up
+
+    IFS=',' read -r -a subnet_array <<< "$subnets"
+    for subnet in "${subnet_array[@]}" ; do
+      echo "Configuring extra IP $subnet to VLAN interface $vlan_interface"
+      docker_run_with_host_net ip addr add dev $vlan_interface $subnet
+    done
+
+    docker_run_with_host_net iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT
+    docker_run_with_host_net iptables -t filter -A FORWARD -o $bridge_interface -i $vlan_interface -j ACCEPT
   done
-  docker_run_with_host_net iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT
-  docker_run_with_host_net iptables -t filter -A FORWARD -o $bridge_interface -i $vlan_interface -j ACCEPT
 }
 
 function delete_vlan_subnets {
@@ -415,6 +441,9 @@ EOF
   configure_vlan_subnets
   setup_external_servers
   load_images
+  if [[ $FLEXIBLE_IPAM == true ]]; then
+      update_kind_ipam_routes "add"
+  fi
 
   if [[ $ANTREA_CNI == true ]]; then
     cmd=$(dirname $0)
@@ -438,6 +467,7 @@ EOF
 }
 
 function destroy {
+  update_kind_ipam_routes "del"
   if [[ $UNTIL_TIME_IN_MINS != "" ]]; then
       clean_kind
   else
@@ -573,14 +603,14 @@ while [[ $# -gt 0 ]]
       ;;
     --vlan-subnets)
       add_option "--vlan-subnets" "create"
-      VLAN_SUBNETS="$2"
-      shift 2
-      ;;
-    --vlan-id)
-      add_option "--vlan-id" "create"
-      VLAN_ID="$2"
+      VLAN_SUBNETS+=("$2")
       shift 2
       ;;
+    --flexible-ipam)
+      add_option "--flexible-ipam" "create"
+      FLEXIBLE_IPAM=true
+      shift
+      ;; 
     --images)
       add_option "--image" "create"
       IMAGES="$2"
@@ -670,12 +700,6 @@ if [[ $ACTION == "destroy" ]]; then
       exit
 fi
 
-if [[ -n "$VLAN_SUBNETS" || -n "$VLAN_ID" ]]; then
-  if [[ -z "$VLAN_SUBNETS" || -z "$VLAN_ID" ]]; then
-    echoerr "'--vlan-subnets' and '--vlan-id' must be specified together"
-    exit 1
-  fi
-fi
 
 kind_version=$(kind version | awk  '{print $2}')
 kind_version=${kind_version:1} # strip leading 'v'

diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -293,13 +293,12 @@ printf -v COMMON_IMAGES "%s " "${COMMON_IMAGES_LIST[@]}"
 
 vlan_args=""
 if $extra_vlan; then
-  vlan_args="$vlan_args --vlan-id 10"
   if [[ "$ipfamily" == "v4" ]]; then
-    vlan_args="$vlan_args --vlan-subnets 172.100.10.1/24"
+    vlan_args="$vlan_args --vlan-subnets 10=172.100.10.1/24"
   elif [[ "$ipfamily" == "v6" ]]; then
-    vlan_args="$vlan_args --vlan-subnets fd00:172:100:10::1/96"
+    vlan_args="$vlan_args --vlan-subnets 10=fd00:172:100:10::1/96"
   elif [[ "$ipfamily" == "dual" ]]; then
-    vlan_args="$vlan_args --vlan-subnets 172.100.10.1/24,fd00:172:100:10::1/96"
+    vlan_args="$vlan_args --vlan-subnets 10=172.100.10.1/24,fd00:172:100:10::1/96"
   fi
 fi
 

diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go
@@ -694,8 +694,7 @@ func testMain(m *testing.M) int {
 	flag.StringVar(&testOptions.externalAgnhostIPs, "external-agnhost-ips", "", "IP addresses of external agnhost, at most one IP per IP family")
 	flag.StringVar(&testOptions.externalFRRIPs, "external-frr-ips", "", "IP addresses of external FRR, at most one IP per IP family")
 	flag.StringVar(&testOptions.externalFRRCID, "external-frr-cid", "", "Container ID of external FRR")
-	flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family")
-	flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in")
+	flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "ID and IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family")
 	flag.Parse()
 
 	cleanupLogging := testOptions.setupLogging()

diff --git a/test/e2e/framework.go b/test/e2e/framework.go
@@ -219,7 +219,6 @@ type TestOptions struct {
 
 	externalAgnhostIPs string
 	vlanSubnets        string
-	vlanID             int
 
 	externalFRRIPs string
 	// FRR cannot currently be configured remotely over networking. As a result, the e2e tests for BGPPolicy can only
@@ -525,24 +524,32 @@ func (data *TestData) collectExternalInfo() error {
 		}
 	}
 
-	subnets := strings.Split(testOptions.vlanSubnets, ",")
-	for _, subnet := range subnets {
-		if subnet == "" {
-			continue
-		}
-		gatewayIP, _, err := net.ParseCIDR(subnet)
+	vlanSubnetsList := strings.Split(testOptions.vlanSubnets, "=")
+	vlanIDStr := vlanSubnetsList[0]
+	if vlanIDStr != "" {
+		vlanID, err := strconv.Atoi(vlanIDStr)
 		if err != nil {
-			return fmt.Errorf("invalid vlan subnet %s: %w", subnet, err)
+			return fmt.Errorf("invalid vlan id %s: %w", vlanIDStr, err)
 		}
-		if gatewayIP.To4() != nil {
-			externalInfo.vlanSubnetIPv4 = subnet
-			externalInfo.vlanGatewayIPv4 = gatewayIP.String()
-		} else {
-			externalInfo.vlanSubnetIPv6 = subnet
-			externalInfo.vlanGatewayIPv6 = gatewayIP.String()
+		externalInfo.vlanID = vlanID
+		subnets := strings.Split(vlanSubnetsList[1], ",")
+		for _, subnet := range subnets {
+			if subnet == "" {
+				continue
+			}
+			gatewayIP, _, err := net.ParseCIDR(subnet)
+			if err != nil {
+				return fmt.Errorf("invalid vlan subnet %s: %w", subnet, err)
+			}
+			if gatewayIP.To4() != nil {
+				externalInfo.vlanSubnetIPv4 = subnet
+				externalInfo.vlanGatewayIPv4 = gatewayIP.String()
+			} else {
+				externalInfo.vlanSubnetIPv6 = subnet
+				externalInfo.vlanGatewayIPv6 = gatewayIP.String()
+			}
 		}
 	}
-	externalInfo.vlanID = testOptions.vlanID
 
 	frrIPs := strings.Split(testOptions.externalFRRIPs, ",")
 	for _, ip := range frrIPs {