From b180c658e72c56a0e1b945cbf37bd4274b05455b Mon Sep 17 00:00:00 2001 From: KM ANJU <95232459+KMAnju-2021@users.noreply.github.com> Date: Thu, 19 Sep 2024 20:07:22 +0530 Subject: [PATCH] To support running Antrea Flexible IPAM tests on KIND (#5974) Signed-off-by: KMAnju-2021 --- CONTRIBUTING.md | 1 + ci/jenkins/jobs/macros.yaml | 24 ++++++++- ci/jenkins/jobs/projects-lab.yaml | 37 ++++++++++++++ ci/jenkins/test.sh | 13 ++--- ci/kind/kind-setup.sh | 84 ++++++++++++++++++++----------- ci/kind/test-e2e-kind.sh | 7 ++- test/e2e/fixtures.go | 3 +- test/e2e/framework.go | 37 ++++++++------ 8 files changed, 148 insertions(+), 58 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 979dd5c96a9..6dfc965c5af 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,6 +192,7 @@ Here are the trigger phrases for individual checks: * `/test-kind-networkpolicy`: Linux IPv4 networkpolicy tests on Kind cluster. * `/test-kind-ipv6-only-networkpolicy`: Linux IPv6 only networkpolicy tests on Kind cluster. * `/test-kind-ipv6-networkpolicy`: Linux dual stack networkpolicy tests on Kind cluster. +* `/test-kind-flexible-ipam-e2e`: Flexible IPAM e2e tests on Kind clusters. Here are the trigger phrases for groups of checks: diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml index 6d0272aec3b..d13c442d815 100644 --- a/ci/jenkins/jobs/macros.yaml +++ b/ci/jenkins/jobs/macros.yaml @@ -252,7 +252,29 @@ return_code=$? set -ex ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}" - exit $return_code + exit $return_code + +- builder: + name: builder-kind-flexible-ipam-e2e + builders: + - shell: |- + #!/bin/bash + set -ex + DOCKER_REGISTRY="$(head -n1 ci/docker-registry)" + KIND_TIMEOUT=135 + FIRST_VLAN_SUBNET='11=192.168.241.1/24' + SECOND_VLAN_SUBNET='12=192.168.242.1/24' + FULL_CLUSTER_NAME='{kind_cluster_name}'-"${{BUILD_NUMBER}}" + ./ci/kind/kind-setup.sh destroy --all --until ${{KIND_TIMEOUT}} + ./ci/kind/kind-install.sh + ./ci/kind/kind-setup.sh --flexible-ipam create "${{FULL_CLUSTER_NAME}}" --vlan-subnets $FIRST_VLAN_SUBNET --vlan-subnets $SECOND_VLAN_SUBNET + kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config + set +e + ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind-flexible-ipam" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" + return_code=$? + set -e + ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}" + exit $return_code - builder: name: builder-rancher-e2e diff --git a/ci/jenkins/jobs/projects-lab.yaml b/ci/jenkins/jobs/projects-lab.yaml index 2eb57f7c8c7..d7f62977e22 100644 --- a/ci/jenkins/jobs/projects-lab.yaml +++ b/ci/jenkins/jobs/projects-lab.yaml @@ -1582,3 +1582,40 @@ default-excludes: true fingerprint: false only-if-success: false + - '{name}-{test_name}-for-pull-request': + test_name: kind-flexible-ipam-e2e + node: 'antrea-kind-flexible-ipam-testbed' + description: 'This is the {test_name} test for {name}.' + branches: + - ${{sha1}} + builders: + - builder-kind-flexible-ipam-e2e: + kind_cluster_name: '{test_name}' + trigger_phrase: ^(?!Thanks for your PR).*/test-kind-flexible-ipam-e2e.* + white_list_target_branches: [] + allow_whitelist_orgs_as_admins: true + admin_list: '{antrea_admin_list}' + org_list: '{antrea_org_list}' + white_list: '{antrea_white_list}' + only_trigger_phrase: true + trigger_permit_all: true + status_context: jenkins-kind-flexible-ipam-e2e + status_url: --none-- + success_status: Build finished. + failure_status: Failed. Add comment /test-kind-flexible-ipam-e2e to re-trigger. + error_status: Failed. Add comment /test-kind-flexible-ipam-e2e to re-trigger. + triggered_status: null + started_status: null + wrappers: + - timeout: + fail: true + timeout: 135 + type: absolute + publishers: + - archive: + allow-empty: true + artifacts: 'antrea-test-logs.tar.gz' + case-sensitive: true + default-excludes: true + fingerprint: false + only-if-success: false diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index 39ececc0e3d..0428b1e26cf 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -201,7 +201,7 @@ function clean_antrea { for antrea_yml in ${WORKDIR}/*.yml; do kubectl delete -f $antrea_yml --ignore-not-found=true || true done - docker images --format "{{.Repository}}:{{.Tag}}" | grep 'antrea'| xargs -r docker rmi || true + docker images --format "{{.Repository}}:{{.Tag}}" | grep 'antrea'| xargs -r docker rmi -f || true docker images | grep '' | awk '{print $3}' | xargs -r docker rmi || true check_and_cleanup_docker_build_cache } @@ -528,7 +528,6 @@ function deliver_antrea { if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then redeploy_k8s_if_ip_mode_changes fi - echo "====== Building Antrea for the Following Commit ======" export GO111MODULE=on export GOPATH=${WORKDIR}/go @@ -605,12 +604,12 @@ function deliver_antrea { scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" flow-aggregator.tar jenkins@[${IP}]:${DEFAULT_WORKDIR}/flow-aggregator.tar ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/antrea-ubuntu.tar; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/flow-aggregator.tar" || true done - elif [[ $TESTBED_TYPE == "kind" ]]; then + elif [[ $TESTBED_TYPE == "kind" || $TESTBED_TYPE == "kind-flexible-ipam" ]]; then kind load docker-image antrea/antrea-agent-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER} kind load docker-image antrea/antrea-controller-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER} kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} kubectl config use-context kind-${KIND_CLUSTER} - docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml + docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml elif [[ $TESTBED_TYPE == "jumper" ]]; then kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/.ssh/id_rsa" antrea-ubuntu.tar jenkins@[${IP}]:${DEFAULT_WORKDIR}/antrea-ubuntu.tar @@ -663,7 +662,7 @@ function run_e2e { mkdir -p "${WORKDIR}/.kube" mkdir -p "${WORKDIR}/.ssh" - if [[ $TESTBED_TYPE != "kind" ]]; then + if [[ $TESTBED_TYPE != "kind" && $TESTBED_TYPE != "kind-flexible-ipam" ]]; then cp -f "${WORKDIR}/kube.conf" "${WORKDIR}/.kube/config" fi generate_ssh_config @@ -676,6 +675,8 @@ function run_e2e { go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus --antrea-ipam elif [[ $TESTBED_TYPE == "kind" ]]; then go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus + elif [[ $TESTBED_TYPE == "kind-flexible-ipam" ]]; then + go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider kind -timeout=100m --prometheus --antrea-ipam else go test -v antrea.io/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=100m --prometheus fi @@ -999,7 +1000,7 @@ EOF } export KUBECONFIG=${KUBECONFIG_PATH} -if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then +if [[ $TESTBED_TYPE == "flexible-ipam" || $TESTBED_TYPE == "kind-flexible-ipam" ]]; then MANIFEST_ARGS="$MANIFEST_ARGS --flexible-ipam --multicast --verbose-log" fi diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index 1aa6bbc66cb..e57e79616e4 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -29,9 +29,8 @@ SERVICE_CIDR="" IP_FAMILY="ipv4" NUM_WORKERS=2 SUBNETS="" +VLAN_SUBNETS=() EXTRA_NETWORKS="" -VLAN_SUBNETS="" -VLAN_ID="" ENCAP_MODE="" PROXY=true KUBE_PROXY_MODE="iptables" @@ -40,6 +39,7 @@ K8S_VERSION="" KUBE_NODE_IPAM=true DEPLOY_EXTERNAL_AGNHOST=false DEPLOY_EXTERNAL_FRR=false +FLEXIBLE_IPAM=false positional_args=() options=() @@ -70,11 +70,10 @@ where: --subnets: a subnet creates a separate Docker bridge network (named 'antrea-') with the assigned subnet. A worker Node will be connected to one of those network. Default is empty: all worker Nodes connected to the default Docker bridge network created by kind. - --vlan-subnets: specify the subnets of the VLAN to which all Nodes will be connected, in addition to the primary network. - The IP expression of the subnet will be used as the gateway IP. For example, '--vlan-subnets 10.100.100.1/24' means - that a VLAN sub-interface will be created on the primary Docker bridge, and it will be assigned the 10.100.100.1/24 address. - --vlan-id: specify the ID of the VLAN to which all Nodes will be connected, in addition to the primary network. Note, - '--vlan-subnets' and '--vlan-id' must be specified together. + --vlan-subnets: specify the id and subnets of the VLAN to which all Nodes will be connected, in addition to the primary network. + The IP expression of the subnet will be used as the gateway IP. For example, '--vlan-subnets 10=172.100.10.1/24,fd00:172:100:10::1/96,' means + that a VLAN sub-interface will be created on the primary Docker bridge, and it will be assigned the 10.100.100.1/24 and fd00:172:100:10::1/96 addresses + and vlan-id 10. This option can be specified multiple times. --extra-networks: an extra network creates a separate Docker bridge network (named 'antrea-') with the assigned subnet. All worker Nodes will be connected to all the extra networks, in addition to the default Docker bridge network. Note, '--extra-networks' and '--subnets' cannot be specified together. @@ -244,25 +243,52 @@ function configure_extra_networks { done } +# update_kind_ipam_routes add and del routes for non-ipam test-pods. +function update_kind_ipam_routes { + local operation="$1" + if [[ "$operation" == "del" ]]; then + echo "Deleting routes" + else + echo "Adding routes" + fi + + node_data=$(kubectl get nodes -o jsonpath='{range .items[*]}{.spec.podCIDR}{" "}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' 2>/dev/null || true) + if [[ -z $node_data ]]; then + return + fi + echo "$node_data"| while read pod_cidr node_ip; do + docker_run_with_host_net ip route "$operation" "$pod_cidr" via "$node_ip" >/dev/null 2>&1 || true + done +} + function configure_vlan_subnets { - if [[ -z $VLAN_SUBNETS || -z $VLAN_ID ]]; then + if [[ ${#VLAN_SUBNETS[@]} -eq 0 ]]; then return fi echo "Configuring VLAN subnets" bridge_id=$(docker network inspect kind -f {{.ID}}) bridge_interface="br-${bridge_id:0:12}" - vlan_interface="br-${bridge_id:0:7}.$VLAN_ID" - - docker_run_with_host_net ip link add link $bridge_interface name $vlan_interface type vlan id $VLAN_ID - docker_run_with_host_net ip link set $vlan_interface up - IFS=',' read -r -a vlan_subnets <<< "$VLAN_SUBNETS" - for s in "${vlan_subnets[@]}" ; do - echo "Configuring extra IP $s to vlan interface $vlan_interface" - docker_run_with_host_net ip addr add dev $vlan_interface $s + + for vlan_subnet in "${VLAN_SUBNETS[@]}"; do + # Extract VLAN ID and subnets + vlan_id=$(echo $vlan_subnet | cut -d= -f1) + subnets=$(echo $vlan_subnet | cut -d= -f2) + + vlan_interface="br-${bridge_id:0:7}.$vlan_id" + + docker_run_with_host_net ip link add link $bridge_interface name $vlan_interface type vlan id $vlan_id + docker_run_with_host_net ip link set $vlan_interface up + + IFS=',' read -r -a subnet_array <<< "$subnets" + for subnet in "${subnet_array[@]}" ; do + echo "Configuring extra IP $subnet to VLAN interface $vlan_interface" + docker_run_with_host_net ip addr add dev $vlan_interface $subnet + done + + docker_run_with_host_net iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT + docker_run_with_host_net iptables -t filter -A FORWARD -o $bridge_interface -i $vlan_interface -j ACCEPT done - docker_run_with_host_net iptables -t filter -A FORWARD -i $bridge_interface -o $vlan_interface -j ACCEPT - docker_run_with_host_net iptables -t filter -A FORWARD -o $bridge_interface -i $vlan_interface -j ACCEPT } function delete_vlan_subnets { @@ -415,6 +441,9 @@ EOF configure_vlan_subnets setup_external_servers load_images + if [[ $FLEXIBLE_IPAM == true ]]; then + update_kind_ipam_routes "add" + fi if [[ $ANTREA_CNI == true ]]; then cmd=$(dirname $0) @@ -438,6 +467,7 @@ EOF } function destroy { + update_kind_ipam_routes "del" if [[ $UNTIL_TIME_IN_MINS != "" ]]; then clean_kind else @@ -573,14 +603,14 @@ while [[ $# -gt 0 ]] ;; --vlan-subnets) add_option "--vlan-subnets" "create" - VLAN_SUBNETS="$2" - shift 2 - ;; - --vlan-id) - add_option "--vlan-id" "create" - VLAN_ID="$2" + VLAN_SUBNETS+=("$2") shift 2 ;; + --flexible-ipam) + add_option "--flexible-ipam" "create" + FLEXIBLE_IPAM=true + shift + ;; --images) add_option "--image" "create" IMAGES="$2" @@ -670,12 +700,6 @@ if [[ $ACTION == "destroy" ]]; then exit fi -if [[ -n "$VLAN_SUBNETS" || -n "$VLAN_ID" ]]; then - if [[ -z "$VLAN_SUBNETS" || -z "$VLAN_ID" ]]; then - echoerr "'--vlan-subnets' and '--vlan-id' must be specified together" - exit 1 - fi -fi kind_version=$(kind version | awk '{print $2}') kind_version=${kind_version:1} # strip leading 'v' diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 1e91c95c874..bd642ea5963 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -293,13 +293,12 @@ printf -v COMMON_IMAGES "%s " "${COMMON_IMAGES_LIST[@]}" vlan_args="" if $extra_vlan; then - vlan_args="$vlan_args --vlan-id 10" if [[ "$ipfamily" == "v4" ]]; then - vlan_args="$vlan_args --vlan-subnets 172.100.10.1/24" + vlan_args="$vlan_args --vlan-subnets 10=172.100.10.1/24" elif [[ "$ipfamily" == "v6" ]]; then - vlan_args="$vlan_args --vlan-subnets fd00:172:100:10::1/96" + vlan_args="$vlan_args --vlan-subnets 10=fd00:172:100:10::1/96" elif [[ "$ipfamily" == "dual" ]]; then - vlan_args="$vlan_args --vlan-subnets 172.100.10.1/24,fd00:172:100:10::1/96" + vlan_args="$vlan_args --vlan-subnets 10=172.100.10.1/24,fd00:172:100:10::1/96" fi fi diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index e948b988d82..dcd95d1ad36 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -694,8 +694,7 @@ func testMain(m *testing.M) int { flag.StringVar(&testOptions.externalAgnhostIPs, "external-agnhost-ips", "", "IP addresses of external agnhost, at most one IP per IP family") flag.StringVar(&testOptions.externalFRRIPs, "external-frr-ips", "", "IP addresses of external FRR, at most one IP per IP family") flag.StringVar(&testOptions.externalFRRCID, "external-frr-cid", "", "Container ID of external FRR") - flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family") - flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in") + flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "ID and IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family") flag.Parse() cleanupLogging := testOptions.setupLogging() diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 27d82df258f..a497b49b53f 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -219,7 +219,6 @@ type TestOptions struct { externalAgnhostIPs string vlanSubnets string - vlanID int externalFRRIPs string // FRR cannot currently be configured remotely over networking. As a result, the e2e tests for BGPPolicy can only @@ -525,24 +524,32 @@ func (data *TestData) collectExternalInfo() error { } } - subnets := strings.Split(testOptions.vlanSubnets, ",") - for _, subnet := range subnets { - if subnet == "" { - continue - } - gatewayIP, _, err := net.ParseCIDR(subnet) + vlanSubnetsList := strings.Split(testOptions.vlanSubnets, "=") + vlanIDStr := vlanSubnetsList[0] + if vlanIDStr != "" { + vlanID, err := strconv.Atoi(vlanIDStr) if err != nil { - return fmt.Errorf("invalid vlan subnet %s: %w", subnet, err) + return fmt.Errorf("invalid vlan id %s: %w", vlanIDStr, err) } - if gatewayIP.To4() != nil { - externalInfo.vlanSubnetIPv4 = subnet - externalInfo.vlanGatewayIPv4 = gatewayIP.String() - } else { - externalInfo.vlanSubnetIPv6 = subnet - externalInfo.vlanGatewayIPv6 = gatewayIP.String() + externalInfo.vlanID = vlanID + subnets := strings.Split(vlanSubnetsList[1], ",") + for _, subnet := range subnets { + if subnet == "" { + continue + } + gatewayIP, _, err := net.ParseCIDR(subnet) + if err != nil { + return fmt.Errorf("invalid vlan subnet %s: %w", subnet, err) + } + if gatewayIP.To4() != nil { + externalInfo.vlanSubnetIPv4 = subnet + externalInfo.vlanGatewayIPv4 = gatewayIP.String() + } else { + externalInfo.vlanSubnetIPv6 = subnet + externalInfo.vlanGatewayIPv6 = gatewayIP.String() + } } } - externalInfo.vlanID = testOptions.vlanID frrIPs := strings.Split(testOptions.externalFRRIPs, ",") for _, ip := range frrIPs {