Support building local kind Node image

When trigger Manually upstream conformance test on Linux, if the input kubernetes version is "latest", the job will build kind Node image with the latest Kubernetes source code. Signed-off-by: Jiajing Hu <[email protected]>
antrea-io · Jun 7, 2024 · 8649906 · 8649906
1 parent 264372d
commit 8649906
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 6 deletions.
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -71,10 +71,24 @@ jobs:
           ./hack/build-antrea-linux-all.sh --pull --distro ${{ inputs.antrea-image-distro }}
       - name: Install Kind
         run: |
-          KIND_VERSION=$(head -n1 ./ci/kind/version || echo v0.20.0)
+          KIND_VERSION=$(head -n1 ./ci/kind/version || echo v0.23.0)
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
           chmod +x ./kind
           sudo mv kind /usr/local/bin
+      - name: Prepare local machine for conformance test
+        run: |
+          if [[ "${{ inputs.k8s-version }}" =~ ^(v)?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            exit 0
+          fi
+          sudo apt install -y skopeo jq
+          # Building a local Kind Node image with the latest Kubernetes version will consume a lot of disk space.
+          # We need to free up some disk space before building the image.
+          sudo apt-get clean
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf "/usr/local/share/boost" || true
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
+
       - name: Create K8s cluster
         run: |
           # If an image does not exist (unified vs split), a warning will be printed, but the script
@@ -108,7 +122,12 @@ jobs:
           kubectl rollout status -n kube-system ds/antrea-agent --timeout=5m
       - name: Run e2e tests
         run: |
-          ./ci/run-k8s-e2e-tests.sh "--e2e-${{ inputs.test-suite }}"
+          conformance_image_options=()
+          # If the K8s version is 'latest', we need to find the latest conformance image.
+          if [ ${{ inputs.k8s-version == 'latest' }} ]; then
+            conformance_image_options+=(--conformance-image "registry.k8s.io/conformance:$(skopeo list-tags docker://registry.k8s.io/conformance | jq -r '.Tags[]' | sort -V | tail -n 1)")
+          fi
+          ./ci/run-k8s-e2e-tests.sh "--e2e-${{ inputs.test-suite }}" ${conformance_image_options[@]}
       - name: Upload test log
         uses: actions/upload-artifact@v4
         if: ${{ failure() || inputs.always-upload-logs }}

diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh
@@ -39,6 +39,7 @@ PROMETHEUS=false
 K8S_VERSION=""
 KUBE_NODE_IPAM=true
 DEPLOY_EXTERNAL_SERVER=false
+CONFORMANCE_IMAGE=""
 positional_args=()
 options=()
 
@@ -386,7 +387,7 @@ EOF
 
   IMAGE_OPT=""
   if [[ "$K8S_VERSION" != "" ]]; then
-    if [[ "$K8S_VERSION" != v* ]]; then
+    if [[ "$K8S_VERSION" != v* ]] && [[ "$K8S_VERSION" != "latest" ]]; then
       K8S_VERSION="v${K8S_VERSION}"
     fi
     IMAGE_OPT="--image kindest/node:${K8S_VERSION}"
@@ -665,6 +666,14 @@ if version_lt "$kind_version" "0.12.0" && [[ "$KUBE_PROXY_MODE" == "none" ]]; th
     exit 1
 fi
 
+# When k8s-version is not vx.y.z, we need to build the K8s node image in local.
+if [[ ! $K8S_VERSION =~ ^(v)?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "=== Building Kind node image with the K8s version ${K8S_VERSION} ==="
+    kind_node_image="kindest/node:${K8S_VERSION}"
+    git clone -b master https://github.com/kubernetes/kubernetes.git ${THIS_DIR}/kubernetes
+    kind build node-image --image=$kind_node_image ${THIS_DIR}/kubernetes
+fi
+
 if [[ $ACTION == "create" ]]; then
     if [[ ! -z $SUBNETS ]] && [[ ! -z $EXTRA_NETWORKS ]]; then
         echoerr "Only one of '--subnets' and '--extra-networks' can be specified"

diff --git a/ci/run-k8s-e2e-tests.sh b/ci/run-k8s-e2e-tests.sh
@@ -45,10 +45,11 @@ IMAGE_PULL_POLICY="Always"
 CONFORMANCE_IMAGE_CONFIG_PATH="${THIS_DIR}/conformance-image-config.yaml"
 SONOBUOY_IMAGE="antrea/sonobuoy:v0.56.16"
 SYSTEMD_LOGS_IMAGE="antrea/systemd-logs:v0.4"
+CONFORMANCE_IMAGE=""
 
 _usage="Usage: $0 [--e2e-conformance] [--e2e-network-policy] [--e2e-focus <TestRegex>] [--e2e-skip <SkipRegex>]
                   [--kubeconfig <Kubeconfig>] [--kubernetes-version <ConformanceImageVersion>]
-                  [--log-mode <SonobuoyResultLogLevel>]
+                  [--log-mode <SonobuoyResultLogLevel>] [--conformance-image <ConformanceImage>]
 Run the K8s e2e community tests (Conformance & Network Policy) which are relevant to Project Antrea,
 using the sonobuoy tool. Possible exit codes are 0 (all tests pass), 1 (all tests were run, but at
 least one failed) and 2 (internal error when running tests, not a test failure).
@@ -64,6 +65,7 @@ least one failed) and 2 (internal error when running tests, not a test failure).
         --log-mode                                                Use the flag to set either 'report', 'detail', or 'dump' level data for sonobuoy results.
         --image-pull-policy                                       The ImagePullPolicy Sonobuoy should use for the aggregators and workers. (default Always)
         --sonobuoy-image SonobuoyImage                            Sonobuoy image to use. Default is $SONOBUOY_IMAGE.
+        --conformance-image                                       Conformance image to use. If not set, the default image will be used.
         --help, -h                                                Print this message and exit
 
 This tool uses sonobuoy (https://github.com/vmware-tanzu/sonobuoy) to run the K8s e2e community
@@ -138,6 +140,10 @@ case $key in
     SYSTEMD_LOGS_IMAGE="$2"
     shift 2
     ;;
+    --conformance-image)
+    CONFORMANCE_IMAGE="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -167,6 +173,10 @@ errors=0
 function run_sonobuoy() {
     local focus_regex="$1"
     local skip_regex="$2"
+    conformance_image_option=()
+    if [[ "$CONFORMANCE_IMAGE" != "" ]]; then
+        conformance_image_option=(--kube-conformance-image "$CONFORMANCE_IMAGE")
+    fi
 
     $SONOBUOY delete --wait=10 $KUBECONFIG_OPTION
     echo "Running tests with sonobuoy. While test is running, check logs with: $SONOBUOY $KUBECONFIG_OPTION logs -f."
@@ -176,14 +186,14 @@ function run_sonobuoy() {
                 $KUBECONFIG_OPTION \
                 $KUBE_CONFORMANCE_IMAGE_VERSION_OPTION \
                 --mode "certified-conformance" --image-pull-policy ${IMAGE_PULL_POLICY} \
-                --sonobuoy-image ${SONOBUOY_IMAGE} --systemd-logs-image ${SYSTEMD_LOGS_IMAGE} --e2e-repo-config ${CONFORMANCE_IMAGE_CONFIG_PATH}
+                --sonobuoy-image ${SONOBUOY_IMAGE} --systemd-logs-image ${SYSTEMD_LOGS_IMAGE} --e2e-repo-config ${CONFORMANCE_IMAGE_CONFIG_PATH} ${conformance_image_option[@]}
     else
         $SONOBUOY run --wait \
                 $KUBECONFIG_OPTION \
                 --e2e-parallel=true \
                 $KUBE_CONFORMANCE_IMAGE_VERSION_OPTION \
                 --e2e-focus "$focus_regex" --e2e-skip "$skip_regex" --image-pull-policy ${IMAGE_PULL_POLICY} \
-                --sonobuoy-image ${SONOBUOY_IMAGE} --systemd-logs-image ${SYSTEMD_LOGS_IMAGE} --e2e-repo-config ${CONFORMANCE_IMAGE_CONFIG_PATH}
+                --sonobuoy-image ${SONOBUOY_IMAGE} --systemd-logs-image ${SYSTEMD_LOGS_IMAGE} --e2e-repo-config ${CONFORMANCE_IMAGE_CONFIG_PATH} ${conformance_image_option[@]}
     fi
     set +x
     results_path=$($SONOBUOY retrieve $KUBECONFIG_OPTION)