Run Multiple kind Jobs on same testbed

Signed-off-by: Pulkit Jain <[email protected]>
antrea-io · Apr 17, 2024 · 6e73ae9 · 6e73ae9
1 parent e974fcf
commit 6e73ae9
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 20 deletions.
diff --git a/Makefile b/Makefile
@@ -20,7 +20,6 @@ OVS_VERSION        := $(shell head -n 1 build/images/deps/ovs-version)
 GO_VERSION         := $(shell head -n 1 build/images/deps/go-version)
 CNI_BINARIES_VERSION := $(shell head -n 1 build/images/deps/cni-binaries-version)
 NANOSERVER_VERSION := $(shell head -n 1 build/images/deps/nanoserver-version)
-BUILD_TAG          := $(shell build/images/build-tag.sh)
 WIN_BUILD_TAG      := $(shell echo $(GO_VERSION) $(CNI_BINARIES_VERSION) $(NANOSERVER_VERSION)|md5sum|head -c 10)
 WIN_OVS_VERSION	   := $(shell head -n 1 build/images/deps/ovs-version-windows)
 WIN_BUILD_OVS_TAG  := $(NANOSERVER_VERSION)-$(WIN_OVS_VERSION)
@@ -42,7 +41,6 @@ endif
 WIN_BUILD_ARGS := DOCKER_BUILD_ARGS
 DOCKER_BUILD_ARGS += --build-arg OVS_VERSION=$(OVS_VERSION)
 DOCKER_BUILD_ARGS += --build-arg GO_VERSION=$(GO_VERSION)
-DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG)
 WIN_BUILD_ARGS := --build-arg GO_VERSION=$(GO_VERSION)
 WIN_BUILD_ARGS += --build-arg CNI_BINARIES_VERSION=$(CNI_BINARIES_VERSION)
 WIN_BUILD_ARGS += --build-arg NANOSERVER_VERSION=$(NANOSERVER_VERSION)
@@ -57,6 +55,8 @@ all: build
 include versioning.mk
 
 LDFLAGS += $(VERSION_LDFLAGS)
+BUILD_TAG = $(BUILD_TAGS)
+DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG)
 
 UNAME_S := $(shell uname -s)
 USERID  := $(shell id -u)

diff --git a/build/images/base/build.sh b/build/images/base/build.sh
@@ -29,13 +29,19 @@ Build the antrea base image.
         --push                  Push the built image to the registry
         --platform <PLATFORM>   Target platform for the image if server is multi-platform capable
         --distro <distro>       Target Linux distribution
-        --no-cache              Do not use the local build cache nor the cached image from the registry"
+        --no-cache              Do not use the local build cache nor the cached image from the registry
+        --build-tag             Custom build tag for images."
+
+function print_usage {
+    echoerr "$_usage"
+}
 
 PULL=false
 PUSH=false
 NO_CACHE=false
 PLATFORM=""
 DISTRO="ubuntu"
+CUSTOM_BUILD_TAG=""
 
 while [[ $# -gt 0 ]]
 do
@@ -62,6 +68,10 @@ case $key in
     NO_CACHE=true
     shift
     ;;
+    --build-tag)
+    CUSTOM_BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -104,6 +114,10 @@ SURICATA_VERSION=$(head -n 1 ../deps/suricata-version)
 
 BUILD_TAG=$(../build-tag.sh)
 
+if [[ $CUSTOM_BUILD_TAG == "" ]]; then
+    CUSTOM_BUILD_TAG=$BUILD_TAG
+fi
+
 if $PULL; then
     # The ubuntu image is also used for the UBI build (for the cni-binaries intermediate image).
     if [[ ${DOCKER_REGISTRY} == "" ]]; then
@@ -138,7 +152,8 @@ fi
 function docker_build_and_push() {
     local image="$1"
     local dockerfile="$2"
-    local build_args="--build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION --build-arg SURICATA_VERSION=$SURICATA_VERSION --build-arg BUILD_TAG=$BUILD_TAG"
+    local custom_build_tag="$CUSTOM_BUILD_TAG"
+    local build_args="--build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION --build-arg SURICATA_VERSION=$SURICATA_VERSION --build-arg BUILD_TAG=$custom_build_tag"
     local cache_args=""
     if $PUSH; then
         cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_TAG,mode=max"
@@ -148,7 +163,7 @@ function docker_build_and_push() {
     else
         cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max"
     fi
-    docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile .
+    docker buildx build $PLATFORM_ARG -o type=docker -t $image:$custom_build_tag $cache_args $build_args -f $dockerfile .
 
     if $PUSH; then
         docker push $image:$BUILD_TAG

diff --git a/build/images/ovs/build.sh b/build/images/ovs/build.sh
@@ -29,13 +29,19 @@ Build the antrea openvswitch image.
         --push                  Push the built image to the registry
         --platform <PLATFORM>   Target platform for the image if server is multi-platform capable
         --distro <distro>       Target Linux distribution
-        --no-cache              Do not use the local build cache nor the cached image from the registry"
+        --no-cache              Do not use the local build cache nor the cached image from the registry
+        --build-tag             Custom build tag for images."
+
+function print_usage {
+    echoerr "$_usage"
+}
 
 PULL=false
 PUSH=false
 NO_CACHE=false
 PLATFORM=""
 DISTRO="ubuntu"
+CUSTOM_BUILD_TAG=""
 
 while [[ $# -gt 0 ]]
 do
@@ -62,6 +68,10 @@ case $key in
     NO_CACHE=true
     shift
     ;;
+    --build-tag)
+    CUSTOM_BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -103,6 +113,10 @@ OVS_VERSION=$(head -n 1 ../deps/ovs-version)
 
 BUILD_TAG=$(../build-tag.sh)
 
+if [[ $CUSTOM_BUILD_TAG == "" ]]; then
+    CUSTOM_BUILD_TAG=$BUILD_TAG
+fi
+
 if $PULL; then
     if [ "$DISTRO" == "ubuntu" ]; then
         if [[ ${DOCKER_REGISTRY} == "" ]]; then
@@ -120,6 +134,7 @@ fi
 function docker_build_and_push() {
     local image="$1"
     local dockerfile="$2"
+    local custom_build_tag="$CUSTOM_BUILD_TAG"
     local build_args="--build-arg OVS_VERSION=$OVS_VERSION"
     local cache_args=""
     if $PUSH; then
@@ -130,7 +145,7 @@ function docker_build_and_push() {
     else
         cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max"
     fi
-    docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile .
+    docker buildx build $PLATFORM_ARG -o type=docker -t $image:$custom_build_tag $cache_args $build_args -f $dockerfile .
 
     if $PUSH; then
         docker push $image:$BUILD_TAG

diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml
@@ -227,7 +227,7 @@
           ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}'
           kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config
           set +e
-          ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}"
+          ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "${conformance_type}-${{BUILD_NUMBER}}"
           return_code=$?
           set -ex
           ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"
@@ -250,7 +250,7 @@
           ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}'
           kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config
           set +e
-          ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}"
+          ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "e2e-${{BUILD_NUMBER}}"
           return_code=$?
           set -ex
           ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"

diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -55,6 +55,7 @@ CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=4h"
 CLEAN_STALE_IMAGES_CONTAINERD="crictl rmi --prune"
 PRINT_DOCKER_STATUS="docker system df -v"
 PRINT_CONTAINERD_STATUS="crictl ps --state Exited"
+CUSTOM_BUILD_TAG=""
 
 _usage="Usage: $0 [--kubeconfig <KubeconfigSavePath>] [--workdir <HomePath>]
                   [--testcase <windows-install-ovs|windows-containerd-conformance|windows-conformance|windows-containerd-networkpolicy|windows-networkpolicy|windows-containerd-e2e|windows-e2e|e2e|conformance|networkpolicy|multicast-e2e>]
@@ -69,7 +70,8 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o
         --testbed-type           The testbed type to run tests. It can be flexible-ipam, jumper or legacy.
         --ip-mode                IP mode for flexible-ipam e2e test. Default is $DEFAULT_IP_MODE. It can also be ipv6 or ds.
         --win-image-node         Name of the windows image node in containerd cluster. Images are built by docker on this node.
-        --kind-cluster-name      Name of the kind Cluster." 
+        --kind-cluster-name      Name of the kind Cluster.
+        --build-tag              Custom build tag for images." 
 
 function print_usage {
     echoerr "$_usage"
@@ -120,6 +122,10 @@ case $key in
     WIN_IMAGE_NODE="$2"
     shift 2
     ;;
+    --build-tag)
+    CUSTOM_BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -540,7 +546,11 @@ function deliver_antrea {
     fi
     chmod -R g-w build/images/ovs
     chmod -R g-w build/images/base
-    DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
+    if [[ $TESTBED_TYPE == "kind" ]]; then
+        DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --build-tag CUSTOM_BUILD_TAG --pull
+    else
+        DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
+    fi
     make flow-aggregator-image
 
     # Enable verbose log for troubleshooting.
@@ -594,10 +604,12 @@ function deliver_antrea {
             ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/antrea-ubuntu.tar; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/flow-aggregator.tar" || true
         done
     elif [[ $TESTBED_TYPE == "kind" ]]; then
-            kind load docker-image antrea/antrea-agent-ubuntu:latest --name ${KIND_CLUSTER}
-            kind load docker-image antrea/antrea-controller-ubuntu:latest --name ${KIND_CLUSTER}
+            kind load docker-image antrea/antrea-agent-ubuntu:$CUSTOM_BUILD_TAG --name ${KIND_CLUSTER}
+            kind load docker-image antrea/antrea-controller-ubuntu:$CUSTOM_BUILD_TAG --name ${KIND_CLUSTER}
             kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} 
             kubectl config use-context kind-${KIND_CLUSTER}
+            sed -i s|"antrea/antrea/agent-ubuntu:latest"|"antrea/antrea-agent-ubuntu:$CUSTOM_BUILD_TAG"|g ./build/yamls/antrea.yml
+            sed -i s|"antrea/antrea/agent-controller:latest"|"antrea/antrea-controller-ubuntu:$CUSTOM_BUILD_TAG"|g ./build/yamls/antrea.yml
             docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml 
     elif [[ $TESTBED_TYPE == "jumper" ]]; then
         kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do

diff --git a/hack/build-antrea-linux-all.sh b/hack/build-antrea-linux-all.sh
@@ -29,14 +29,20 @@ all Dockerfiles.
         --coverage              Build the image with support for code coverage.
         --platform <PLATFORM>   Target platform for the images if server is multi-platform capable.
         --distro <distro>       Target Linux distribution.
-        --no-cache              Do not use the local build cache nor the cached image from the registry."
+        --no-cache              Do not use the local build cache nor the cached image from the registry.
+        --build-tag             Custom build tag for images."
+
+function print_usage {
+    echoerr "$_usage"
+}
 
 PULL=false
 PUSH=false
 NO_CACHE=false
 COVERAGE=false
 PLATFORM=""
 DISTRO="ubuntu"
+CUSTOM_BUILD_TAG=""
 
 while [[ $# -gt 0 ]]
 do
@@ -67,6 +73,10 @@ case $key in
     NO_CACHE=true
     shift
     ;;
+    --build-tag)
+    CUSTOM_BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -118,6 +128,9 @@ GO_VERSION=$(head -n 1 build/images/deps/go-version)
 
 BUILD_TAG=$(build/images/build-tag.sh)
 echo "BUILD_TAG: $BUILD_TAG"
+if [ "$CUSTOM_BUILD_TAG" != "" ]; then
+    ARGS="$ARGS --build-tag $CUSTOM_BUILD_TAG"
+fi
 
 # We pull all images ahead of time, instead of calling the independent build.sh
 # scripts with "--pull". We do not want to overwrite the antrea/openvswitch
@@ -159,15 +172,30 @@ export NO_PULL=1
 export DOCKER_BUILDKIT=1
 if [ "$DISTRO" == "ubuntu" ]; then
     if $COVERAGE; then
-        make build-controller-ubuntu-coverage
-        make build-agent-ubuntu-coverage
+        if [ "$CUSTOM_BUILD_TAG" != "" ]; then
+            make build-controller-ubuntu-coverage CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG
+            make build-agent-ubuntu-coverage CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG
+        else
+            make build-controller-ubuntu-coverage
+            make build-agent-ubuntu-coverage
+        fi
     else
-        make build-controller-ubuntu
-        make build-agent-ubuntu
+        if [ "$CUSTOM_BUILD_TAG" != "" ]; then
+            make build-controller-ubuntu CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG
+            make build-agent-ubuntu CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG
+        else
+            make build-controller-ubuntu
+            make build-agent-ubuntu
+        fi
     fi
 elif [ "$DISTRO" == "ubi" ]; then
-    make build-controller-ubi
-    make build-agent-ubi
+    if [ "$CUSTOM_BUILD_TAG" != "" ]; then
+        make build-controller-ubi CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG
+        make build-agent-ubi CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG
+    else
+        make build-controller-ubi
+        make build-agent-ubi
+    fi
 fi
 
 popd > /dev/null
diff --git a/versioning.mk b/versioning.mk
@@ -34,6 +34,13 @@ else
         DOCKER_IMG_VERSION := $(VERSION)
 endif
 
+ifndef CUSTOM_BUILD_TAG
+        BUILD_TAGS := $(shell build/images/build-tag.sh)
+else
+        BUILD_TAGS := $(CUSTOM_BUILD_TAG)
+        DOCKER_IMG_VERSION := $(BUILD_TAGS)
+endif
+
 VERSION_LDFLAGS = -X antrea.io/antrea/pkg/version.Version=$(VERSION)
 VERSION_LDFLAGS += -X antrea.io/antrea/pkg/version.GitSHA=$(GIT_SHA)
 VERSION_LDFLAGS += -X antrea.io/antrea/pkg/version.GitTreeState=$(GIT_TREE_STATE)
@@ -46,3 +53,4 @@ version-info:
 	@echo "GIT_TREE_STATE: $(GIT_TREE_STATE)"
 	@echo "RELEASE_STATUS: $(RELEASE_STATUS)"
 	@echo "DOCKER_IMG_VERSION: $(DOCKER_IMG_VERSION)"
+        @echo "BUILD_TAG: $(BUILD_TAG)"