From e0e30e58f2c5f5da5a2685f12a7a3d98a223c870 Mon Sep 17 00:00:00 2001 From: Pulkit Jain Date: Mon, 19 Feb 2024 18:12:23 +0530 Subject: [PATCH] Run Multiple kind Jobs on same testbed Signed-off-by: Pulkit Jain --- Makefile | 4 ++-- build/images/base/build.sh | 21 ++++++++++++++--- build/images/ovs/build.sh | 19 +++++++++++++-- ci/jenkins/jobs/macros.yaml | 4 ++-- ci/jenkins/test.sh | 20 ++++++++++++---- hack/build-antrea-linux-all.sh | 42 ++++++++++++++++++++++++++++------ versioning.mk | 8 +++++++ 7 files changed, 98 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index cc337b1f7c6..0e9e716ad55 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,6 @@ OVS_VERSION := $(shell head -n 1 build/images/deps/ovs-version) GO_VERSION := $(shell head -n 1 build/images/deps/go-version) CNI_BINARIES_VERSION := $(shell head -n 1 build/images/deps/cni-binaries-version) NANOSERVER_VERSION := $(shell head -n 1 build/images/deps/nanoserver-version) -BUILD_TAG := $(shell build/images/build-tag.sh) WIN_BUILD_TAG := $(shell echo $(GO_VERSION) $(CNI_BINARIES_VERSION) $(NANOSERVER_VERSION)|md5sum|head -c 10) WIN_OVS_VERSION := $(shell head -n 1 build/images/deps/ovs-version-windows) WIN_BUILD_OVS_TAG := $(NANOSERVER_VERSION)-$(WIN_OVS_VERSION) @@ -42,7 +41,6 @@ endif WIN_BUILD_ARGS := DOCKER_BUILD_ARGS DOCKER_BUILD_ARGS += --build-arg OVS_VERSION=$(OVS_VERSION) DOCKER_BUILD_ARGS += --build-arg GO_VERSION=$(GO_VERSION) -DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG) WIN_BUILD_ARGS := --build-arg GO_VERSION=$(GO_VERSION) WIN_BUILD_ARGS += --build-arg CNI_BINARIES_VERSION=$(CNI_BINARIES_VERSION) WIN_BUILD_ARGS += --build-arg NANOSERVER_VERSION=$(NANOSERVER_VERSION) @@ -57,6 +55,8 @@ all: build include versioning.mk LDFLAGS += $(VERSION_LDFLAGS) +BUILD_TAG = $(BUILD_TAGS) +DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG) UNAME_S := $(shell uname -s) USERID := $(shell id -u) diff --git a/build/images/base/build.sh b/build/images/base/build.sh index 80ad459cf38..150344d120c 100755 --- a/build/images/base/build.sh +++ b/build/images/base/build.sh @@ -29,13 +29,19 @@ Build the antrea base image. --push Push the built image to the registry --platform Target platform for the image if server is multi-platform capable --distro Target Linux distribution - --no-cache Do not use the local build cache nor the cached image from the registry" + --no-cache Do not use the local build cache nor the cached image from the registry + --build-tag Custom build tag for images." + +function print_usage { + echoerr "$_usage" +} PULL=false PUSH=false NO_CACHE=false PLATFORM="" DISTRO="ubuntu" +CUSTOM_BUILD_TAG="" while [[ $# -gt 0 ]] do @@ -62,6 +68,10 @@ case $key in NO_CACHE=true shift ;; + --build-tag) + CUSTOM_BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -104,6 +114,10 @@ SURICATA_VERSION=$(head -n 1 ../deps/suricata-version) BUILD_TAG=$(../build-tag.sh) +if [[ $CUSTOM_BUILD_TAG == "" ]]; then + CUSTOM_BUILD_TAG=$BUILD_TAG +fi + if $PULL; then # The ubuntu image is also used for the UBI build (for the cni-binaries intermediate image). if [[ ${DOCKER_REGISTRY} == "" ]]; then @@ -138,7 +152,8 @@ fi function docker_build_and_push() { local image="$1" local dockerfile="$2" - local build_args="--build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION --build-arg SURICATA_VERSION=$SURICATA_VERSION --build-arg BUILD_TAG=$BUILD_TAG" + local custom_build_tag="$CUSTOM_BUILD_TAG" + local build_args="--build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION --build-arg SURICATA_VERSION=$SURICATA_VERSION --build-arg BUILD_TAG=$custom_build_tag" local cache_args="" if $PUSH; then cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_TAG,mode=max" @@ -148,7 +163,7 @@ function docker_build_and_push() { else cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max" fi - docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile . + docker buildx build $PLATFORM_ARG -o type=docker -t $image:$custom_build_tag $cache_args $build_args -f $dockerfile . if $PUSH; then docker push $image:$BUILD_TAG diff --git a/build/images/ovs/build.sh b/build/images/ovs/build.sh index ac15f1269f4..0da4b46c127 100755 --- a/build/images/ovs/build.sh +++ b/build/images/ovs/build.sh @@ -29,13 +29,19 @@ Build the antrea openvswitch image. --push Push the built image to the registry --platform Target platform for the image if server is multi-platform capable --distro Target Linux distribution - --no-cache Do not use the local build cache nor the cached image from the registry" + --no-cache Do not use the local build cache nor the cached image from the registry + --build-tag Custom build tag for images." + +function print_usage { + echoerr "$_usage" +} PULL=false PUSH=false NO_CACHE=false PLATFORM="" DISTRO="ubuntu" +CUSTOM_BUILD_TAG="" while [[ $# -gt 0 ]] do @@ -62,6 +68,10 @@ case $key in NO_CACHE=true shift ;; + --build-tag) + CUSTOM_BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -103,6 +113,10 @@ OVS_VERSION=$(head -n 1 ../deps/ovs-version) BUILD_TAG=$(../build-tag.sh) +if [[ $CUSTOM_BUILD_TAG == "" ]]; then + CUSTOM_BUILD_TAG=$BUILD_TAG +fi + if $PULL; then if [ "$DISTRO" == "ubuntu" ]; then if [[ ${DOCKER_REGISTRY} == "" ]]; then @@ -120,6 +134,7 @@ fi function docker_build_and_push() { local image="$1" local dockerfile="$2" + local custom_build_tag="$CUSTOM_BUILD_TAG" local build_args="--build-arg OVS_VERSION=$OVS_VERSION" local cache_args="" if $PUSH; then @@ -130,7 +145,7 @@ function docker_build_and_push() { else cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max" fi - docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile . + docker buildx build $PLATFORM_ARG -o type=docker -t $image:$custom_build_tag $cache_args $build_args -f $dockerfile . if $PUSH; then docker push $image:$BUILD_TAG diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml index fe83a8c27e2..34dcfe8b9c5 100644 --- a/ci/jenkins/jobs/macros.yaml +++ b/ci/jenkins/jobs/macros.yaml @@ -227,7 +227,7 @@ ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}' kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config set +e - ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" + ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "${conformance_type}-${{BUILD_NUMBER}}" return_code=$? set -ex ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}" @@ -250,7 +250,7 @@ ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}' kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config set +e - ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" + ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "e2e-${{BUILD_NUMBER}}" return_code=$? set -ex ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}" diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index d00942f1fc7..0afad06e8a2 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -55,6 +55,7 @@ CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=4h" CLEAN_STALE_IMAGES_CONTAINERD="crictl rmi --prune" PRINT_DOCKER_STATUS="docker system df -v" PRINT_CONTAINERD_STATUS="crictl ps --state Exited" +CUSTOM_BUILD_TAG="latest" _usage="Usage: $0 [--kubeconfig ] [--workdir ] [--testcase ] @@ -69,7 +70,8 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o --testbed-type The testbed type to run tests. It can be flexible-ipam, jumper or legacy. --ip-mode IP mode for flexible-ipam e2e test. Default is $DEFAULT_IP_MODE. It can also be ipv6 or ds. --win-image-node Name of the windows image node in containerd cluster. Images are built by docker on this node. - --kind-cluster-name Name of the kind Cluster." + --kind-cluster-name Name of the kind Cluster. + --build-tag Custom build tag for images." function print_usage { echoerr "$_usage" @@ -120,6 +122,10 @@ case $key in WIN_IMAGE_NODE="$2" shift 2 ;; + --build-tag) + CUSTOM_BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -537,7 +543,11 @@ function deliver_antrea { fi chmod -R g-w build/images/ovs chmod -R g-w build/images/base - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + if [[ $TESTBED_TYPE == "kind" ]]; then + DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --build-tag CUSTOM_BUILD_TAG --pull + else + DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + fi make flow-aggregator-image # Enable verbose log for troubleshooting. @@ -591,10 +601,12 @@ function deliver_antrea { ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/antrea-ubuntu.tar; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/flow-aggregator.tar" || true done elif [[ $TESTBED_TYPE == "kind" ]]; then - kind load docker-image antrea/antrea-agent-ubuntu:latest --name ${KIND_CLUSTER} - kind load docker-image antrea/antrea-controller-ubuntu:latest --name ${KIND_CLUSTER} + kind load docker-image antrea/antrea-agent-ubuntu:$CUSTOM_BUILD_TAG --name ${KIND_CLUSTER} + kind load docker-image antrea/antrea-controller-ubuntu:$CUSTOM_BUILD_TAG --name ${KIND_CLUSTER} kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} kubectl config use-context kind-${KIND_CLUSTER} + sed -i s|"antrea/antrea/agent-ubuntu:latest"|"antrea/antrea-agent-ubuntu:$CUSTOM_BUILD_TAG"|g ./build/yamls/antrea.yml + sed -i s|"antrea/antrea/agent-controller:latest"|"antrea/antrea-controller-ubuntu:$CUSTOM_BUILD_TAG"|g ./build/yamls/antrea.yml docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml elif [[ $TESTBED_TYPE == "jumper" ]]; then kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do diff --git a/hack/build-antrea-linux-all.sh b/hack/build-antrea-linux-all.sh index cfc5cfa4fd6..14de6de5614 100755 --- a/hack/build-antrea-linux-all.sh +++ b/hack/build-antrea-linux-all.sh @@ -29,7 +29,12 @@ all Dockerfiles. --coverage Build the image with support for code coverage. --platform Target platform for the images if server is multi-platform capable. --distro Target Linux distribution. - --no-cache Do not use the local build cache nor the cached image from the registry." + --no-cache Do not use the local build cache nor the cached image from the registry. + --build-tag Custom build tag for images." + +function print_usage { + echoerr "$_usage" +} PULL=false PUSH=false @@ -37,6 +42,7 @@ NO_CACHE=false COVERAGE=false PLATFORM="" DISTRO="ubuntu" +CUSTOM_BUILD_TAG="" while [[ $# -gt 0 ]] do @@ -67,6 +73,10 @@ case $key in NO_CACHE=true shift ;; + --build-tag) + CUSTOM_BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -118,6 +128,9 @@ GO_VERSION=$(head -n 1 build/images/deps/go-version) BUILD_TAG=$(build/images/build-tag.sh) echo "BUILD_TAG: $BUILD_TAG" +if [ "$CUSTOM_BUILD_TAG" != "" ]; then + ARGS="$ARGS --build-tag $CUSTOM_BUILD_TAG" +fi # We pull all images ahead of time, instead of calling the independent build.sh # scripts with "--pull". We do not want to overwrite the antrea/openvswitch @@ -159,15 +172,30 @@ export NO_PULL=1 export DOCKER_BUILDKIT=1 if [ "$DISTRO" == "ubuntu" ]; then if $COVERAGE; then - make build-controller-ubuntu-coverage - make build-agent-ubuntu-coverage + if [ "$CUSTOM_BUILD_TAG" != "" ]; then + make build-controller-ubuntu-coverage CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG + make build-agent-ubuntu-coverage CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG + else + make build-controller-ubuntu-coverage + make build-agent-ubuntu-coverage + fi else - make build-controller-ubuntu - make build-agent-ubuntu + if [ "$CUSTOM_BUILD_TAG" != "" ]; then + make build-controller-ubuntu CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG + make build-agent-ubuntu CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG + else + make build-controller-ubuntu + make build-agent-ubuntu + fi fi elif [ "$DISTRO" == "ubi" ]; then - make build-controller-ubi - make build-agent-ubi + if [ "$CUSTOM_BUILD_TAG" != "" ]; then + make build-controller-ubi CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG + make build-agent-ubi CUSTOM_BUILD_TAG=$CUSTOM_BUILD_TAG + else + make build-controller-ubi + make build-agent-ubi + fi fi popd > /dev/null diff --git a/versioning.mk b/versioning.mk index 8d027156dc8..155b75e6538 100644 --- a/versioning.mk +++ b/versioning.mk @@ -34,6 +34,13 @@ else DOCKER_IMG_VERSION := $(VERSION) endif +ifndef CUSTOM_BUILD_TAG + BUILD_TAGS := $(shell build/images/build-tag.sh) +else + BUILD_TAGS := $(CUSTOM_BUILD_TAG) + DOCKER_IMG_VERSION := $(BUILD_TAGS) +endif + VERSION_LDFLAGS = -X antrea.io/antrea/pkg/version.Version=$(VERSION) VERSION_LDFLAGS += -X antrea.io/antrea/pkg/version.GitSHA=$(GIT_SHA) VERSION_LDFLAGS += -X antrea.io/antrea/pkg/version.GitTreeState=$(GIT_TREE_STATE) @@ -46,3 +53,4 @@ version-info: @echo "GIT_TREE_STATE: $(GIT_TREE_STATE)" @echo "RELEASE_STATUS: $(RELEASE_STATUS)" @echo "DOCKER_IMG_VERSION: $(DOCKER_IMG_VERSION)" + @echo "BUILD_TAG: $(BUILD_TAG)"