From 37146df0cf8624134bc790043a5659a86cc5f87f Mon Sep 17 00:00:00 2001 From: Shuyang Xin Date: Mon, 29 Jan 2024 11:22:05 +0800 Subject: [PATCH] Update BuildKit Configurations to Enhance Compatibility * Add BuildKit cache cleanup in CI * Declare DOCKER_BUILDKIT when building image for docker compatibility. fixes #5941 Signed-off-by: Shuyang Xin --- ci/jenkins/test-mc.sh | 1 + ci/jenkins/test-rancher.sh | 1 + ci/jenkins/test-vmc.sh | 1 + ci/jenkins/test.sh | 1 + ci/jenkins/utils.sh | 13 +++++++++++++ ci/test-conformance-aks.sh | 3 ++- ci/test-conformance-eks.sh | 3 ++- ci/test-conformance-gke.sh | 3 ++- docs/minikube.md | 2 +- hack/build-antrea-linux-all.sh | 1 + 10 files changed, 25 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 36f7968dc25..d8cfc6cbc32 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -137,6 +137,7 @@ function clean_images() { docker images | grep -E 'mc-controller|antrea-ubuntu' | awk '{print $3}' | xargs -r docker rmi -f || true # Clean up dangling images generated in previous builds. docker image prune -f --filter "until=24h" || true > /dev/null + check_and_cleanup_buildkit_cache } function cleanup_multicluster_ns { diff --git a/ci/jenkins/test-rancher.sh b/ci/jenkins/test-rancher.sh index bb2d82eec71..d0148faecec 100644 --- a/ci/jenkins/test-rancher.sh +++ b/ci/jenkins/test-rancher.sh @@ -138,6 +138,7 @@ function deliver_antrea { git show --numstat make clean ${CLEAN_STALE_IMAGES} + check_and_upgrade_golang chmod -R g-w build/images/ovs chmod -R g-w build/images/base DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh index 8376edac8c5..9212c86580a 100755 --- a/ci/jenkins/test-vmc.sh +++ b/ci/jenkins/test-vmc.sh @@ -355,6 +355,7 @@ function deliver_antrea { # because they might be being used in other builds running simultaneously. docker image prune -af --filter "until=1h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd $GIT_CHECKOUT_DIR diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index b8f31dc4d91..6fb4b04e660 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -186,6 +186,7 @@ function clean_antrea { done docker images | grep 'antrea' | awk '{print $3}' | xargs -r docker rmi || true docker images | grep '' | awk '{print $3}' | xargs -r docker rmi || true + check_and_cleanup_buildkit_cache } function clean_for_windows_install_cni { diff --git a/ci/jenkins/utils.sh b/ci/jenkins/utils.sh index 988ed891513..43b2bb6be43 100644 --- a/ci/jenkins/utils.sh +++ b/ci/jenkins/utils.sh @@ -14,6 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +function check_and_cleanup_buildkit_cache() { + free_space_threshold=40 + + docker builder prune -af --keep-storage=10gb > /dev/null + free_space=$(df -h -B 1G / | awk 'NR==2 {print $4}') + if [[ $free_space -lt $free_space_threshold ]]; then + # If the first round cleanup doesn't free up sufficient disk space, + # we will have to clean up all buildkit cache to release enough disk space. + docker builder prune -af > /dev/null + fi + docker system df -v +} + function check_and_upgrade_golang() { if [ -z "${GOLANG_RELEASE_DIR}" ]; then GOLANG_RELEASE_DIR="/var/lib/jenkins/golang-releases" diff --git a/ci/test-conformance-aks.sh b/ci/test-conformance-aks.sh index 4d508434d70..1683120a334 100755 --- a/ci/test-conformance-aks.sh +++ b/ci/test-conformance-aks.sh @@ -193,10 +193,11 @@ function deliver_antrea_to_aks() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make + VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/ci/test-conformance-eks.sh b/ci/test-conformance-eks.sh index aaf915e1b6b..f96a176839e 100755 --- a/ci/test-conformance-eks.sh +++ b/ci/test-conformance-eks.sh @@ -270,10 +270,11 @@ function deliver_antrea_to_eks() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make + VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/ci/test-conformance-gke.sh b/ci/test-conformance-gke.sh index 283413d2865..f0ae150b190 100755 --- a/ci/test-conformance-gke.sh +++ b/ci/test-conformance-gke.sh @@ -211,10 +211,11 @@ function deliver_antrea_to_gke() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make -C ${GIT_CHECKOUT_DIR} + VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/docs/minikube.md b/docs/minikube.md index ebd9f4424e1..bfe3b70a8cc 100644 --- a/docs/minikube.md +++ b/docs/minikube.md @@ -26,7 +26,7 @@ minikube start --cni=antrea.yml --network-plugin=cni These instructions assume that you have built the Antrea Docker image locally (e.g. by running `make` from the root of the repository, or in case of arm64 architecture by running -`DOCKER_BUILDKIT=1 ./hack/build-antrea-ubuntu-all.sh --platform linux/arm64`). +`./hack/build-antrea-ubuntu-all.sh --platform linux/arm64`). ```bash # load the Antrea Docker image in the minikube nodes diff --git a/hack/build-antrea-linux-all.sh b/hack/build-antrea-linux-all.sh index 56604ccd08c..b48766ce7b6 100755 --- a/hack/build-antrea-linux-all.sh +++ b/hack/build-antrea-linux-all.sh @@ -165,6 +165,7 @@ cd build/images/base cd - export NO_PULL=1 +export DOCKER_BUILDKIT=1 if [ "$DISTRO" == "ubuntu" ]; then if $COVERAGE; then make build-controller-ubuntu-coverage