diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 36f7968dc25..74a821bdf43 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -137,6 +137,7 @@ function clean_images() { docker images | grep -E 'mc-controller|antrea-ubuntu' | awk '{print $3}' | xargs -r docker rmi -f || true # Clean up dangling images generated in previous builds. docker image prune -f --filter "until=24h" || true > /dev/null + check_and_cleanup_buildkit_cache } function cleanup_multicluster_ns { @@ -296,7 +297,7 @@ function deliver_antrea_multicluster { chmod -R g-w build/images/ovs chmod -R g-w build/images/base - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull echo "====== Delivering Antrea to all Nodes ======" docker save -o ${WORKDIR}/antrea-ubuntu.tar antrea/antrea-agent-ubuntu:latest antrea/antrea-controller-ubuntu:latest diff --git a/ci/jenkins/test-rancher.sh b/ci/jenkins/test-rancher.sh index bb2d82eec71..3fa025a2a71 100644 --- a/ci/jenkins/test-rancher.sh +++ b/ci/jenkins/test-rancher.sh @@ -138,9 +138,10 @@ function deliver_antrea { git show --numstat make clean ${CLEAN_STALE_IMAGES} + check_and_upgrade_golang chmod -R g-w build/images/ovs chmod -R g-w build/images/base - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull make flow-aggregator-image # Enable verbose log for troubleshooting. diff --git a/ci/jenkins/test-vm.sh b/ci/jenkins/test-vm.sh index 556999e9200..9b8c3f45e95 100755 --- a/ci/jenkins/test-vm.sh +++ b/ci/jenkins/test-vm.sh @@ -129,12 +129,12 @@ function apply_antrea { chmod -R g-w build/images/base # Pull images from Dockerhub first then try Harbor. for i in `seq 3`; do - ./hack/build-antrea-linux-all.sh --pull && break + DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break done if [ $? -ne 0 ]; then echoerr "Failed to build antrea images with Dockerhub" for i in `seq 3`; do - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull && break + DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break done if [ $? -ne 0 ]; then echoerr "Failed to build antrea images with Harbor" diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh index 8376edac8c5..8823464809a 100755 --- a/ci/jenkins/test-vmc.sh +++ b/ci/jenkins/test-vmc.sh @@ -355,6 +355,7 @@ function deliver_antrea { # because they might be being used in other builds running simultaneously. docker image prune -af --filter "until=1h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd $GIT_CHECKOUT_DIR @@ -365,18 +366,18 @@ function deliver_antrea { # Pull images from Dockerhub first then try Harbor. for i in `seq 3`; do if [[ "$COVERAGE" == true ]]; then - VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull --coverage && break + VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull --coverage && break else - VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull && break + VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break fi done if [ $? -ne 0 ]; then echoerr "Failed to build antrea images with Dockerhub" for i in `seq 3`; do if [[ "$COVERAGE" == true ]]; then - VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull --coverage && break + VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull --coverage && break else - VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull && break + VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break fi done if [ $? -ne 0 ]; then diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index b8f31dc4d91..bccb2ee3ccf 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -186,6 +186,7 @@ function clean_antrea { done docker images | grep 'antrea' | awk '{print $3}' | xargs -r docker rmi || true docker images | grep '' | awk '{print $3}' | xargs -r docker rmi || true + check_and_cleanup_buildkit_cache } function clean_for_windows_install_cni { @@ -378,7 +379,7 @@ function deliver_antrea_linux { set -e echo "==== Start building and delivering Linux Docker images ====" - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull if [[ "$TESTCASE" == "windows-networkpolicy-process" ]]; then make windows-bin fi @@ -580,7 +581,7 @@ function deliver_antrea_linux_containerd { set -e echo "==== Start building and delivering Linux containerd images ====" - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull docker save -o antrea-ubuntu.tar antrea/antrea-agent-ubuntu:latest antrea/antrea-controller-ubuntu:latest echo "===== Pull necessary images on Control-Plane node =====" harbor_images=("agnhost:2.13" "nginx:1.15-alpine") @@ -704,7 +705,7 @@ function deliver_antrea { fi chmod -R g-w build/images/ovs chmod -R g-w build/images/base - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull make flow-aggregator-image # Enable verbose log for troubleshooting. diff --git a/ci/jenkins/utils.sh b/ci/jenkins/utils.sh index 988ed891513..9916ee04b29 100644 --- a/ci/jenkins/utils.sh +++ b/ci/jenkins/utils.sh @@ -14,6 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +function check_and_cleanup_buildkit_cache() { + free_space=$(df -h -B 1G / | awk 'NR==2 {print $4}') + free_space_threshold=40 + if [[ $free_space -lt $free_space_threshold ]]; then + # If cleaning up unused dangling images doesn't free up sufficient disk space, + # we will have to reduce the buildkit cache to 10GB to release enough disk space. + docker buildx prune -af --keep-storage=10gb > /dev/null + if [[ $free_space -lt $free_space_threshold ]]; then + # If the first round cleanup doesn't free up sufficient disk space, + # we will have to clean up all buildkit cache to release enough disk space. + docker buildx prune -af > /dev/null + fi + fi + docker buildx du +} + function check_and_upgrade_golang() { if [ -z "${GOLANG_RELEASE_DIR}" ]; then GOLANG_RELEASE_DIR="/var/lib/jenkins/golang-releases" diff --git a/ci/test-conformance-aks.sh b/ci/test-conformance-aks.sh index 4d508434d70..9e4ea727920 100755 --- a/ci/test-conformance-aks.sh +++ b/ci/test-conformance-aks.sh @@ -193,10 +193,11 @@ function deliver_antrea_to_aks() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make + VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/ci/test-conformance-eks.sh b/ci/test-conformance-eks.sh index aaf915e1b6b..8de9d6385ef 100755 --- a/ci/test-conformance-eks.sh +++ b/ci/test-conformance-eks.sh @@ -270,10 +270,11 @@ function deliver_antrea_to_eks() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make + VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/ci/test-conformance-gke.sh b/ci/test-conformance-gke.sh index 283413d2865..3b94a1ed8bf 100755 --- a/ci/test-conformance-gke.sh +++ b/ci/test-conformance-gke.sh @@ -211,10 +211,11 @@ function deliver_antrea_to_gke() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_buildkit_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make -C ${GIT_CHECKOUT_DIR} + VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1