Update BuildKit Configurations to Enhance Compatibility (#5937) (#6450)

* Add BuildKit cache cleanup in CI
* Declare DOCKER_BUILDKIT when building image for docker compatibility.

fixes #5941

Signed-off-by: Shuyang Xin <[email protected]>

ci/jenkins/test-mc.sh

@@ -137,6 +137,7 @@ function clean_images() {
     docker images | grep -E 'mc-controller|antrea-ubuntu' | awk '{print $3}' | xargs -r docker rmi -f || true
     # Clean up dangling images generated in previous builds.
     docker image prune -f --filter "until=24h" || true > /dev/null
+    check_and_cleanup_docker_build_cache
 }
 
 function cleanup_multicluster_ns {

ci/jenkins/test-rancher.sh

@@ -138,6 +138,7 @@ function deliver_antrea {
     git show --numstat
     make clean
     ${CLEAN_STALE_IMAGES}
+    check_and_upgrade_golang
     chmod -R g-w build/images/ovs
     chmod -R g-w build/images/base
     DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull

ci/jenkins/test-vmc.sh

@@ -355,6 +355,7 @@ function deliver_antrea {
     # because they might be being used in other builds running simultaneously.
     docker image prune -af --filter "until=1h" > /dev/null
     docker system df -v
+    check_and_cleanup_docker_build_cache
     set -e
 
     cd $GIT_CHECKOUT_DIR

ci/jenkins/test.sh

@@ -186,6 +186,7 @@ function clean_antrea {
     done
     docker images | grep 'antrea' | awk '{print $3}' | xargs -r docker rmi || true
     docker images | grep '<none>' | awk '{print $3}' | xargs -r docker rmi || true
+    check_and_cleanup_docker_build_cache
 }
 
 function clean_for_windows_install_cni {

ci/jenkins/utils.sh

@@ -14,6 +14,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+function check_and_cleanup_docker_build_cache() {
+    free_space=$(df -h -B 1G / | awk 'NR==2 {print $4}')
+    free_space_threshold=40
+    if [[ $free_space -lt $free_space_threshold ]]; then
+        # If cleaning up unused dangling images doesn't free up sufficient disk space,
+        # we will have to reduce the builder cache to 10GB to release enough disk space.
+        docker builder prune -af --keep-storage=10gb > /dev/null
+        free_space=$(df -h -B 1G / | awk 'NR==2 {print $4}')
+        if [[ $free_space -lt $free_space_threshold ]]; then
+            # If the first round cleanup doesn't free up sufficient disk space,
+            # we will have to clean up all builder cache to release enough disk space.
+            docker builder prune -af > /dev/null
+        fi
+    fi
+    docker system df -v
+}
+
 function check_and_upgrade_golang() {
     if [ -z "${GOLANG_RELEASE_DIR}" ]; then
         GOLANG_RELEASE_DIR="/var/lib/jenkins/golang-releases"

ci/test-conformance-aks.sh

@@ -193,10 +193,12 @@ function deliver_antrea_to_aks() {
     # because they might be being used in other builds running simultaneously.
     docker image prune -f --filter "until=2h" > /dev/null
     docker system df -v
+    check_and_cleanup_docker_build_cache
+
     set -e
 
     cd ${GIT_CHECKOUT_DIR}
-    VERSION="$CLUSTER" make
+    VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull
     if [[ "$?" -ne "0" ]]; then
         echo "=== Antrea Image build failed ==="
         exit 1

ci/test-conformance-eks.sh

@@ -270,10 +270,11 @@ function deliver_antrea_to_eks() {
     # because they might be being used in other builds running simultaneously.
     docker image prune -f --filter "until=2h" > /dev/null
     docker system df -v
+    check_and_cleanup_docker_build_cache
     set -e
 
     cd ${GIT_CHECKOUT_DIR}
-    VERSION="$CLUSTER" make
+    VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull
     if [[ "$?" -ne "0" ]]; then
         echo "=== Antrea Image build failed ==="
         exit 1

ci/test-conformance-gke.sh

@@ -211,10 +211,11 @@ function deliver_antrea_to_gke() {
     # because they might be being used in other builds running simultaneously.
     docker image prune -f --filter "until=2h" > /dev/null
     docker system df -v
+    check_and_cleanup_docker_build_cache
     set -e
 
     cd ${GIT_CHECKOUT_DIR}
-    VERSION="$CLUSTER" make -C ${GIT_CHECKOUT_DIR}
+    VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull
     if [[ "$?" -ne "0" ]]; then
         echo "=== Antrea Image build failed ==="
         exit 1

docs/minikube.md

@@ -26,7 +26,7 @@ minikube start --cni=antrea.yml --network-plugin=cni
 
 These instructions assume that you have built the Antrea Docker image locally
 (e.g. by running `make` from the root of the repository, or in case of arm64 architecture by running
-`DOCKER_BUILDKIT=1 ./hack/build-antrea-ubuntu-all.sh --platform linux/arm64`).
+`./hack/build-antrea-linux-all.sh --platform linux/arm64`).
 
 ```bash
 # load the Antrea Docker image in the minikube nodes

hack/build-antrea-linux-all.sh

@@ -167,6 +167,7 @@ cd build/images/base
 cd -
 
 export NO_PULL=1
+export DOCKER_BUILDKIT=1
 if [ "$DISTRO" == "ubuntu" ]; then
     if $COVERAGE; then
         make build-controller-ubuntu-coverage