Skip to content

Commit

Permalink
Update BuildKit Configurations to Enhance Compatibility
Browse files Browse the repository at this point in the history 
* Add BuildKit cache cleanup in CI
* Declare DOCKER_BUILDKIT when building image for docker compatibility.

fixes #5941

Signed-off-by: Shuyang Xin <[email protected]>
  • Loading branch information
@XinShuYang
XinShuYang committed Jan 30, 2024
1 parent 5135e06 commit 47fba78
Show file tree
Hide file tree
Showing 9 changed files with 37 additions and 14 deletions.
3 changes: 2 additions & 1 deletion ci/jenkins/test-mc.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ function clean_images() {
docker images | grep -E 'mc-controller|antrea-ubuntu' | awk '{print $3}' | xargs -r docker rmi -f || true
# Clean up dangling images generated in previous builds.
docker image prune -f --filter "until=24h" || true > /dev/null
check_and_cleanup_buildkit_cache
}

function cleanup_multicluster_ns {
Expand Down Expand Up @@ -296,7 +297,7 @@ function deliver_antrea_multicluster {
chmod -R g-w build/images/ovs
chmod -R g-w build/images/base

DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
echo "====== Delivering Antrea to all Nodes ======"
docker save -o ${WORKDIR}/antrea-ubuntu.tar antrea/antrea-agent-ubuntu:latest antrea/antrea-controller-ubuntu:latest

Expand Down
3 changes: 2 additions & 1 deletion ci/jenkins/test-rancher.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,10 @@ function deliver_antrea {
git show --numstat
make clean
${CLEAN_STALE_IMAGES}
check_and_upgrade_golang
chmod -R g-w build/images/ovs
chmod -R g-w build/images/base
DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
make flow-aggregator-image

# Enable verbose log for troubleshooting.
Expand Down
4 changes: 2 additions & 2 deletions ci/jenkins/test-vm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,12 @@ function apply_antrea {
chmod -R g-w build/images/base
# Pull images from Dockerhub first then try Harbor.
for i in `seq 3`; do
./hack/build-antrea-linux-all.sh --pull && break
DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break
done
if [ $? -ne 0 ]; then
echoerr "Failed to build antrea images with Dockerhub"
for i in `seq 3`; do
DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull && break
DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break
done
if [ $? -ne 0 ]; then
echoerr "Failed to build antrea images with Harbor"
Expand Down
9 changes: 5 additions & 4 deletions ci/jenkins/test-vmc.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ function deliver_antrea {
# because they might be being used in other builds running simultaneously.
docker image prune -af --filter "until=1h" > /dev/null
docker system df -v
check_and_cleanup_buildkit_cache
set -e

cd $GIT_CHECKOUT_DIR
Expand All @@ -365,18 +366,18 @@ function deliver_antrea {
# Pull images from Dockerhub first then try Harbor.
for i in `seq 3`; do
if [[ "$COVERAGE" == true ]]; then
VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull --coverage && break
VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull --coverage && break
else
VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull && break
VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break
fi
done
if [ $? -ne 0 ]; then
echoerr "Failed to build antrea images with Dockerhub"
for i in `seq 3`; do
if [[ "$COVERAGE" == true ]]; then
VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull --coverage && break
VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull --coverage && break
else
VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull && break
VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull && break
fi
done
if [ $? -ne 0 ]; then
Expand Down
7 changes: 4 additions & 3 deletions ci/jenkins/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ function clean_antrea {
done
docker images | grep 'antrea' | awk '{print $3}' | xargs -r docker rmi || true
docker images | grep '<none>' | awk '{print $3}' | xargs -r docker rmi || true
check_and_cleanup_buildkit_cache
}

function clean_for_windows_install_cni {
Expand Down Expand Up @@ -378,7 +379,7 @@ function deliver_antrea_linux {
set -e

echo "==== Start building and delivering Linux Docker images ===="
DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
if [[ "$TESTCASE" == "windows-networkpolicy-process" ]]; then
make windows-bin
fi
Expand Down Expand Up @@ -580,7 +581,7 @@ function deliver_antrea_linux_containerd {
set -e

echo "==== Start building and delivering Linux containerd images ===="
DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
docker save -o antrea-ubuntu.tar antrea/antrea-agent-ubuntu:latest antrea/antrea-controller-ubuntu:latest
echo "===== Pull necessary images on Control-Plane node ====="
harbor_images=("agnhost:2.13" "nginx:1.15-alpine")
Expand Down Expand Up @@ -704,7 +705,7 @@ function deliver_antrea {
fi
chmod -R g-w build/images/ovs
chmod -R g-w build/images/base
DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
DOCKER_REGISTRY="${DOCKER_REGISTRY}" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
make flow-aggregator-image

# Enable verbose log for troubleshooting.
Expand Down
16 changes: 16 additions & 0 deletions ci/jenkins/utils.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.

function check_and_cleanup_buildkit_cache() {
free_space=$(df -h -B 1G / | awk 'NR==2 {print $4}')
free_space_threshold=40
if [[ $free_space -lt $free_space_threshold ]]; then
# If cleaning up unused dangling images doesn't free up sufficient disk space,
# we will have to reduce the buildkit cache to 10GB to release enough disk space.
docker buildx prune -af --keep-storage=10gb > /dev/null
if [[ $free_space -lt $free_space_threshold ]]; then
# If the first round cleanup doesn't free up sufficient disk space,
# we will have to clean up all buildkit cache to release enough disk space.
docker buildx prune -af > /dev/null
fi
fi
docker buildx du
}

function check_and_upgrade_golang() {
if [ -z "${GOLANG_RELEASE_DIR}" ]; then
GOLANG_RELEASE_DIR="/var/lib/jenkins/golang-releases"
Expand Down
3 changes: 2 additions & 1 deletion ci/test-conformance-aks.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,10 +193,11 @@ function deliver_antrea_to_aks() {
# because they might be being used in other builds running simultaneously.
docker image prune -f --filter "until=2h" > /dev/null
docker system df -v
check_and_cleanup_buildkit_cache
set -e

cd ${GIT_CHECKOUT_DIR}
VERSION="$CLUSTER" make
VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
if [[ "$?" -ne "0" ]]; then
echo "=== Antrea Image build failed ==="
exit 1
Expand Down
3 changes: 2 additions & 1 deletion ci/test-conformance-eks.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,11 @@ function deliver_antrea_to_eks() {
# because they might be being used in other builds running simultaneously.
docker image prune -f --filter "until=2h" > /dev/null
docker system df -v
check_and_cleanup_buildkit_cache
set -e

cd ${GIT_CHECKOUT_DIR}
VERSION="$CLUSTER" make
VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
if [[ "$?" -ne "0" ]]; then
echo "=== Antrea Image build failed ==="
exit 1
Expand Down
3 changes: 2 additions & 1 deletion ci/test-conformance-gke.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,10 +211,11 @@ function deliver_antrea_to_gke() {
# because they might be being used in other builds running simultaneously.
docker image prune -f --filter "until=2h" > /dev/null
docker system df -v
check_and_cleanup_buildkit_cache
set -e

cd ${GIT_CHECKOUT_DIR}
VERSION="$CLUSTER" make -C ${GIT_CHECKOUT_DIR}
VERSION="$CLUSTER" DOCKER_BUILDKIT=1 ./hack/build-antrea-linux-all.sh --pull
if [[ "$?" -ne "0" ]]; then
echo "=== Antrea Image build failed ==="
exit 1
Expand Down

0 comments on commit 47fba78

Please sign in to comment.