From 7a7a257f6298660cd16fce33ac1d41cd1d500709 Mon Sep 17 00:00:00 2001 From: Qingsheng Ren Date: Tue, 6 Aug 2024 15:44:06 +0800 Subject: [PATCH] [FLINK-35974][e2e] Use docker compose v2 for E2E tests GitHub CI image dropped support of Docker Compose v1 (with syntax `docker-compose`). The new v2 version (`docker compose`) should be used for E2E scripts --- flink-end-to-end-tests/test-scripts/common_docker.sh | 1 - .../test-scripts/common_yarn_docker.sh | 6 +++--- .../docker-hadoop-secure-cluster/README.md | 8 ++++---- .../test-scripts/test_docker_embedded_job.sh | 8 ++++---- flink-end-to-end-tests/test-scripts/test_nat.sh | 10 +++++----- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/flink-end-to-end-tests/test-scripts/common_docker.sh b/flink-end-to-end-tests/test-scripts/common_docker.sh index 2ce590cd424c0..08637498e8d36 100644 --- a/flink-end-to-end-tests/test-scripts/common_docker.sh +++ b/flink-end-to-end-tests/test-scripts/common_docker.sh @@ -21,7 +21,6 @@ set -o pipefail source "$(dirname "$0")"/common.sh docker --version -docker-compose --version function containers_health_check() { local container_names=${@:1} diff --git a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh index 299c201868498..0d298d92d79f1 100755 --- a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh +++ b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh @@ -42,14 +42,14 @@ function cluster_shutdown { if [ ${TRAPPED_EXIT_CODE} != 0 ];then debug_copy_and_show_logs fi - docker-compose -f "${END_TO_END_DIR}/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml" down + docker compose -f "${END_TO_END_DIR}/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml" down rm "${FLINK_TARBALL_DIR}/${FLINK_TARBALL}" } on_exit cluster_shutdown function start_hadoop_cluster() { echo "Starting Hadoop cluster" - docker-compose -f "${END_TO_END_DIR}/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml" up -d + docker compose -f "${END_TO_END_DIR}/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml" up -d # Wait for kerberos to be set up local start_time @@ -103,7 +103,7 @@ function build_image() { ln "${cache_path}" "${END_TO_END_DIR}/test-scripts/docker-hadoop-secure-cluster/hadoop/hadoop.tar.gz" echo "Building Hadoop Docker container" - docker-compose -f "${END_TO_END_DIR}/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml" build + docker compose -f "${END_TO_END_DIR}/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml" build } function start_hadoop_cluster_and_prepare_flink() { diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md index d94fc534b06a4..a59d58275a3e1 100644 --- a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md +++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md @@ -25,8 +25,8 @@ Run image ``` cd flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster wget -O hadoop/hadoop.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-2.10.2/hadoop-2.10.2.tar.gz -docker-compose build -docker-compose up +docker compose build +docker compose up ``` Usage @@ -56,7 +56,7 @@ Known issues ### Unable to obtain Kerberos password #### Error -docker-compose up fails for the first time with the error +docker compose up fails for the first time with the error ``` Login failure for nn/hadoop.docker.com@EXAMPLE.COM from keytab /etc/security/keytabs/nn.service.keytab: javax.security.auth.login.LoginException: Unable to obtain password from user @@ -64,7 +64,7 @@ Login failure for nn/hadoop.docker.com@EXAMPLE.COM from keytab /etc/security/key #### Solution -Stop the containers with `docker-compose down` and start again with `docker-compose up -d`. +Stop the containers with `docker compose down` and start again with `docker compose up -d`. ### Java Keystore diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh index 4bee725efc652..92050644255bf 100755 --- a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh +++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh @@ -61,9 +61,9 @@ if ! retry_times $DOCKER_IMAGE_BUILD_RETRIES ${BUILD_BACKOFF_TIME} "build_image fi export USER_LIB=${FLINK_DIR}/examples/batch -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --force-recreate --abort-on-container-exit --exit-code-from job-cluster &> /dev/null -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs job-cluster > $FLINK_LOG_DIR/jobmanager.log -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs taskmanager > $FLINK_LOG_DIR/taskmanager.log -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml rm -f +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --force-recreate --abort-on-container-exit --exit-code-from job-cluster &> /dev/null +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs job-cluster > $FLINK_LOG_DIR/jobmanager.log +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs taskmanager > $FLINK_LOG_DIR/taskmanager.log +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.test.yml rm -f check_result_hash "WordCount" $OUTPUT_VOLUME/docker_wc_out "${RESULT_HASH}" diff --git a/flink-end-to-end-tests/test-scripts/test_nat.sh b/flink-end-to-end-tests/test-scripts/test_nat.sh index b5d1f550f4cc1..f9129daa9f6a0 100755 --- a/flink-end-to-end-tests/test-scripts/test_nat.sh +++ b/flink-end-to-end-tests/test-scripts/test_nat.sh @@ -64,10 +64,10 @@ fi popd export USER_LIB=${FLINK_DIR}/examples/batch -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml up --force-recreate --abort-on-container-exit --exit-code-from job-cluster &> /dev/null -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml logs job-cluster > $FLINK_LOG_DIR/jobmanager.log -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml logs taskmanager1 > $FLINK_LOG_DIR/taskmanager1.log -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml logs taskmanager2 > $FLINK_LOG_DIR/taskmanager2.log -docker-compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml rm -f +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml up --force-recreate --abort-on-container-exit --exit-code-from job-cluster &> /dev/null +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml logs job-cluster > $FLINK_LOG_DIR/jobmanager.log +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml logs taskmanager1 > $FLINK_LOG_DIR/taskmanager1.log +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml logs taskmanager2 > $FLINK_LOG_DIR/taskmanager2.log +docker compose -f ${DOCKER_SCRIPTS}/docker-compose.nat.yml rm -f check_result_hash "WordCount" ${OUTPUT_VOLUME}/${OUTPUT_PREFIX}/ "${RESULT_HASH}"