From 575d93c0e573822415436edcb6d57b040556629c Mon Sep 17 00:00:00 2001 From: Dong Lin Date: Wed, 26 Jul 2023 17:34:53 +0800 Subject: [PATCH] checkpoint --- flink-derived-feature-view/run_and_verify.sh | 2 +- flink-filesystem-join-redis/run_and_verify.sh | 2 +- flink-kafka-join-filesystem/run_and_verify.sh | 2 +- flink-read-write-hdfs/run_and_verify.sh | 2 +- flink-read-write-hive/run_and_verify.sh | 2 +- flink-read-write-mysql/run_and_verify.sh | 2 +- flink-read-write-redis/run_and_verify.sh | 2 +- flink-sliding-feature-view-benchmark/run_and_verify.sh | 2 +- flink-sliding-feature-view/run_and_verify.sh | 2 +- flink-sql-feature-view/run_and_verify.sh | 2 +- spark-derived-feature-view/run_and_verify.sh | 2 +- tools/utils.sh | 10 +++++++++- 12 files changed, 20 insertions(+), 12 deletions(-) diff --git a/flink-derived-feature-view/run_and_verify.sh b/flink-derived-feature-view/run_and_verify.sh index a3439d7..0a486c6 100755 --- a/flink-derived-feature-view/run_and_verify.sh +++ b/flink-derived-feature-view/run_and_verify.sh @@ -25,7 +25,7 @@ docker-compose up -d wait_for_port 8081 "Flink Cluster" python main.py -docker-compose down +tear_down_docker cat data/output.json/* > data/merged_output diff --git a/flink-filesystem-join-redis/run_and_verify.sh b/flink-filesystem-join-redis/run_and_verify.sh index 16bdd06..5d4f3dd 100755 --- a/flink-filesystem-join-redis/run_and_verify.sh +++ b/flink-filesystem-join-redis/run_and_verify.sh @@ -26,7 +26,7 @@ wait_for_port 8081 "Flink Cluster" python initialize_redis.py python main.py -docker-compose down +tear_down_docker cat data/output.json/* > data/merged_output diff --git a/flink-kafka-join-filesystem/run_and_verify.sh b/flink-kafka-join-filesystem/run_and_verify.sh index 965b5b6..5b44867 100755 --- a/flink-kafka-join-filesystem/run_and_verify.sh +++ b/flink-kafka-join-filesystem/run_and_verify.sh @@ -50,7 +50,7 @@ while true; do done kill "${PID}" -docker-compose down +tear_down_docker sort_and_compare_files data/kafka-output data/expected_output.txt diff --git a/flink-read-write-hdfs/run_and_verify.sh b/flink-read-write-hdfs/run_and_verify.sh index a3439d7..0a486c6 100755 --- a/flink-read-write-hdfs/run_and_verify.sh +++ b/flink-read-write-hdfs/run_and_verify.sh @@ -25,7 +25,7 @@ docker-compose up -d wait_for_port 8081 "Flink Cluster" python main.py -docker-compose down +tear_down_docker cat data/output.json/* > data/merged_output diff --git a/flink-read-write-hive/run_and_verify.sh b/flink-read-write-hive/run_and_verify.sh index d2f4782..1f7acdc 100755 --- a/flink-read-write-hive/run_and_verify.sh +++ b/flink-read-write-hive/run_and_verify.sh @@ -45,7 +45,7 @@ while true; do done kill "${PID}" -docker-compose down +tear_down_docker cat data/output/.part-* > data/merged_output diff --git a/flink-read-write-mysql/run_and_verify.sh b/flink-read-write-mysql/run_and_verify.sh index 6e1da36..ef34379 100755 --- a/flink-read-write-mysql/run_and_verify.sh +++ b/flink-read-write-mysql/run_and_verify.sh @@ -27,6 +27,6 @@ wait_for_port 8081 "Flink Cluster" python initialize_mysql_table.py python main.py exit_code=$? -docker-compose down +tear_down_docker exit $exit_code diff --git a/flink-read-write-redis/run_and_verify.sh b/flink-read-write-redis/run_and_verify.sh index ff762d7..417a815 100755 --- a/flink-read-write-redis/run_and_verify.sh +++ b/flink-read-write-redis/run_and_verify.sh @@ -26,6 +26,6 @@ wait_for_port 8081 "Flink Cluster" python main.py exit_code=$? -docker-compose down +tear_down_docker exit $exit_code diff --git a/flink-sliding-feature-view-benchmark/run_and_verify.sh b/flink-sliding-feature-view-benchmark/run_and_verify.sh index a24d238..17276b0 100755 --- a/flink-sliding-feature-view-benchmark/run_and_verify.sh +++ b/flink-sliding-feature-view-benchmark/run_and_verify.sh @@ -25,4 +25,4 @@ wait_for_port 8081 "Flink Cluster" python main.py --records-num 1000 -docker-compose down +tear_down_docker diff --git a/flink-sliding-feature-view/run_and_verify.sh b/flink-sliding-feature-view/run_and_verify.sh index 127c3c4..54ef7a3 100755 --- a/flink-sliding-feature-view/run_and_verify.sh +++ b/flink-sliding-feature-view/run_and_verify.sh @@ -50,7 +50,7 @@ while true; do done kill "${PID}" -docker-compose down +tear_down_docker sort_and_compare_files data/kafka-output data/expected_output.txt diff --git a/flink-sql-feature-view/run_and_verify.sh b/flink-sql-feature-view/run_and_verify.sh index fc5d0c8..9740a7f 100755 --- a/flink-sql-feature-view/run_and_verify.sh +++ b/flink-sql-feature-view/run_and_verify.sh @@ -25,7 +25,7 @@ docker-compose up -d wait_for_port 8081 "Flink Cluster" python main.py -docker-compose down +tear_down_docker cat data/output.json/* > data/merged_output diff --git a/spark-derived-feature-view/run_and_verify.sh b/spark-derived-feature-view/run_and_verify.sh index 1c1f1b5..ab85baa 100755 --- a/spark-derived-feature-view/run_and_verify.sh +++ b/spark-derived-feature-view/run_and_verify.sh @@ -28,7 +28,7 @@ wait_for_port 7077 "Spark Cluster" cp -r ./data /tmp/feathub-data python3 main.py -docker-compose down +tear_down_docker cat data/output.json/_temporary/0/*/* > data/merged_output diff --git a/tools/utils.sh b/tools/utils.sh index 614d4a4..bd4d1c6 100644 --- a/tools/utils.sh +++ b/tools/utils.sh @@ -39,4 +39,12 @@ function sort_and_compare_files { sort "${FILE2}" | diff - "${FILE1}".sorted rm "${FILE1}".sorted -} \ No newline at end of file +} + +function tear_down_docker { + docker-compose down + docker container prune -f + docker volume prune -f + docker network prune -f + docker builder prune -a -f +}