checkpoint

flink-extended · Jul 26, 2023 · 575d93c · 575d93c
1 parent 2413335
commit 575d93c
Show file tree

Hide file tree

Showing 12 changed files with 20 additions and 12 deletions.
diff --git a/flink-derived-feature-view/run_and_verify.sh b/flink-derived-feature-view/run_and_verify.sh
@@ -25,7 +25,7 @@ docker-compose up -d
 wait_for_port 8081 "Flink Cluster"
 
 python main.py
-docker-compose down
+tear_down_docker
 
 cat data/output.json/* > data/merged_output
 

diff --git a/flink-filesystem-join-redis/run_and_verify.sh b/flink-filesystem-join-redis/run_and_verify.sh
@@ -26,7 +26,7 @@ wait_for_port 8081 "Flink Cluster"
 
 python initialize_redis.py
 python main.py
-docker-compose down
+tear_down_docker
 
 cat data/output.json/* > data/merged_output
 

diff --git a/flink-kafka-join-filesystem/run_and_verify.sh b/flink-kafka-join-filesystem/run_and_verify.sh
@@ -50,7 +50,7 @@ while true; do
 done
 
 kill "${PID}"
-docker-compose down
+tear_down_docker
 
 sort_and_compare_files data/kafka-output data/expected_output.txt
 

diff --git a/flink-read-write-hdfs/run_and_verify.sh b/flink-read-write-hdfs/run_and_verify.sh
@@ -25,7 +25,7 @@ docker-compose up -d
 wait_for_port 8081 "Flink Cluster"
 
 python main.py
-docker-compose down
+tear_down_docker
 
 cat data/output.json/* > data/merged_output
 

diff --git a/flink-read-write-hive/run_and_verify.sh b/flink-read-write-hive/run_and_verify.sh
@@ -45,7 +45,7 @@ while true; do
 done
 
 kill "${PID}"
-docker-compose down
+tear_down_docker
 
 cat data/output/.part-* > data/merged_output
 

diff --git a/flink-read-write-mysql/run_and_verify.sh b/flink-read-write-mysql/run_and_verify.sh
@@ -27,6 +27,6 @@ wait_for_port 8081 "Flink Cluster"
 python initialize_mysql_table.py
 python main.py
 exit_code=$?
-docker-compose down
+tear_down_docker
 
 exit $exit_code
diff --git a/flink-read-write-redis/run_and_verify.sh b/flink-read-write-redis/run_and_verify.sh
@@ -26,6 +26,6 @@ wait_for_port 8081 "Flink Cluster"
 
 python main.py
 exit_code=$?
-docker-compose down
+tear_down_docker
 
 exit $exit_code
diff --git a/flink-sliding-feature-view-benchmark/run_and_verify.sh b/flink-sliding-feature-view-benchmark/run_and_verify.sh
@@ -25,4 +25,4 @@ wait_for_port 8081 "Flink Cluster"
 
 python main.py --records-num 1000
 
-docker-compose down
+tear_down_docker
diff --git a/flink-sliding-feature-view/run_and_verify.sh b/flink-sliding-feature-view/run_and_verify.sh
@@ -50,7 +50,7 @@ while true; do
 done
 
 kill "${PID}"
-docker-compose down
+tear_down_docker
 
 sort_and_compare_files data/kafka-output data/expected_output.txt
 

diff --git a/flink-sql-feature-view/run_and_verify.sh b/flink-sql-feature-view/run_and_verify.sh
@@ -25,7 +25,7 @@ docker-compose up -d
 wait_for_port 8081 "Flink Cluster"
 
 python main.py
-docker-compose down
+tear_down_docker
 
 cat data/output.json/* > data/merged_output
 

diff --git a/spark-derived-feature-view/run_and_verify.sh b/spark-derived-feature-view/run_and_verify.sh
@@ -28,7 +28,7 @@ wait_for_port 7077 "Spark Cluster"
 cp -r ./data /tmp/feathub-data
 python3 main.py
 
-docker-compose down
+tear_down_docker
 
 cat data/output.json/_temporary/0/*/* > data/merged_output
 

diff --git a/tools/utils.sh b/tools/utils.sh
@@ -39,4 +39,12 @@ function sort_and_compare_files {
 	sort "${FILE2}" | diff - "${FILE1}".sorted
 
 	rm "${FILE1}".sorted
-}
+}
+
+function tear_down_docker {
+  docker-compose down
+  docker container prune -f
+  docker volume prune -f
+  docker network prune -f
+  docker builder prune -a -f
+}