tw launch scripts

viash-io · Jun 18, 2024 · ee70d55 · ee70d55
1 parent 2e4eec4
commit ee70d55
Show file tree

Hide file tree

Showing 3 changed files with 130 additions and 24 deletions.
diff --git a/...code_remote-input_s3-output_s3-work_s3.sh → ...t_s3-output_s3-work_s3_nxf23101_vsh085.sh b/...code_remote-input_s3-output_s3-work_s3.sh → ...t_s3-output_s3-work_s3_nxf23101_vsh085.sh
@@ -3,16 +3,19 @@
 set -e
 
 script_name=$(basename "$0")
-TMPOUT=s3://data-intuitive-tmp/test-nextflow-wave-fusion/output/${script_name%.sh}
+BUCKET=""  # REPLACE
+TMPOUT=$BUCKET/test-nextflow-wave-fusion/output/${script_name%.sh}
 OUT=output/${script_name%.sh}
-RES=s3://data-intuitive-tmp/test-nextflow-wave-fusion/resources
-WORK=s3://data-intuitive-tmp/test-nextflow-wave-fusion/work/${script_name%.sh}
+RES=$BUCKET/test-nextflow-wave-fusion
+WORK=$BUCKET/test-nextflow-wave-fusion/work/${script_name%.sh}
 NXF_CONFIG=/tmp/${script_name%.sh}.config
 PARAMS=/tmp/${script_name%.sh}.yaml
+WORKSPACE=""  # REPLACE
+COMPUTE_ENV=""  # REPLACE
+TOWER_API_ENDPOINT="" # REPLACE
 
-# set aws profile
-export AWS_PROFILE=di
-echo "aws profile: $AWS_PROFILE"
+export TOWER_ACCESS_TOKEN=""  # REPLACE
+export TOWER_API_ENDPOINT=""  # REPLACE
 
 # clear output dir
 echo "Clearing output directory"
@@ -21,19 +24,10 @@ aws s3 rm $TMPOUT --recursive
 
 # run component
 echo "Running component"
-
 cat > $NXF_CONFIG <<EOF
-process {
-    executor = 'awsbatch'
-    queue = 'nextflow_queue_nofusion'
-    container = 'bash:4.0'
-}
-
-aws {
-    region = 'eu-west-1'
-    batch {
-        cliPath = '/home/ec2-user/miniconda/bin/aws'
-    }
+tower {
+    enabled = true
+    endpoint = '$TOWER_API_ENDPOINT'   
 }
 EOF
 
@@ -44,19 +38,19 @@ publish_dir: "$TMPOUT"
 EOF
 
 tw launch \
-  viash-io/test-nextflow-wave-fusion \
-  --revision main_build \
+  test-nextflow-wave-fusion \
+  --revision build_0_8_5 \
   --main-script target/nextflow/method/main.nf \
   --work-dir $WORK \
-  --workspace XXX \
-  --compute-env YYY \
+  --workspace $WORKSPACE \
+  --compute-env $COMPUTE_ENV \
   --config $NXF_CONFIG \
   --params-file $PARAMS
 
 # sync output
 echo "Syncing output"
-aws s3 sync $TMPOUT $OUT
+echo "aws s3 sync $TMPOUT $OUT"
 
 # check if output is correct
 echo "Checking output"
-scripts/verify_output.sh "$OUT"
+echo "scripts/verify_output.sh $OUT"
diff --git a/scripts/run_tw-code_remote-input_s3-output_s3-work_s3_nxf23101_vsh086.sh b/scripts/run_tw-code_remote-input_s3-output_s3-work_s3_nxf23101_vsh086.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+script_name=$(basename "$0")
+BUCKET=""  # REPLACE
+TMPOUT=$BUCKET/test-nextflow-wave-fusion/output/${script_name%.sh}
+OUT=output/${script_name%.sh}
+RES=$BUCKET/test-nextflow-wave-fusion
+WORK=$BUCKET/test-nextflow-wave-fusion/work/${script_name%.sh}
+NXF_CONFIG=/tmp/${script_name%.sh}.config
+PARAMS=/tmp/${script_name%.sh}.yaml
+WORKSPACE=""  # REPLACE
+COMPUTE_ENV=""  # REPLACE
+TOWER_API_ENDPOINT="" # REPLACE
+
+export TOWER_ACCESS_TOKEN=""  # REPLACE
+export TOWER_API_ENDPOINT=""  # REPLACE
+
+# clear output dir
+echo "Clearing output directory"
+aws s3 rm $TMPOUT --recursive
+[ -d "$OUT" ] && rm -r "$OUT"
+
+# run component
+echo "Running component"
+cat > $NXF_CONFIG <<EOF
+tower {
+    enabled = true
+    endpoint = '$TOWER_API_ENDPOINT'   
+}
+EOF
+
+cat > $PARAMS <<EOF
+input: $RES/input1.txt
+multiple_input: "$RES/input1.txt;$RES/input2.txt"
+publish_dir: "$TMPOUT"
+EOF
+
+tw launch \
+  test-nextflow-wave-fusion \
+  --revision build_0_8_6 \
+  --main-script target/nextflow/method/main.nf \
+  --work-dir $WORK \
+  --workspace $WORKSPACE \
+  --compute-env $COMPUTE_ENV \
+  --config $NXF_CONFIG \
+  --params-file $PARAMS
+
+# sync output
+echo "Syncing output"
+echo "aws s3 sync $TMPOUT $OUT"
+
+# check if output is correct
+echo "Checking output"
+echo "scripts/verify_output.sh $OUT"
diff --git a/scripts/run_tw-code_remote-input_s3-output_s3-work_s3_nxf23101_vsh090rc6.sh b/scripts/run_tw-code_remote-input_s3-output_s3-work_s3_nxf23101_vsh090rc6.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+script_name=$(basename "$0")
+BUCKET=""  # REPLACE
+TMPOUT=$BUCKET/test-nextflow-wave-fusion/output/${script_name%.sh}
+OUT=output/${script_name%.sh}
+RES=$BUCKET/test-nextflow-wave-fusion
+WORK=$BUCKET/test-nextflow-wave-fusion/work/${script_name%.sh}
+NXF_CONFIG=/tmp/${script_name%.sh}.config
+PARAMS=/tmp/${script_name%.sh}.yaml
+WORKSPACE=""  # REPLACE
+COMPUTE_ENV=""  # REPLACE
+TOWER_API_ENDPOINT="" # REPLACE
+
+export TOWER_ACCESS_TOKEN=""  # REPLACE
+export TOWER_API_ENDPOINT=""  # REPLACE
+
+# clear output dir
+echo "Clearing output directory"
+aws s3 rm $TMPOUT --recursive
+[ -d "$OUT" ] && rm -r "$OUT"
+
+# run component
+echo "Running component"
+cat > $NXF_CONFIG <<EOF
+tower {
+    enabled = true
+    endpoint = '$TOWER_API_ENDPOINT'   
+}
+EOF
+
+cat > $PARAMS <<EOF
+input: $RES/input1.txt
+multiple_input: "$RES/input1.txt;$RES/input2.txt"
+publish_dir: "$TMPOUT"
+EOF
+
+tw launch \
+  test-nextflow-wave-fusion \
+  --revision build_0_9_0_RC6 \
+  --main-script target/nextflow/method/main.nf \
+  --work-dir $WORK \
+  --workspace $WORKSPACE \
+  --compute-env $COMPUTE_ENV \
+  --config $NXF_CONFIG \
+  --params-file $PARAMS
+
+# sync output
+echo "Syncing output"
+echo "aws s3 sync $TMPOUT $OUT"
+
+# check if output is correct
+echo "Checking output"
+echo "scripts/verify_output.sh $OUT"