Merge pull request mriffle#15 from ajmaurais/panorama_download

Get rid of panorama raw file placeholder files.
ajmaurais · Sep 9, 2024 · 29f4f53 · 29f4f53
2 parents 2fd4cbb + 323e328
commit 29f4f53
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 25 deletions.
diff --git a/modules/panorama.nf b/modules/panorama.nf
@@ -64,7 +64,7 @@ process PANORAMA_GET_RAW_FILE_LIST {
         val aws_secret_id
 
     output:
-        tuple val(web_dav_url), path("*.download"), emit: raw_file_placeholders
+        path('download_files.txt'), emit: raw_files
         path("*.stdout"), emit: stdout
         path("*.stderr"), emit: stderr
 
@@ -81,11 +81,11 @@ process PANORAMA_GET_RAW_FILE_LIST {
         -e raw \
         -w "${web_dav_url}" \
         -k \$PANORAMA_API_KEY \
-        -o panorama_files.txt \
+        -o all_files.txt \
         > >(tee "panorama-get-files.stdout") 2> >(tee "panorama-get-files.stderr" >&2) && \
-        grep -P '${regex}' panorama_files.txt | xargs -I % sh -c 'touch %.download'
 
-    echo "Done!" # Needed for proper exit
+    # Filter raw files by file_glob and prepend web_dav_url to file names
+    grep -P '${regex}' all_files.txt | xargs printf '${web_dav_url.replaceAll("%", "%%")}/%s\n' > download_files.txt
     """
 }
 
@@ -98,31 +98,31 @@ process PANORAMA_GET_FILE {
     secret 'PANORAMA_API_KEY'
 
     input:
-        val web_dav_dir_url
+        val web_dav_url
         val aws_secret_id
 
     output:
-        path("${file(web_dav_dir_url).name}"), emit: panorama_file
+        path("${file(web_dav_url).name}"), emit: panorama_file
         path("*.stdout"), emit: stdout
         path("*.stderr"), emit: stderr
 
     script:
-        file_name = file(web_dav_dir_url).name
+        file_name = file(web_dav_url).name
         """
         ${setupPanoramaAPIKeySecret(aws_secret_id, task.executor)}
 
         echo "Downloading ${file_name} from Panorama..."
             ${exec_java_command(task.memory)} \
             -d \
-            -w "${web_dav_dir_url}" \
+            -w "${web_dav_url}" \
             -k \$PANORAMA_API_KEY \
             > >(tee "panorama-get-${file_name}.stdout") 2> >(tee "panorama-get-${file_name}.stderr" >&2)
         echo "Done!" # Needed for proper exit
         """
 
     stub:
     """
-    touch "${file(web_dav_dir_url).name}"
+    touch "${file(web_dav_url).name}"
     touch stub.stderr stub.stdout
     """
 }
@@ -136,32 +136,31 @@ process PANORAMA_GET_RAW_FILE {
     secret 'PANORAMA_API_KEY'
 
     input:
-        tuple val(web_dav_dir_url), path(download_file_placeholder)
+        val web_dav_url
         val aws_secret_id
 
     output:
-        path("${download_file_placeholder.baseName}"), emit: panorama_file
+        path("${file(web_dav_url).name}"), emit: panorama_file
         path("*.stdout"), emit: stdout
         path("*.stderr"), emit: stderr
 
     script:
-        raw_file_name = download_file_placeholder.baseName
-
+        raw_file_name = file(web_dav_url).name
         """
         ${setupPanoramaAPIKeySecret(aws_secret_id, task.executor)}
 
         echo "Downloading ${raw_file_name} from Panorama..."
             ${exec_java_command(task.memory)} \
             -d \
-            -w "${web_dav_dir_url}${raw_file_name}" \
+            -w "${web_dav_url}" \
             -k \$PANORAMA_API_KEY \
             > >(tee "panorama-get-${raw_file_name}.stdout") 2> >(tee "panorama-get-${raw_file_name}.stderr" >&2)
         echo "Done!" # Needed for proper exit
         """
 
     stub:
     """
-    touch "${download_file_placeholder.baseName}"
+    touch "${file(web_dav_url).name}"
     touch stub.stderr stub.stdout
     """
 }
@@ -175,23 +174,23 @@ process PANORAMA_GET_SKYR_FILE {
     secret 'PANORAMA_API_KEY'
 
     input:
-        val web_dav_dir_url
+        val web_dav_url
         val aws_secret_id
 
     output:
-        path("${file(web_dav_dir_url).name}"), emit: panorama_file
+        path("${file(web_dav_url).name}"), emit: panorama_file
         path("*.stdout"), emit: stdout
         path("*.stderr"), emit: stderr
 
     script:
-        file_name = file(web_dav_dir_url).name
+        file_name = file(web_dav_url).name
         """
         ${setupPanoramaAPIKeySecret(aws_secret_id, task.executor)}
 
         echo "Downloading ${file_name} from Panorama..."
             ${exec_java_command(task.memory)} \
             -d \
-            -w "${web_dav_dir_url}" \
+            -w "${web_dav_url}" \
             -k \$PANORAMA_API_KEY \
             > >(tee "panorama-get-${file_name}.stdout") 2> >(tee "panorama-get-${file_name}.stderr" >&2)
         echo "Done!" # Needed for proper exit
@@ -232,9 +231,9 @@ process UPLOAD_FILE {
         """
 
     stub:
-    '''
-    touch stub.stderr stub.stdout
-    '''
+    """
+    touch "panorama-upload-${file(file_to_upload).name}.stdout" "panorama-upload-${file(file_to_upload).name}.stderr"
+    """
 }
 
 process IMPORT_SKYLINE {

diff --git a/workflows/get_mzmls.nf b/workflows/get_mzmls.nf
@@ -22,9 +22,12 @@ workflow get_mzmls {
                                     .filter{ it.length() > 0 } // skip empty lines
 
             // get raw files from panorama
-            panorama_raw_list_ch = PANORAMA_GET_RAW_FILE_LIST(spectra_dirs_ch, spectra_glob, aws_secret_id)
-            placeholder_ch = panorama_raw_list_ch.raw_file_placeholders.transpose()
-            PANORAMA_GET_RAW_FILE(placeholder_ch, aws_secret_id)
+            PANORAMA_GET_RAW_FILE_LIST(spectra_dirs_ch, spectra_glob, aws_secret_id)
+            raw_url_ch = PANORAMA_GET_RAW_FILE_LIST.out.raw_files
+                .splitText()
+                .map{ it -> it.strip() }
+
+            PANORAMA_GET_RAW_FILE(raw_url_ch, aws_secret_id)
 
             mzml_ch = MSCONVERT(
                 PANORAMA_GET_RAW_FILE.out.panorama_file,