add proper support for panorama public

mriffle · Jan 13, 2025 · 6380639 · 6380639
1 parent bcf26cc
commit 6380639
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 20 deletions.
diff --git a/main.nf b/main.nf
@@ -86,7 +86,7 @@ workflow {
     }
 
     // if accessing panoramaweb and running on aws, set up an aws secret
-    if(workflow.profile == 'aws' && is_panorama_used) {
+    if(workflow.profile == 'aws' && is_panorama_authentication_required()) {
         GET_AWS_USER_ID()
         BUILD_AWS_SECRETS(GET_AWS_USER_ID.out)
         aws_secret_id = BUILD_AWS_SECRETS.out.aws_secret_id
@@ -424,23 +424,28 @@ workflow {
 
 }
 
-// return true if any entry in the list created from the param is a panoramaweb URL
-def any_entry_is_panorama(param) {
+// return true if the URL requires panorama authentication (panorama public does not)
+def panorama_auth_required_for_url(url) {
+    return url.startsWith(params.panorama.domain) && !url.contains("/_webdav/Panorama%20Public/")
+}
+
+// return true if any entry in the list required panorama authentication
+def any_entry_requires_panorama_auth(param) {
     values = param_to_list(param)
-    return values.any { it.startsWith(params.panorama.domain) }
+    return values.any { panorama_auth_required_for_url(it) }
 }
 
-// return true if panoramaweb will be accessed by this Nextflow run
-def is_panorama_used() {
+// return true if panoramaweb authentication will be required by this workflow run
+def is_panorama_authentication_required() {
 
     return params.panorama.upload ||
-           (params.fasta && params.fasta.startsWith(params.panorama.domain)) ||
-           (params.spectral_library && params.spectral_library.startsWith(params.panorama.domain)) ||
-           (params.replicate_metadata && params.replicate_metadata.startsWith(params.panorama.domain)) ||
-           (params.skyline.template_file && params.skyline.template_file.startsWith(params.panorama.domain)) ||
-           (params.quant_spectra_dir && any_entry_is_panorama(params.quant_spectra_dir)) ||
-           (params.chromatogram_library_spectra_dir && any_entry_is_panorama(params.chromatogram_library_spectra_dir)) ||
-           (params.skyline_skyr_file && any_entry_is_panorama(params.skyline_skyr_file))
+           (params.fasta && panorama_auth_required_for_url(params.fasta)) ||
+           (params.spectral_library && panorama_auth_required_for_url(params.spectral_library)) ||
+           (params.replicate_metadata && panorama_auth_required_for_url(params.replicate_metadata)) ||
+           (params.skyline.template_file && panorama_auth_required_for_url(params.skyline.template_file)) ||
+           (params.quant_spectra_dir && any_entry_requires_panorama_auth(params.quant_spectra_dir)) ||
+           (params.chromatogram_library_spectra_dir && any_entry_requires_panorama_auth(params.chromatogram_library_spectra_dir)) ||
+           (params.skyline_skyr_file && any_entry_requires_panorama_auth(params.skyline_skyr_file))
 
 }
 

diff --git a/modules/panorama.nf b/modules/panorama.nf
@@ -88,6 +88,40 @@ process PANORAMA_GET_MS_FILE_LIST {
     """
 }
 
+process PANORAMA_PUBLIC_GET_MS_FILE_LIST {
+    cache false
+    label 'process_low_constant'
+    label 'error_retry'
+    container params.images.panorama_client
+    publishDir params.output_directories.panorama, failOnError: true, mode: 'copy'
+
+    input:
+        each web_dav_url
+        val file_glob
+
+    output:
+        path('download_files.txt'), emit: ms_files
+        path("*.stdout"), emit: stdout
+        path("*.stderr"), emit: stderr
+
+    script:
+    // convert glob to regex that we can use to grep lines from a file of filenames
+    String regex = '^' + escapeRegex(file_glob).replaceAll("\\*", ".*") + '$'
+
+    """
+    echo "Running file list from Panorama Public..."
+        ${exec_java_command(task.memory)} \
+        -l \
+        -w "${web_dav_url}" \
+        -k "${params.panorama.public.key}" \
+        -o all_files.txt \
+        > >(tee "panorama-get-files.stdout") 2> >(tee "panorama-get-files.stderr" >&2) && \
+
+    # Filter raw files by file_glob and prepend web_dav_url to file names
+    grep -P '${regex}' all_files.txt | xargs -d'\\n' printf '${web_dav_url.replaceAll("%", "%%")}/%s\\n' > download_files.txt
+    """
+}
+
 process PANORAMA_GET_FILE {
     label 'process_low_constant'
     label 'error_retry'
@@ -164,6 +198,40 @@ process PANORAMA_GET_MS_FILE {
     """
 }
 
+process PANORAMA_PUBLIC_GET_MS_FILE {
+    label 'process_low_constant'
+    label 'error_retry'
+    maxForks 4
+    container params.images.panorama_client
+    storeDir "${params.panorama_cache_directory}"
+
+    input:
+        val web_dav_url
+
+    output:
+        path("${file(web_dav_url).name}"), emit: panorama_file
+        path("*.stdout"), emit: stdout
+        path("*.stderr"), emit: stderr
+
+    script:
+        raw_file_name = file(web_dav_url).name
+        """
+        echo "Downloading ${raw_file_name} from Panorama Public..."
+            ${exec_java_command(task.memory)} \
+            -d \
+            -w "${web_dav_url}" \
+            -k "${params.panorama.public.key}" \
+            > >(tee "panorama-get-${raw_file_name}.stdout") 2> >(tee "panorama-get-${raw_file_name}.stderr" >&2)
+        echo "Done!" # Needed for proper exit
+        """
+
+    stub:
+    """
+    touch "${file(web_dav_url).name}"
+    touch stub.stderr stub.stdout
+    """
+}
+
 process PANORAMA_GET_SKYR_FILE {
     label 'process_low_constant'
     label 'error_retry'

diff --git a/nextflow.config b/nextflow.config
@@ -66,8 +66,9 @@ params {
     // "msconvert" subdirectory of the results directory.
     msconvert_only = false
 
-    // Parameters related to uploading results to PanoramaWeb
+    // Parameters related to PanoramaWeb
     panorama.domain = 'https://panoramaweb.org'
+    panorama.public.key = '7d503a4147133c448c6eaf83bc9b8bc22ace4b7f6d36ca61c9d1ca836c510d10'
     panorama.upload = false             // Whether or not to upload to PanoramaWeb
     panorama.upload_url = null          // The webdav URL of a folder to hold all uploaded files
     panorama.import_skyline = false     // whether or not to import the Skyline into Panorama's internal database

diff --git a/workflows/get_input_files.nf b/workflows/get_input_files.nf
@@ -40,15 +40,15 @@ workflow get_input_files {
     main:
 
         // get files from Panorama as necessary
-        if(params.fasta.startsWith(params.panorama.domain)) {
+        if(panorama_auth_required_for_url(params.fasta)) {
             PANORAMA_GET_FASTA(params.fasta, aws_secret_id)
             fasta = PANORAMA_GET_FASTA.out.panorama_file
         } else {
             fasta = Channel.value(file(params.fasta, checkIfExists: true))
         }
 
         if(params.spectral_library) {
-            if(params.spectral_library.startsWith(params.panorama.domain)) {
+            if(panorama_auth_required_for_url(params.spectral_library)) {
                 PANORAMA_GET_SPECTRAL_LIBRARY(params.spectral_library, aws_secret_id)
                 spectral_library = PANORAMA_GET_SPECTRAL_LIBRARY.out.panorama_file
             } else {
@@ -59,7 +59,7 @@ workflow get_input_files {
         }
 
         if(params.skyline.template_file != null) {
-            if(params.skyline.template_file.startsWith(params.panorama.domain)) {
+            if(panorama_auth_required_for_url(params.skyline.template_file)) {
                 PANORAMA_GET_SKYLINE_TEMPLATE(params.skyline.template_file, aws_secret_id)
                 skyline_template_zipfile = PANORAMA_GET_SKYLINE_TEMPLATE.out.panorama_file
             } else {
@@ -73,7 +73,7 @@ workflow get_input_files {
 
             // Split skyr files stored on Panorama and locally into separate channels.
             Channel.fromList(param_to_list(params.skyline.skyr_file)).branch{
-                panorama_files: it.startsWith(params.panorama.domain)
+                panorama_files: panorama_auth_required_for_url(it)
                 local_files: true
                     return file(it, checkIfExists: true)
                 }.set{skyr_paths}
@@ -88,7 +88,7 @@ workflow get_input_files {
         }
 
         if(params.replicate_metadata != null) {
-            if(params.replicate_metadata.trim().startsWith(params.panorama.domain)) {
+            if(panorama_auth_required_for_url(params.replicate_metadata.trim())) {
                 PANORAMA_GET_METADATA(params.replicate_metadata, aws_secret_id)
                 replicate_metadata = PANORAMA_GET_METADATA.out.panorama_file
             } else {
@@ -101,3 +101,8 @@ workflow get_input_files {
             replicate_metadata = METADATA_PLACEHOLDER.out
         }
 }
+
+// return true if the URL requires panorama authentication (panorama public does not)
+def panorama_auth_required_for_url(url) {
+    return url.startsWith(params.panorama.domain) && !url.contains("/_webdav/Panorama%20Public/")
+}
diff --git a/workflows/get_mzmls.nf b/workflows/get_mzmls.nf
@@ -1,6 +1,8 @@
 // modules
 include { PANORAMA_GET_MS_FILE } from "../modules/panorama"
 include { PANORAMA_GET_MS_FILE_LIST } from "../modules/panorama"
+include { PANORAMA_PUBLIC_GET_MS_FILE } from "../modules/panorama"
+include { PANORAMA_PUBLIC_GET_MS_FILE_LIST } from "../modules/panorama"
 include { MSCONVERT } from "../modules/msconvert"
 
 // useful functions and variables
@@ -22,7 +24,8 @@ workflow get_mzmls {
         spectra_dirs = param_to_list(spectra_dir)
         spectra_dirs_ch = Channel.fromList(spectra_dirs)
             .branch{
-                panorama_dirs: it.startsWith(params.panorama.domain)
+                panorama_public_dirs: is_panorama_url(it) && !panorama_auth_required_for_url(it)
+                panorama_dirs: panorama_auth_required_for_url(it)
                 local_dirs: true
             }
 
@@ -42,8 +45,16 @@ workflow get_mzmls {
             .flatten()
             .set{panorama_url_ch}
 
+        // List files matching spectra_glob in panorama public directories
+        PANORAMA_PUBLIC_GET_MS_FILE_LIST(spectra_dirs_ch.panorama_public_dirs, spectra_glob)
+        PANORAMA_PUBLIC_GET_MS_FILE_LIST.out.ms_files
+            .map{it -> it.readLines().collect{ line -> line.strip() }}
+            .flatten()
+            .set{panorama_public_url_ch}
+
         // make sure that all files have the same extension
         all_paths_ch = panorama_url_ch.concat(
+            panorama_public_url_ch,
             local_file_ch.map{
                 it -> it.name
             }
@@ -70,7 +81,11 @@ workflow get_mzmls {
         // Download files from panorama if applicable
         PANORAMA_GET_MS_FILE(panorama_url_ch, aws_secret_id)
 
+        // Download files from panorama public if applicable
+        PANORAMA_PUBLIC_GET_MS_FILE(panorama_public_url_ch)
+
         PANORAMA_GET_MS_FILE.out.panorama_file
+            .concat(PANORAMA_PUBLIC_GET_MS_FILE.out.panorama_file)
             .concat(local_file_ch)
             .branch{
                 mzml: it.name.endsWith('.mzML')
@@ -84,3 +99,11 @@ workflow get_mzmls {
 
         mzml_ch = MSCONVERT.out.concat(ms_file_ch.mzml)
 }
+
+def is_panorama_url(url) {
+    return url.startsWith(params.panorama.domain)
+}
+
+def panorama_auth_required_for_url(url) {
+    return is_panorama_url(url) && !url.contains("/_webdav/Panorama%20Public/")
+}