Fix integration tests

CHANGELOG.md

@@ -164,6 +164,7 @@
 
 * `convert/from_10xh5_to_h5mu`: add .uns slot to mdata root when metrics file is provided (PR #887).
 
+* Use `params.resources_test` in test workflows in order to point to an alternative location (e.g. a cache).
 * Fix ingestion components not working when optional arguments are unset (PR #894).
 
 ## DOCUMENTATION

_viash.yaml

@@ -1,7 +1,5 @@
 viash_version: 0.9.0
 
-version: dev
-
 source: src
 target: target
 

src/compression/tar_extract/config.vsh.yaml

@@ -1,7 +1,7 @@
 name: tar_extract
 namespace: compression
 description: Extract files from a tar archive
-arguments:                     
+arguments:
   - name: "--input"
     alternatives: ["-i"]
     type: file
@@ -44,4 +44,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [ singlecpu, lowmem ]
+      label: [ singlecpu, lowmem ]

src/workflows/gdo/gdo_singlesample/integration_test.sh

@@ -1,22 +1,16 @@
 #!/bin/bash
 
-
-
 # get the root of the directory
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 # ensure that the command below is run from the root of the repository
 cd "$REPO_ROOT"
 
-export NXF_VER=21.10.6
-
-viash ns build -q gdo_singlesample
-
-nextflow run . \
+nextflow \
+  run . \
   -main-script src/workflows/gdo/gdo_singlesample/test.nf \
-  -profile docker,no_publish \
   -entry test_wf \
-  -with-trace work/trace.txt \
+  -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
   -c src/workflows/utils/integration_tests.config
 

src/workflows/gdo/gdo_singlesample/test.nf

@@ -2,9 +2,11 @@ nextflow.enable.dsl=2
 
 include { gdo_singlesample } from params.rootDir + "/target/nextflow/workflows/gdo/gdo_singlesample/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
+
+  resources_test = file(params.resources_test)
 
   output_ch = Channel.fromList([
       [
@@ -33,4 +35,4 @@ workflow test_wf {
       assert (output_list.collect({it[1].output.getFileName().toString()}) as Set).equals(["simple_execution_test.final.h5mu"] as Set)
 
     }
-}
+}

src/workflows/ingestion/bd_rhapsody/integration_test.sh

@@ -1,22 +1,16 @@
 #!/bin/bash
 
-
-
 # get the root of the directory
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 # ensure that the command below is run from the root of the repository
 cd "$REPO_ROOT"
 
-export NXF_VER=21.10.6
-
 nextflow \
   run . \
   -main-script src/workflows/ingestion/bd_rhapsody/test.nf \
   -entry test_wf \
-  -resume \
   -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
-  -c src/workflows/utils/integration_tests.config \
-  -with-trace work/trace.txt
+  -c src/workflows/utils/integration_tests.config 
 

src/workflows/ingestion/bd_rhapsody/test.nf

@@ -3,15 +3,17 @@ nextflow.enable.dsl=2
 include { bd_rhapsody } from params.rootDir + "/target/nextflow/workflows/ingestion/bd_rhapsody/main.nf"
 include { bd_rhapsody_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/bd_rhapsody_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
+
+  resources_test = file(params.resources_test)
 
   output_ch = Channel.fromList(
     [
       [
         id: "foo",
-        reads: file("$resources_test/bdrhap_5kjrt/raw/12*.fastq.gz"),
+        reads: file("${params.resources_test}/bdrhap_5kjrt/raw/12*.fastq.gz"),
         reference_archive: resources_test.resolve("reference_gencodev41_chr1/reference_bd_rhapsody.tar.gz"),
         abseq_reference: resources_test.resolve("bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"),
         cell_calling_data: "mRNA",
@@ -44,4 +46,4 @@ workflow test_wf {
 
     // | view { output -> output[1]}
     // | check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
-}
+}

src/workflows/ingestion/cellranger_mapping/integration_test.sh

@@ -1,22 +1,15 @@
 #!/bin/bash
 
-
-
 # get the root of the directory
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 # ensure that the command below is run from the root of the repository
 cd "$REPO_ROOT"
 
-viash ns build -q ingestion/cellranger_mapping --setup cb --platform nextflow
-
-
 nextflow \
   run . \
   -main-script src/workflows/ingestion/cellranger_mapping/test.nf \
   -entry test_wf \
-  -resume \
   -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
-  -c src/workflows/utils/integration_tests.config \
-  -with-trace work/trace.txt
+  -c src/workflows/utils/integration_tests.config

src/workflows/ingestion/cellranger_mapping/test.nf

@@ -3,9 +3,11 @@ nextflow.enable.dsl=2
 include { cellranger_mapping } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_mapping/main.nf"
 include { cellranger_mapping_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_mapping_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
+
+  resources_test = file(params.resources_test)
 
   output_ch = Channel.fromList([
       [  
@@ -33,4 +35,4 @@ workflow test_wf {
       assert output_list.size() == 1 : "output channel should contain one event"
       assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
     }
-}
+}

src/workflows/ingestion/cellranger_multi/integration_test.sh

@@ -1,32 +1,23 @@
 #!/bin/bash
 
-
-
 # get the root of the directory
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 # ensure that the command below is run from the root of the repository
 cd "$REPO_ROOT"
 
-export NXF_VER=22.10.3
-
 nextflow \
   run . \
   -main-script src/workflows/ingestion/cellranger_multi/test.nf \
   -entry test_wf \
-  -resume \
-  -profile no_publish,docker \
+  -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
-  -c src/workflows/utils/integration_tests.config \
-  -with-trace work/trace.txt
-
+  -c src/workflows/utils/integration_tests.config
 
 nextflow \
   run . \
   -main-script src/workflows/ingestion/cellranger_multi/test.nf \
   -entry test_wf2 \
-  -resume \
-  -profile no_publish,docker \
+  -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
-  -c src/workflows/utils/integration_tests.config \
-  -with-trace work/trace.txt
+  -c src/workflows/utils/integration_tests.config

src/workflows/ingestion/cellranger_multi/test.nf

@@ -3,23 +3,36 @@ nextflow.enable.dsl=2
 include { cellranger_multi } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_multi/main.nf"
 include { cellranger_multi_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_multi_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  resources_test = file("${params.rootDir}/resources_test")
+
+  resources_test = file(params.resources_test)
 
   output_ch = Channel.fromList([
       [
         id: "foo",
-        input:[resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R1_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R2_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R1_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R2_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R1_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R2_001.fastq.gz")],
+        input:[
+          resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R1_001.fastq.gz"),
+          resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R2_001.fastq.gz"),
+          resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R1_001.fastq.gz"),
+          resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R2_001.fastq.gz"),
+          resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R1_001.fastq.gz"),
+          resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R2_001.fastq.gz")
+        ],
         gex_reference: resources_test.resolve("reference_gencodev41_chr1/reference_cellranger.tar.gz"),
         vdj_reference: resources_test.resolve("10x_5k_anticmv/raw/refdata-cellranger-vdj-GRCh38-alts-ensembl-7.0.0.tar.gz"),
         feature_reference: resources_test.resolve("10x_5k_anticmv/raw/feature_reference.csv"),
-        library_id: ["5k_human_antiCMV_T_TBNK_connect_GEX_1_subset", "5k_human_antiCMV_T_TBNK_connect_AB_subset", "5k_human_antiCMV_T_TBNK_connect_VDJ_subset"],
-        library_type: ["Gene Expression", "Antibody Capture", "VDJ"]
+        library_id: [
+          "5k_human_antiCMV_T_TBNK_connect_GEX_1_subset",
+          "5k_human_antiCMV_T_TBNK_connect_AB_subset",
+          "5k_human_antiCMV_T_TBNK_connect_VDJ_subset"
+        ],
+        library_type: [
+          "Gene Expression",
+          "Antibody Capture",
+          "VDJ"
+        ]
       ]
     ])
     | map{ state -> [state.id, state] }
@@ -43,8 +56,9 @@ workflow test_wf {
 }
 
 workflow test_wf2 {
-  // Test cell multiplexing
-  resources_test = file("${params.rootDir}/resources_test")
+
+  resources_test = file(params.resources_test)
+
   output_ch = Channel.fromList([
       [
         id: "foo",
@@ -74,4 +88,4 @@ workflow test_wf2 {
       assert output_list.size() == 3 : "output channel should contain three events"
       assert (output_list.collect{it[0]} as Set) == (["Liver_BC1andOvarian_BC2", "Pancreas_BC4", "Colorectal_BC3"] as Set) : "Output ID should be same as input ID"
     }
-}
+}

src/workflows/ingestion/cellranger_postprocessing/integration_test.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-
-
 # get the root of the directory
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
@@ -14,16 +12,12 @@ nextflow \
   -entry test_wf \
   -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
-  -c src/workflows/utils/integration_tests.config \
-  -with-trace work/trace.txt \
-  -resume
+  -c src/workflows/utils/integration_tests.config
 
 nextflow \
   run . \
   -main-script src/workflows/ingestion/cellranger_postprocessing/test.nf \
   -entry test_wf2 \
   -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
-  -c src/workflows/utils/integration_tests.config \
-  -with-trace work/trace.txt \
-  -resume
+  -c src/workflows/utils/integration_tests.config

src/workflows/ingestion/cellranger_postprocessing/test.nf

@@ -4,9 +4,11 @@ include { cellranger_postprocessing } from params.rootDir + "/target/nextflow/wo
 include { from_10xh5_to_h5mu } from params.rootDir + "/target/nextflow/convert/from_10xh5_to_h5mu/main.nf"
 include { cellranger_postprocessing_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_postprocessing_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
+
+  resources_test = file(params.resources_test)
 
   output_ch = Channel.fromList([
       [
@@ -55,9 +57,8 @@ workflow test_wf {
 }
 
 workflow test_wf2 {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
 
+  resources_test = file(params.resources_test)
 
   output_ch = Channel.fromList([
       [
@@ -101,4 +102,4 @@ workflow test_wf2 {
     | map { output_list ->
       assert output_list.size() == 1 : "output channel should contain one event"
     }
-}
+}

src/workflows/ingestion/conversion/integration_test.sh

@@ -5,13 +5,10 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
 # ensure that the command below is run from the root of the repository
 cd "$REPO_ROOT"
 
-export NXF_VER=24.04.4
-
 nextflow \
   run . \
   -main-script src/workflows/ingestion/conversion/test.nf \
   -entry test_wf \
   -profile docker,no_publish \
   -c src/workflows/utils/labels_ci.config \
-  -c src/workflows/utils/integration_tests.config \
-  -with-trace work/trace.txt
+  -c src/workflows/utils/integration_tests.config

src/workflows/ingestion/conversion/test.nf

@@ -3,9 +3,11 @@ nextflow.enable.dsl=2
 include { conversion } from params.rootDir + "/target/nextflow/workflows/ingestion/conversion/main.nf"
 include { conversion_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/conversion_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
+
+  resources_test = file(params.resources_test)
 
   output_ch = Channel.fromList([
       [
@@ -52,4 +54,4 @@ workflow test_wf {
     | map { output_list ->
       assert output_list.size() == 4 : "output channel should contain four events"
     }
-}
+}