Fix integration tests

CHANGELOG.md

@@ -173,6 +173,8 @@
 
 * `convert/from_10xh5_to_h5mu`: add .uns slot to mdata root when metrics file is provided (PR #887).
 
+* Use `params.resources_test` in test workflows in order to point to an alternative location (e.g. a cache).
+
 ## DOCUMENTATION
 
 * Update authorship of components (PR #835).

_viash.yaml

@@ -1,7 +1,5 @@
 viash_version: 0.9.0
 
-version: dev
-
 source: src
 target: target
 

src/compression/tar_extract/config.vsh.yaml

@@ -1,7 +1,7 @@
 name: tar_extract
 namespace: compression
 description: Extract files from a tar archive
-arguments:                     
+arguments:
   - name: "--input"
     alternatives: ["-i"]
     type: file
@@ -44,4 +44,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [ singlecpu, lowmem ]
+      label: [ singlecpu, lowmem ]

src/workflows/gdo/gdo_singlesample/test.nf

@@ -2,14 +2,16 @@ nextflow.enable.dsl=2
 
 include { gdo_singlesample } from params.rootDir + "/target/nextflow/workflows/gdo/gdo_singlesample/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
   // allow changing the resources_test dir
   resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = Channel.fromList([
       [
         id: "simple_execution_test",
-        input: resources_test.resolve("10x_5k_lung_crispr/SC3_v3_NextGem_DI_CRISPR_A549_5K.h5mu"),
+        input: file(params.resources_test).resolve("10x_5k_lung_crispr/SC3_v3_NextGem_DI_CRISPR_A549_5K.h5mu"),
         min_counts: 3,
         max_counts: 10000000,
         min_guides_per_cell: 2,
@@ -33,4 +35,4 @@ workflow test_wf {
       assert (output_list.collect({it[1].output.getFileName().toString()}) as Set).equals(["simple_execution_test.final.h5mu"] as Set)
 
     }
-}
+}

src/workflows/ingestion/bd_rhapsody/test.nf

@@ -3,6 +3,8 @@ nextflow.enable.dsl=2
 include { bd_rhapsody } from params.rootDir + "/target/nextflow/workflows/ingestion/bd_rhapsody/main.nf"
 include { bd_rhapsody_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/bd_rhapsody_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
   // allow changing the resources_test dir
   resources_test = file("${params.rootDir}/resources_test")
@@ -11,9 +13,9 @@ workflow test_wf {
     [
       [
         id: "foo",
-        reads: file("$resources_test/bdrhap_5kjrt/raw/12*.fastq.gz"),
-        reference_archive: resources_test.resolve("reference_gencodev41_chr1/reference_bd_rhapsody.tar.gz"),
-        abseq_reference: resources_test.resolve("bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"),
+        reads: file("${params.resources_test}/bdrhap_5kjrt/raw/12*.fastq.gz"),
+        reference_archive: file(params.resources_test).resolve("reference_gencodev41_chr1/reference_bd_rhapsody.tar.gz"),
+        abseq_reference: file(params.resources_test).resolve("bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"),
         cell_calling_data: "mRNA",
         exact_cell_count: 4900
       ] 
@@ -44,4 +46,4 @@ workflow test_wf {
 
     // | view { output -> output[1]}
     // | check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
-}
+}

src/workflows/ingestion/cellranger_mapping/test.nf

@@ -3,15 +3,15 @@ nextflow.enable.dsl=2
 include { cellranger_mapping } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_mapping/main.nf"
 include { cellranger_mapping_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_mapping_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = Channel.fromList([
       [  
         id: "foo",
-        input: resources_test.resolve("cellranger_tiny_fastq/cellranger_tiny_fastq"),
-        reference: resources_test.resolve("cellranger_tiny_fastq/cellranger_tiny_ref"),
+        input: file(params.resources_test).resolve("cellranger_tiny_fastq/cellranger_tiny_fastq"),
+        reference: file(params.resources_test).resolve("cellranger_tiny_fastq/cellranger_tiny_ref"),
         output_type: "filtered",
       ]
     ])
@@ -33,4 +33,4 @@ workflow test_wf {
       assert output_list.size() == 1 : "output channel should contain one event"
       assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
     }
-}
+}

src/workflows/ingestion/cellranger_multi/test.nf

@@ -3,21 +3,22 @@ nextflow.enable.dsl=2
 include { cellranger_multi } from params.rootDir + "/target/nextflow/workflows/ingestion/cellranger_multi/main.nf"
 include { cellranger_multi_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_multi_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = Channel.fromList([
       [
         id: "foo",
-        input:[resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R1_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R2_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R1_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R2_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R1_001.fastq.gz"),
-               resources_test.resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R2_001.fastq.gz")],
-        gex_reference: resources_test.resolve("reference_gencodev41_chr1/reference_cellranger.tar.gz"),
-        vdj_reference: resources_test.resolve("10x_5k_anticmv/raw/refdata-cellranger-vdj-GRCh38-alts-ensembl-7.0.0.tar.gz"),
-        feature_reference: resources_test.resolve("10x_5k_anticmv/raw/feature_reference.csv"),
+        input:[file(params.resources_test).resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R1_001.fastq.gz"),
+               file(params.resources_test).resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_GEX_1_subset_S1_L001_R2_001.fastq.gz"),
+               file(params.resources_test).resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R1_001.fastq.gz"),
+               file(params.resources_test).resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_AB_subset_S2_L004_R2_001.fastq.gz"),
+               file(params.resources_test).resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R1_001.fastq.gz"),
+               file(params.resources_test).resolve("10x_5k_anticmv/raw/5k_human_antiCMV_T_TBNK_connect_VDJ_subset_S1_L001_R2_001.fastq.gz")],
+        gex_reference: file(params.resources_test).resolve("reference_gencodev41_chr1/reference_cellranger.tar.gz"),
+        vdj_reference: file(params.resources_test).resolve("10x_5k_anticmv/raw/refdata-cellranger-vdj-GRCh38-alts-ensembl-7.0.0.tar.gz"),
+        feature_reference: file(params.resources_test).resolve("10x_5k_anticmv/raw/feature_reference.csv"),
         library_id: ["5k_human_antiCMV_T_TBNK_connect_GEX_1_subset", "5k_human_antiCMV_T_TBNK_connect_AB_subset", "5k_human_antiCMV_T_TBNK_connect_VDJ_subset"],
         library_type: ["Gene Expression", "Antibody Capture", "VDJ"]
       ]
@@ -74,4 +75,4 @@ workflow test_wf2 {
       assert output_list.size() == 3 : "output channel should contain three events"
       assert (output_list.collect{it[0]} as Set) == (["Liver_BC1andOvarian_BC2", "Pancreas_BC4", "Colorectal_BC3"] as Set) : "Output ID should be same as input ID"
     }
-}
+}

src/workflows/ingestion/cellranger_postprocessing/test.nf

@@ -4,15 +4,15 @@ include { cellranger_postprocessing } from params.rootDir + "/target/nextflow/wo
 include { from_10xh5_to_h5mu } from params.rootDir + "/target/nextflow/convert/from_10xh5_to_h5mu/main.nf"
 include { cellranger_postprocessing_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/cellranger_postprocessing_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = Channel.fromList([
       [
         id: "foo",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"),
-        input_og: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"),
+        input_og: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"),
         perform_correction: true,
         min_genes: 100,
         min_counts: 1000,
@@ -101,4 +101,4 @@ workflow test_wf2 {
     | map { output_list ->
       assert output_list.size() == 1 : "output channel should contain one event"
     }
-}
+}

src/workflows/ingestion/conversion/test.nf

@@ -3,34 +3,34 @@ nextflow.enable.dsl=2
 include { conversion } from params.rootDir + "/target/nextflow/workflows/ingestion/conversion/main.nf"
 include { conversion_test } from params.rootDir + "/target/nextflow/test_workflows/ingestion/conversion_test/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = Channel.fromList([
       [
         id: "10xh5_test",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5"),
         input_type: "10xh5",
         modality: null
       ],
       [
         id: "10xmtx_test",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix"),
         input_type: "10xmtx",
         modality: null,
         output: "\$id.h5mu"
       ],
       [
         id: "10xmtx",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix"),
         input_type: "10xmtx",
         modality: "rna",
         output: "\$key.h5mu"
       ],
       [
         id: "h5ad",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix_rna.h5ad"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix_rna.h5ad"),
         input_type: "h5ad",
         modality: "rna",
         output: "\$key.h5mu"
@@ -52,4 +52,4 @@ workflow test_wf {
     | map { output_list ->
       assert output_list.size() == 4 : "output channel should contain four events"
     }
-}
+}

src/workflows/ingestion/demux/test.nf

@@ -3,27 +3,28 @@ nextflow.enable.dsl=2
 
 include { demux } from params.rootDir + "/target/nextflow/workflows/ingestion/demux/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  resources_test = file("${params.rootDir}/resources_test")
 
   // or when running from s3:
   Channel.fromList([
     [
       id: "mkfastq_test",
-      input: resources_test.resolve("cellranger_tiny_bcl/bcl"),
-      sample_sheet: resources_test.resolve("cellranger_tiny_bcl/bcl/sample_sheet.csv"),
+      input: file(params.resources_test).resolve("cellranger_tiny_bcl/bcl"),
+      sample_sheet: file(params.resources_test).resolve("cellranger_tiny_bcl/bcl/sample_sheet.csv"),
       demultiplexer: "mkfastq"
     ],
     [
       id: "bclconvert_test",
-      input: resources_test.resolve("cellranger_tiny_bcl/bcl2/"),
-      sample_sheet: resources_test.resolve("cellranger_tiny_bcl/bcl2/sample_sheet.csv"),
+      input: file(params.resources_test).resolve("cellranger_tiny_bcl/bcl2/"),
+      sample_sheet: file(params.resources_test).resolve("cellranger_tiny_bcl/bcl2/sample_sheet.csv"),
       demultiplexer: "bclconvert"
     ],
     [
       id: "bcl2fastq_test",
-      input: resources_test.resolve("cellranger_tiny_bcl/bcl"),
-      sample_sheet: resources_test.resolve("cellranger_tiny_bcl/bcl/sample_sheet.csv"),
+      input: file(params.resources_test).resolve("cellranger_tiny_bcl/bcl"),
+      sample_sheet: file(params.resources_test).resolve("cellranger_tiny_bcl/bcl/sample_sheet.csv"),
       demultiplexer: "bcl2fastq",
       ignore_missing: true
     ]
@@ -50,4 +51,4 @@ workflow test_wf {
     | map { output_list ->
       assert output_list.size() == 3 : "There should be three outputs"
     }
-}
+}

src/workflows/ingestion/make_reference/test.nf

@@ -2,16 +2,16 @@ nextflow.enable.dsl=2
 
 include { make_reference } from params.rootDir + "/target/nextflow/workflows/ingestion/make_reference/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = Channel.fromList([
       [
         id: "gencode_v41_ercc",
-        genome_fasta: resources_test.resolve("reference_gencodev41_chr1/reference.fa.gz"),
-        transcriptome_gtf: resources_test.resolve("reference_gencodev41_chr1/reference.gtf.gz"),
-        ercc: resources_test.resolve("reference_gencodev41_chr1/ERCC92.zip"),
+        genome_fasta: file(params.resources_test).resolve("reference_gencodev41_chr1/reference.fa.gz"),
+        transcriptome_gtf: file(params.resources_test).resolve("reference_gencodev41_chr1/reference.gtf.gz"),
+        ercc: file(params.resources_test).resolve("reference_gencodev41_chr1/ERCC92.zip"),
         subset_regex: "(ERCC-00002|chr1)",
         target: ["cellranger", "bd_rhapsody", "star"]
       ]        

src/workflows/integration/bbknn_leiden/test.nf

@@ -2,18 +2,19 @@ nextflow.enable.dsl=2
 
 include { bbknn_leiden } from params.rootDir + "/target/nextflow/workflows/integration/bbknn_leiden/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  resources_test = file("${params.rootDir}/resources_test")
   output_ch =
     Channel.fromList([
       [
         id: "simple_execution_test",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
         layer: "log_normalized"
       ],
       [
        id: "no_leiden_resolutions_test",
-       input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
+       input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
        layer: "log_normalized",
        leiden_resolution: []
       ]
@@ -73,4 +74,4 @@ workflow test_wf2 {
       assert output_list.size() == 1 : "output channel should contain 1 event."
       assert output_list.collect{it[0]} == ["test_output_arg"]
     }
-}
+}

src/workflows/integration/harmony_leiden/test.nf

@@ -2,14 +2,15 @@ nextflow.enable.dsl=2
 
 include { harmony_leiden } from params.rootDir + "/target/nextflow/workflows/integration/harmony_leiden/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = 
     Channel.fromList([
       [
         id: "simple_execution_test",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
         layer: "log_normalized",
         obs_covariates: "sample_id",
         embedding: "X_pca",
@@ -18,7 +19,7 @@ workflow test_wf {
       ],
       [
         id: "no_leiden_resolutions_test",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
         layer: "log_normalized",
         obs_covariates: "sample_id",
         embedding: "X_pca",

src/workflows/integration/scanorama_leiden/test.nf

@@ -2,20 +2,20 @@ nextflow.enable.dsl=2
 
 include { scanorama_leiden } from params.rootDir + "/target/nextflow/workflows/integration/scanorama_leiden/main.nf"
 
+params.resources_test = params.rootDir + "/resources_test"
+
 workflow test_wf {
-  // allow changing the resources_test dir
-  resources_test = file("${params.rootDir}/resources_test")
 
   output_ch = Channel.fromList([
       [
         id: "simple_execution_test",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
         layer: "log_normalized",
         leiden_resolution: [1.0, 0.25],
       ],
       [
         id: "no_leiden_resolutions_test",
-        input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
+        input: file(params.resources_test).resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"),
         layer: "log_normalized",
         leiden_resolution: [],
       ]
@@ -77,4 +77,4 @@ workflow test_wf2 {
       assert output_list.size() == 1 : "output channel should contain 1 event"
       assert output_list.collect{it[0]} == ["test_output_arg"]
     }
-}
+}