Ci test debug

.github/workflows/ci.yml

@@ -33,7 +33,6 @@ jobs:
       matrix:
         NXF_VER:
           - "24.04.2"
-          - "latest-everything"
         profile:
           - "docker"
           - "singularity"
@@ -102,11 +101,8 @@ jobs:
       - name: Run Tests (Shard ${{ matrix.shard }}/6)
         run: |
           nf-test test \
-              --debug \
-              --verbose \
               --ci \
               --shard ${{ matrix.shard }}/6 \
-              --changed-since HEAD^ \
               --profile "+${{ matrix.profile }}" \
               --filter pipeline \
               --junitxml=test.xml

CHANGELOG.md

@@ -3,7 +3,17 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## dev
+## 1.5.2dev - (UNRELEASED)
+
+### `Added`
+
+### `Changed`
+
+- [#276](https://github.com/nf-core/demultiplex/pull/276) Template update for nf-core/tools v3.0.2
+
+### `Fixed`
+
+- [#277](https://github.com/nf-core/demultiplex/pull/277) Improved samplesheet generation to always produce all types of samplesheets, added the ability to explicitly set strandedness, and fixed output paths to correctly reflect the `publishDir` subdirectory structure.
 
 ## 1.5.2 - 2024-10-07
 

conf/test_bclconvert.config

@@ -13,8 +13,8 @@
 // Limit resources so that this can run on GitHub Actions
 process {
     resourceLimits = [
-        cpus: 4,
-        memory: '13.GB',
+        cpus: 2,
+        memory: '6.GB',
         time: '4.h'
     ]
 }

conf/test_fqtk.config

@@ -13,8 +13,8 @@
 // Limit resources so that this can run on GitHub Actions
 process {
     resourceLimits = [
-        cpus: 4,
-        memory: '13.GB',
+        cpus: 2,
+        memory: '6.GB',
         time: '4.h'
     ]
 }

modules/local/fastq_to_samplesheet/main.nf

@@ -14,15 +14,18 @@ process FASTQ_TO_SAMPLESHEET {
 
     exec:
 
+    // Calculate the dynamic output directory based on meta.lane
+    def outputDir = meta.lane ? "${params.outdir}/${meta.id}/L00${meta.lane}" : "${params.outdir}/${meta.id}"
+
     // Add relevant fields to the map
     def pipeline_map = [
         sample  : meta.samplename,
-        fastq_1 : meta.fastq_1
+        fastq_1 : outputDir + '/' + file(meta.fastq_1).fileName
     ]
 
     // Add fastq_2 if it's a paired-end sample
     if (!meta.single_end) {
-        pipeline_map.fastq_2 = meta.fastq_2
+        pipeline_map.fastq_2 = outputDir + '/' + file(meta.fastq_2).fileName
     }
 
     // Add pipeline-specific entries

nextflow.config

@@ -27,7 +27,7 @@ params {
     kraken_db                   = null         // file .tar.gz
 
     // Downstream Nextflow pipeline
-    downstream_pipeline         = "default"     // enum string [rnaseq, atacseq, taxprofiler, default]
+    strandedness                = "auto"        // enum string [auto, reverse, forward, unstranded]
 
     // Options: CheckQC
     checkqc_config              = []            // file .yaml

nextflow_schema.json

@@ -35,12 +35,6 @@
                     "type": "string",
                     "format": "file-path",
                     "description": "Path to Illumina v2 samplesheet validator .json file"
-                },
-                "downstream_pipeline": {
-                    "type": "string",
-                    "description": "Name of downstream nf-core pipeline (one of: rnaseq, atacseq, taxprofiler or default). Used to produce the input samplesheet for that pipeline.",
-                    "default": "default",
-                    "enum": ["rnaseq", "atacseq", "taxprofiler", "default"]
                 }
             }
         },
@@ -276,5 +270,11 @@
         {
             "$ref": "#/$defs/generic_options"
         }
-    ]
+    ],
+    "properties": {
+        "strandedness": {
+            "type": "string",
+            "default": "auto"
+        }
+    }
 }

tests/.nftignore

@@ -22,7 +22,7 @@ multiqc/multiqc_data/multiqc_sources.txt
 multiqc/multiqc_plots/{svg,pdf,png}/*.{svg,pdf,png}
 multiqc/multiqc_report.html
 pipeline_info/*.{html,json,txt,yml}
-samplesheet/samplesheet.csv
+samplesheet/*.csv
 sim-data/*-Bases2Fastq-Sim_QC.html
 sim-data/*.fastp.fastq.gz
 sim-data/*.fastp.fastq.gz.md5

tests/bases2fastq.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "Bases2Fastq": {
         "content": [
-            8,
+            10,
             {
                 "BASES2FASTQ": {
                     "bases2fastq": "1.8.0.1260801529, use subject to license available at elementbiosciences.com"
@@ -23,7 +23,7 @@
                 }
             },
             [
-        "fastq",
+                "fastq",
                 "fastq/DefaultSample.samplesheet.csv",
                 "multiqc",
                 "multiqc/multiqc_data",
@@ -148,7 +148,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv",
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv",
                 "sim-data",
                 "sim-data.csv",
                 "sim-data/20230404-Bases2Fastq-Sim_QC.html",
@@ -210,8 +212,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:37:55.855747"
+        "timestamp": "2024-10-17T23:29:10.785062467"
     }
-}
+}

tests/bcl2fastq.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "Bcl2Fastq": {
         "content": [
-            6,
+            8,
             {
                 "BCL2FASTQ": {
                     "bcl2fastq": "2.20.0.422"
@@ -191,7 +191,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "220422_M11111_0222_000000000-K9H97.lane1_no_adapters.csv:md5,ee5db2e12754e069998b0a96e535238c",
@@ -237,8 +239,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:39:25.304794"
+        "timestamp": "2024-10-17T23:30:33.296858681"
     }
-}
+}

tests/bclconvert.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "BCL-CONVERT": {
         "content": [
-            6,
+            8,
             {
                 "BCLCONVERT": {
                     "bclconvert": "4.3.6"
@@ -177,7 +177,9 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "220422_M11111_0222_000000000-K9H97.lane1_no_adapters.csv:md5,ee5db2e12754e069998b0a96e535238c",
@@ -233,8 +235,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-15T16:41:01.01928"
+        "timestamp": "2024-10-17T23:31:58.719385372"
     }
-}
+}

tests/bclconvert_mini.nf.test.snap

@@ -1,7 +1,7 @@
 {
     "BCL-CONVERT-mini": {
         "content": [
-            26,
+            38,
             {
                 "BCLCONVERT": {
                     "bclconvert": "4.3.6"
@@ -248,24 +248,26 @@
                 "pipeline_info",
                 "pipeline_info/nf_core_pipeline_software_mqc_versions.yml",
                 "samplesheet",
-                "samplesheet/samplesheet.csv"
+                "samplesheet/atacseq_samplesheet.csv",
+                "samplesheet/rnaseq_samplesheet.csv",
+                "samplesheet/taxprofiler_samplesheet.csv"
             ],
             [
                 "HBRR1_S1_L001.fastp.fastq.gz:md5,451657490d4816dd5ff904e34ecc7fdf",
                 "HBRR1_S1_L001.fastp.fastq.gz.md5:md5,573455dfd3f2e7fa6b1eee838659c03a",
-                "HBRR1_S1_L001.fastp.json:md5,d34d8a4237b3ff4585f5b3fc84172d02",
+                "HBRR1_S1_L001.fastp.json:md5,abc74d1b3abc74dc86893508598083bf",
                 "HBRR1_S1_L001_R1_001.fastq.gz:md5,451657490d4816dd5ff904e34ecc7fdf",
                 "HBRR1_S1_L001_fastqc_data.txt:md5,e3ef09af2b44d86b7b32494f1ce5fc4c",
                 "HBRR1_S1_L001_summary.txt:md5,53a5e6bac1e853f23d27d6966e52df81",
                 "HBRR2_S2_L001.fastp.fastq.gz:md5,3b9e4fc70ebc608b48d1f0027265e9b8",
                 "HBRR2_S2_L001.fastp.fastq.gz.md5:md5,50fbb4ab07c7641277f2f7352babe4bd",
-                "HBRR2_S2_L001.fastp.json:md5,f0ff03f6bf6c1a3878ae2eebba0f070d",
+                "HBRR2_S2_L001.fastp.json:md5,c876081052efe114c7139501e333289c",
                 "HBRR2_S2_L001_R1_001.fastq.gz:md5,3b9e4fc70ebc608b48d1f0027265e9b8",
                 "HBRR2_S2_L001_fastqc_data.txt:md5,1c5127f5835c98503af2c7fa98da2982",
                 "HBRR2_S2_L001_summary.txt:md5,f3e0bc27d30d0f6e10ad76be47017881",
                 "HBRR3_S3_L001.fastp.fastq.gz:md5,a110eefaee1da4d99398500e55919f0c",
                 "HBRR3_S3_L001.fastp.fastq.gz.md5:md5,e744c0e523aa5cd3d5422ded60f28109",
-                "HBRR3_S3_L001.fastp.json:md5,c2abb3a7d3e46392cad6b0230d744ac2",
+                "HBRR3_S3_L001.fastp.json:md5,bffa2e2719426164cc411b5cb4a9eed9",
                 "HBRR3_S3_L001_R1_001.fastq.gz:md5,a110eefaee1da4d99398500e55919f0c",
                 "HBRR3_S3_L001_fastqc_data.txt:md5,33a618ec1d0f8d16987adf55115a89ae",
                 "HBRR3_S3_L001_summary.txt:md5,fbe9c4281a768a826429e328f5eb2a34",
@@ -298,19 +300,19 @@
                 "fastq_list.csv:md5,07cbe4173e18cbdef16ee8feb4adaf7d",
                 "UHRR1_S4_L001.fastp.fastq.gz:md5,03f215bcb4005804f60acccfe4aaef78",
                 "UHRR1_S4_L001.fastp.fastq.gz.md5:md5,baa87b03873661fc6c7f318e4b19de55",
-                "UHRR1_S4_L001.fastp.json:md5,6b50a023754acafdd32c2bb103650a83",
+                "UHRR1_S4_L001.fastp.json:md5,aaa12ce4b53c4702045a973c73b2baea",
                 "UHRR1_S4_L001_R1_001.fastq.gz:md5,b470c82d2002c563c42786060ae364b9",
                 "UHRR1_S4_L001_fastqc_data.txt:md5,be145fdf6b50140c98d7590d85e891f2",
                 "UHRR1_S4_L001_summary.txt:md5,c6c342cbcad3fc68343284c24e4a8b4a",
                 "UHRR2_S5_L001.fastp.fastq.gz:md5,2186d5f37865f970d223953e9a5412ae",
                 "UHRR2_S5_L001.fastp.fastq.gz.md5:md5,3996768dfb0c678b758399ca0aecb4dd",
-                "UHRR2_S5_L001.fastp.json:md5,db5ef9bec3dd04d37ada557785f43fb7",
+                "UHRR2_S5_L001.fastp.json:md5,336a236f2953704b82ac23ae1b3d538d",
                 "UHRR2_S5_L001_R1_001.fastq.gz:md5,47c36adf557bf4d37b0a84d055125290",
                 "UHRR2_S5_L001_fastqc_data.txt:md5,da424cfd66107dc895705fbdd5e7c94b",
                 "UHRR2_S5_L001_summary.txt:md5,121a0aeaf860fae497a27e8eceff3283",
                 "UHRR3_S6_L001.fastp.fastq.gz:md5,674c4989c2270d88612fcf221e3e0950",
                 "UHRR3_S6_L001.fastp.fastq.gz.md5:md5,22f315bfc3ebbe1fbf9eeca758fdd137",
-                "UHRR3_S6_L001.fastp.json:md5,1e08e9fa69ad134ef69dc11cbbfd03ef",
+                "UHRR3_S6_L001.fastp.json:md5,d5d4fd8b4845ff8f8f94ec02147d92cd",
                 "UHRR3_S6_L001_R1_001.fastq.gz:md5,bdb407e3ded3691479ec25f5f1df3b07",
                 "UHRR3_S6_L001_fastqc_data.txt:md5,8cf45b23b2176f83309b2bd6162adb0d",
                 "UHRR3_S6_L001_summary.txt:md5,d7d5a014b51d64e4da1a113a4ad79eda",
@@ -346,8 +348,8 @@
         ],
         "meta": {
             "nf-test": "0.9.0",
-            "nextflow": "24.09.2"
+            "nextflow": "24.04.4"
         },
-        "timestamp": "2024-10-16T10:59:08.175426"
+        "timestamp": "2024-10-18T14:18:52.915906491"
     }
-}
+}