Merge pull request #566 from Sage-Bionetworks-Workflows/bgrande/issue…

…-555/caching-bugfix

Fix `MapReads` caching bug

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
     environment:
       GENOME: GRCh37
       SNPEFF_CACHE_VERSION: "75"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
     steps:
       - checkout
       - setup_remote_docker
@@ -20,28 +20,28 @@ jobs:
     environment:
       GENOME: GRCh38
       SNPEFF_CACHE_VERSION: "86"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
   snpeffgrcm38:
     << : *buildsnpeff
     environment:
       GENOME: GRCm38
       SNPEFF_CACHE_VERSION: "86"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
   snpeffcanfam3_1:
     << : *buildsnpeff
     environment:
       GENOME: CanFam3.1
       SNPEFF_CACHE_VERSION: "86"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
   snpeffwbcel235:
     << : *buildsnpeff
     environment:
       GENOME: WBcel235
       SNPEFF_CACHE_VERSION: "86"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
   vepgrch37: &buildvep
     docker:
@@ -50,7 +50,7 @@ jobs:
       GENOME: GRCh37
       SPECIES: homo_sapiens
       VEP_VERSION: "99"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
     steps:
       - checkout
       - setup_remote_docker
@@ -67,31 +67,31 @@ jobs:
       GENOME: GRCh38
       SPECIES: homo_sapiens
       VEP_VERSION: "99"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
   vepgrcm38:
     << : *buildvep
     environment:
       GENOME: GRCm38
       SPECIES: mus_musculus
       VEP_VERSION: "99"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
   vepcanfam3_1:
     << : *buildvep
     environment:
       GENOME: CanFam3.1
       SPECIES: canis_familiaris
       VEP_VERSION: "99"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
   vepwbcel235:
     << : *buildvep
     environment:
       GENOME: WBcel235
       SPECIES: caenorhabditis_elegans
       VEP_VERSION: "99"
-      SAREK_TAG: 2.7.1
+      SAREK_TAG: 2.7.2
 
 workflows:
   version: 2

.github/workflows/ci.yml

@@ -36,13 +36,13 @@ jobs:
             environment.yml
       - name: Build new docker image
         if: env.MATCHED_FILES
-        run: docker build --no-cache . -t nfcore/sarek:2.7.1
+        run: docker build --no-cache . -t nfcore/sarek:2.7.2
 
       - name: Pull docker image
         if: ${{ !env.MATCHED_FILES }}
         run: |
           docker pull nfcore/sarek:dev
-          docker tag nfcore/sarek:dev nfcore/sarek:2.7.1
+          docker tag nfcore/sarek:dev nfcore/sarek:2.7.2
 
       - name: Install Nextflow
         env:
@@ -74,8 +74,8 @@ jobs:
           NXF_VER: '20.04.0'
       - name: Pull docker image
         run: |
-          docker pull nfcore/sarek:2.7.1
-          docker pull nfcore/sarek${{ matrix.tools }}:2.7.1.${{ matrix.species }}
+          docker pull nfcore/sarek:2.7.2
+          docker pull nfcore/sarek${{ matrix.tools }}:2.7.2.${{ matrix.species }}
       - name: Run annotation test
         run: nextflow run ${GITHUB_WORKSPACE} -profile test_annotation,docker --tools ${{ matrix.tools }}
 
@@ -96,7 +96,7 @@ jobs:
           # Only check Nextflow pipeline minimum version
           NXF_VER: '20.04.0'
       - name: Pull docker image
-        run: docker pull nfcore/sarek:2.7.1
+        run: docker pull nfcore/sarek:2.7.2
       - name: Get test data
         run: git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data
       - name: Run germline test
@@ -124,7 +124,7 @@ jobs:
           # Only check Nextflow pipeline minimum version
           NXF_VER: '20.04.0'
       - name: Pull docker image
-        run: docker pull nfcore/sarek:2.7.1
+        run: docker pull nfcore/sarek:2.7.2
       - name: Run test for minimal genomes
         run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker --skip_qc all --genome ${{ matrix.genome }} ${{ matrix.intervals }} --tools Manta,mpileup,Strelka,FreeBayes
 
@@ -145,7 +145,7 @@ jobs:
           # Only check Nextflow pipeline minimum version
           NXF_VER: '20.04.0'
       - name: Pull docker image
-        run: docker pull nfcore/sarek:2.7.1
+        run: docker pull nfcore/sarek:2.7.2
       - name: Run ${{ matrix.profile }} test
         run: nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker
 
@@ -166,7 +166,7 @@ jobs:
           # Only check Nextflow pipeline minimum version
           NXF_VER: '20.04.0'
       - name: Pull docker image
-        run: docker pull nfcore/sarek:2.7.1
+        run: docker pull nfcore/sarek:2.7.2
       - name: Run ${{ matrix.profile }} test
         run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner ${{ matrix.aligner }}
 
@@ -197,6 +197,6 @@ jobs:
           # Only check Nextflow pipeline minimum version
           NXF_VER: '20.04.0'
       - name: Pull docker image
-        run: docker pull nfcore/sarek:2.7.1
+        run: docker pull nfcore/sarek:2.7.2
       - name: Run ${{ matrix.tool }} test
         run: nextflow run ${GITHUB_WORKSPACE} -profile test_tool,docker --tools ${{ matrix.tool }} ${{ matrix.intervals }}

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.7.2](https://github.com/nf-core/sarek/releases/tag/2.7.2) - Áhkká
+
+Áhkká is one of the massifs just outside of the Sarek National Park.
+
+### Fixed
+
+- [#566](https://github.com/nf-core/sarek/pull/566) - Fix caching bug affecting a variable number of `MapReads` jobs due to non-deterministic state of `statusMap` during caching evaluation
+
+
 ## [2.7.1](https://github.com/nf-core/sarek/releases/tag/2.7.1) - Pårtejekna
 
 Pårtejekna is one of glaciers of the Pårte Massif.

Dockerfile

@@ -7,7 +7,7 @@ COPY environment.yml /
 RUN conda env create --quiet -f /environment.yml && conda clean -a
 
 # Add conda installation dir to PATH (instead of doing 'conda activate')
-ENV PATH /opt/conda/envs/nf-core-sarek-2.7.1/bin:$PATH
+ENV PATH /opt/conda/envs/nf-core-sarek-2.7.2/bin:$PATH
 
 # Dump the details of the installed packages to a file for posterity
-RUN conda env export --name nf-core-sarek-2.7.1 > nf-core-sarek-2.7.1.yml
+RUN conda env export --name nf-core-sarek-2.7.2 > nf-core-sarek-2.7.2.yml

conf/base.config

@@ -90,11 +90,11 @@ process {
     errorStrategy = {task.exitStatus == 143 ? 'retry' : 'ignore'}
   }
   withName:Snpeff {
-    container = {(params.annotation_cache && params.snpeff_cache) ? 'nfcore/sarek:2.7.1' : "nfcore/sareksnpeff:2.7.1.${params.genome}"}
+    container = {(params.annotation_cache && params.snpeff_cache) ? 'nfcore/sarek:2.7.2' : "nfcore/sareksnpeff:2.7.2.${params.genome}"}
     errorStrategy = {task.exitStatus == 143 ? 'retry' : 'ignore'}
   }
   withLabel:VEP {
-    container = {(params.annotation_cache && params.vep_cache) ? 'nfcore/sarek:2.7.1' : "nfcore/sarekvep:2.7.1.${params.genome}"}
+    container = {(params.annotation_cache && params.vep_cache) ? 'nfcore/sarek:2.7.2' : "nfcore/sarekvep:2.7.2.${params.genome}"}
     errorStrategy = {task.exitStatus == 143 ? 'retry' : 'ignore'}
   }
   withLabel:msisensor {

conf/test.config

@@ -33,11 +33,11 @@ params {
 
 process {
   withName:Snpeff {
-    container = 'nfcore/sareksnpeff:2.7.1.WBcel235'
+    container = 'nfcore/sareksnpeff:2.7.2.WBcel235'
     maxForks = 1
   }
   withLabel:VEP {
-    container = 'nfcore/sarekvep:2.7.1.WBcel235'
+    container = 'nfcore/sarekvep:2.7.2.WBcel235'
     maxForks = 1
   }
 }

containers/snpeff/Dockerfile

@@ -9,7 +9,7 @@ COPY environment.yml /
 RUN conda env create -f /environment.yml && conda clean -a
 
 # Add conda installation dir to PATH (instead of doing 'conda activate')
-ENV PATH /opt/conda/envs/nf-core-sarek-snpeff-2.7.1/bin:$PATH
+ENV PATH /opt/conda/envs/nf-core-sarek-snpeff-2.7.2/bin:$PATH
 
 # Setup default ARG variables
 ARG GENOME=GRCh38
@@ -19,4 +19,4 @@ ARG SNPEFF_CACHE_VERSION=86
 RUN snpEff download -v ${GENOME}.${SNPEFF_CACHE_VERSION}
 
 # Dump the details of the installed packages to a file for posterity
-RUN conda env export --name nf-core-sarek-snpeff-2.7.1 > nf-core-sarek-snpeff-2.7.1.yml
+RUN conda env export --name nf-core-sarek-snpeff-2.7.2 > nf-core-sarek-snpeff-2.7.2.yml

containers/snpeff/environment.yml

@@ -1,6 +1,6 @@
 # You can use this file to create a conda environment for this pipeline:
 # conda env create -f environment.yml
-name: nf-core-sarek-snpeff-2.7.1
+name: nf-core-sarek-snpeff-2.7.2
 channels:
   - conda-forge
   - bioconda

containers/vep/Dockerfile

@@ -9,7 +9,7 @@ COPY environment.yml /
 RUN conda env create -f /environment.yml && conda clean -a
 
 # Add conda installation dir to PATH (instead of doing 'conda activate')
-ENV PATH /opt/conda/envs/nf-core-sarek-vep-2.7.1/bin:$PATH
+ENV PATH /opt/conda/envs/nf-core-sarek-vep-2.7.2/bin:$PATH
 
 # Setup default ARG variables
 ARG GENOME=GRCh38
@@ -27,4 +27,4 @@ RUN vep_install \
   --NO_BIOPERL --NO_HTSLIB --NO_TEST --NO_UPDATE
 
 # Dump the details of the installed packages to a file for posterity
-RUN conda env export --name nf-core-sarek-vep-2.7.1 > nf-core-sarek-vep-2.7.1.yml
+RUN conda env export --name nf-core-sarek-vep-2.7.2 > nf-core-sarek-vep-2.7.2.yml

containers/vep/environment.yml

@@ -1,6 +1,6 @@
 # You can use this file to create a conda environment for this pipeline:
 # conda env create -f environment.yml
-name: nf-core-sarek-vep-2.7.1
+name: nf-core-sarek-vep-2.7.2
 channels:
   - conda-forge
   - bioconda

environment.yml

@@ -1,6 +1,6 @@
 # You can use this file to create a conda environment for this pipeline:
 #   conda env create -f environment.yml
-name: nf-core-sarek-2.7.1
+name: nf-core-sarek-2.7.2
 channels:
   - conda-forge
   - bioconda

main.nf

@@ -4168,6 +4168,9 @@ def extractInfos(channel) {
         statusMap[idPatient, idSample] = status
         [idPatient] + it[3..-1]
     }
+    // This forces a roundtrip as a list to ensure that genderMap and
+    // statusMap are fully populated before being used downstream
+    channel = Channel.fromList(channel.toList().val)
     [genderMap, statusMap, channel]
 }
 

nextflow.config

@@ -5,6 +5,9 @@
  * Default config options for all environments.
  */
 
+// Ensure DSL1 is used as DSL2 is the new default
+nextflow.enable.dsl = 1
+
 // Global default params, used in configs
 params {
   // Workflow flags:
@@ -115,7 +118,7 @@ params {
 // Container slug
 // Stable releases should specify release tag (ie: `2.5.2`)
 // Developmental code should specify dev
-process.container = 'nfcore/sarek:2.7.1'
+process.container = 'nfcore/sarek:2.7.2'
 
 // Load base.config by default for all pipelines
 includeConfig 'conf/base.config'
@@ -246,7 +249,7 @@ manifest {
   description = 'An open-source analysis pipeline to detect germline or somatic variants from whole genome or targeted sequencing'
   mainScript = 'main.nf'
   nextflowVersion = '>=20.04.0'
-  version = '2.7.1'
+  version = '2.7.2'
 }
 
 // Return the minimum between requirements and a maximum limit to ensure that resource requirements don't go over