From d4667f0dd5cc570075dab6486f075b4fa3065571 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Wed, 1 Mar 2023 04:53:41 +0200 Subject: [PATCH 1/3] refactor labels to use the intuitive numbers --- modules/bgzip/bgzip.nf | 2 +- modules/bwa/mem.nf | 2 +- modules/fastqc/fastqc.nf | 2 +- modules/gatk/apply_bqsr.nf | 2 +- modules/gatk/base_recalibrator.nf | 2 +- modules/gatk/collect_wgs_metrics.nf | 2 +- modules/gatk/flag_stat.nf | 2 +- modules/gatk/haplotype_caller.nf | 2 +- .../gatk/haplotype_caller__minor_variants.nf | 2 +- modules/gatk/mark_duplicates.nf | 2 +- modules/lofreq/call.nf | 2 +- modules/lofreq/indelqual.nf | 2 +- modules/multiqc/multiqc.nf | 2 +- modules/samtools/index.nf | 2 +- modules/samtools/merge.nf | 1 - modules/samtools/stats.nf | 2 +- nextflow.config | 25 +++++++++++-------- 17 files changed, 29 insertions(+), 27 deletions(-) diff --git a/modules/bgzip/bgzip.nf b/modules/bgzip/bgzip.nf index dd348e71..1d67ba78 100644 --- a/modules/bgzip/bgzip.nf +++ b/modules/bgzip/bgzip.nf @@ -1,6 +1,6 @@ process BGZIP { tag "${name}" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/bwa/mem.nf b/modules/bwa/mem.nf index 661dbabb..ced71827 100644 --- a/modules/bwa/mem.nf +++ b/modules/bwa/mem.nf @@ -1,6 +1,6 @@ process BWA_MEM { tag "${sampleName}" - label 'cpu_high_memory_high' + label 'cpu_8_memory_8' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/fastqc/fastqc.nf b/modules/fastqc/fastqc.nf index 8a711b3f..595dcd63 100644 --- a/modules/fastqc/fastqc.nf +++ b/modules/fastqc/fastqc.nf @@ -1,6 +1,6 @@ process FASTQC { tag "${sampleName}" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/gatk/apply_bqsr.nf b/modules/gatk/apply_bqsr.nf index 9099886d..8e9f391e 100644 --- a/modules/gatk/apply_bqsr.nf +++ b/modules/gatk/apply_bqsr.nf @@ -1,6 +1,6 @@ process GATK_APPLY_BQSR { tag "$sampleName" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/gatk/base_recalibrator.nf b/modules/gatk/base_recalibrator.nf index 20a18a9a..f504e03c 100644 --- a/modules/gatk/base_recalibrator.nf +++ b/modules/gatk/base_recalibrator.nf @@ -1,6 +1,6 @@ process GATK_BASE_RECALIBRATOR { tag "$sampleName" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish diff --git a/modules/gatk/collect_wgs_metrics.nf b/modules/gatk/collect_wgs_metrics.nf index d4317570..de351b44 100644 --- a/modules/gatk/collect_wgs_metrics.nf +++ b/modules/gatk/collect_wgs_metrics.nf @@ -1,6 +1,6 @@ process GATK_COLLECT_WGS_METRICS { tag "${sampleName}" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/gatk/flag_stat.nf b/modules/gatk/flag_stat.nf index 9d7308cf..9dd7cb32 100644 --- a/modules/gatk/flag_stat.nf +++ b/modules/gatk/flag_stat.nf @@ -1,6 +1,6 @@ process GATK_FLAG_STAT { tag "${sampleName}" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/gatk/haplotype_caller.nf b/modules/gatk/haplotype_caller.nf index 6ec6e57b..108da21f 100644 --- a/modules/gatk/haplotype_caller.nf +++ b/modules/gatk/haplotype_caller.nf @@ -1,6 +1,6 @@ process GATK_HAPLOTYPE_CALLER { tag "$sampleName" - label 'cpu_medium_memory_medium' + label 'cpu_8_memory_8' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish diff --git a/modules/gatk/haplotype_caller__minor_variants.nf b/modules/gatk/haplotype_caller__minor_variants.nf index 38388e03..bb9c40c2 100644 --- a/modules/gatk/haplotype_caller__minor_variants.nf +++ b/modules/gatk/haplotype_caller__minor_variants.nf @@ -1,6 +1,6 @@ process GATK_HAPLOTYPE_CALLER__MINOR_VARIANTS { tag "$sampleName" - label 'cpu_medium_memory_medium' + label 'cpu_4_memory_8' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish diff --git a/modules/gatk/mark_duplicates.nf b/modules/gatk/mark_duplicates.nf index 3792d300..ddf89264 100644 --- a/modules/gatk/mark_duplicates.nf +++ b/modules/gatk/mark_duplicates.nf @@ -1,6 +1,6 @@ process GATK_MARK_DUPLICATES { tag "$sampleName" - label 'cpu_low_memory_high' + label 'cpu_4_memory_8' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/lofreq/call.nf b/modules/lofreq/call.nf index 57683ee0..42887026 100644 --- a/modules/lofreq/call.nf +++ b/modules/lofreq/call.nf @@ -1,6 +1,6 @@ process LOFREQ_CALL { tag "${sampleName}" - label 'cpu_high_memory_medium' + label 'cpu_8_memory_4' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/lofreq/indelqual.nf b/modules/lofreq/indelqual.nf index c5383331..3c08ec01 100644 --- a/modules/lofreq/indelqual.nf +++ b/modules/lofreq/indelqual.nf @@ -1,6 +1,6 @@ process LOFREQ_INDELQUAL { tag "${sampleName}" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/multiqc/multiqc.nf b/modules/multiqc/multiqc.nf index 5648feb6..08acc434 100644 --- a/modules/multiqc/multiqc.nf +++ b/modules/multiqc/multiqc.nf @@ -1,6 +1,6 @@ process MULTIQC { publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish - label 'cpu_low_memory_high' + label 'cpu_4_memory_16' input: path("*") diff --git a/modules/samtools/index.nf b/modules/samtools/index.nf index 92afa8dd..55d26051 100644 --- a/modules/samtools/index.nf +++ b/modules/samtools/index.nf @@ -1,6 +1,6 @@ process SAMTOOLS_INDEX { tag "${sampleName}" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/samtools/merge.nf b/modules/samtools/merge.nf index 514382ef..7937effc 100644 --- a/modules/samtools/merge.nf +++ b/modules/samtools/merge.nf @@ -1,6 +1,5 @@ process SAMTOOLS_MERGE { tag "${sampleName}" - label 'cpu_medium_memory_medium' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/modules/samtools/stats.nf b/modules/samtools/stats.nf index 5c8edc3f..024433fb 100644 --- a/modules/samtools/stats.nf +++ b/modules/samtools/stats.nf @@ -1,6 +1,6 @@ process SAMTOOLS_STATS { tag "${sampleName}" - label 'cpu_low_memory_low' + label 'cpu_2_memory_2' publishDir params.results_dir, mode: params.save_mode, enabled: params.should_publish input: diff --git a/nextflow.config b/nextflow.config index 0da68f49..66cdd250 100644 --- a/nextflow.config +++ b/nextflow.config @@ -11,34 +11,37 @@ params { includeConfig 'default_params.config' } process { + //Default values if a label hasn't been specified within a process cpus = 4 memory = 4.GB + errorStrategy = { task.attempt < 3 ? 'retry' : 'ignore' } - withLabel: 'cpu_low_memory_low' { + //NOTE: These labels are ordered by number of cpus allocated and then the memory + withLabel: 'cpu_2_memory_2' { cpus = 2 memory = 2.GB } - withLabel: 'cpu_low_memory_high' { - cpus = 2 - memory = 16.GB + withLabel: 'cpu_4_memory_8' { + cpus = 4 + memory = 8.GB } - withLabel: 'cpu_medium_memory_medium' { + withLabel: 'cpu_4_memory_16' { cpus = 4 + memory = 16.GB + } + + withLabel: 'cpu_8_memory_4' { + cpus = 8 memory = 4.GB } - withLabel: 'cpu_high_memory_high' { + withLabel: 'cpu_8_memory_8' { cpus = 8 memory = 8.GB } - withLabel: 'cpu_high_memory_medium' { - cpus = 8 - memory = 4.GB - } - withName: 'SAMPLESHEET_VALIDATION' { //NOTE: If this process fails, terminate the pipeline execution errorStrategy = 'terminate' From 51838027f852fe3eda37d498213842e8904787f0 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Wed, 1 Mar 2023 05:01:46 +0200 Subject: [PATCH 2/3] add notes and indicated the upcoming patch release --- nextflow.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 66cdd250..a83f443a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,7 +1,7 @@ manifest { name = 'MAGMA' description = 'MAGMA is a pipeline for comprehensive genomic analyses of Mycobacterium tuberculosis with a focus on clinical decision making as well as research.' - version = '1.0.0' + version = '1.0.1' author = 'TORCH-Consortium' defaultBranch = 'master' homePage = 'https://github.com/TORCH-Consortium/MAGMA' @@ -14,6 +14,8 @@ process { //Default values if a label hasn't been specified within a process cpus = 4 memory = 4.GB + + //Default action is to ignore the process if the second attempt fails errorStrategy = { task.attempt < 3 ? 'retry' : 'ignore' } //NOTE: These labels are ordered by number of cpus allocated and then the memory From e0f3a61551049d65b6c30e9ca2931d22746ca3c6 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Wed, 15 Mar 2023 16:59:13 +0200 Subject: [PATCH 3/3] Update docker.config --- conf/docker.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/docker.config b/conf/docker.config index 121dbc84..168138e0 100644 --- a/conf/docker.config +++ b/conf/docker.config @@ -15,3 +15,8 @@ process { } } + + +docker { + enabled = true +}