From cd6a81e9ce38e66c0fe368ab83e2eb4db8f6c011 Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:44:28 -0600 Subject: [PATCH 01/10] ci(renovate): Allow channels in environment.ymls (#5400) --- .github/renovate.json5 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 7ffc321969b..e19fc0f1dbe 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -35,8 +35,8 @@ // description: "Upgrade conda dependencies", // fileMatch: ["(^|/)environment(.*).ya?ml$"], // matchStrings: [ - // '#\\s*renovate\\s+datasource=conda\\s+depName=(?.*?)\\s+-\\s*[\\w-]+\\s*==?\\s*"?(?.*)"?', - // '# renovate: datasource=conda depName=(?.*?)\\s+-\\s*[\\w-]+\\s*==?\\s*"?(?.*)"?', + // '#\\s*renovate\\s+datasource=conda\\s+depName=(?.*?)\\s+-\\s*[\\w-]+:?:?[\\w-]+\\s*==?\\s*"?(?.*)"?', + // '# renovate: datasource=conda depName=(?.*?)\\s+-\\s*[\\w-]+:?:?[\\w-]+\\s*==?\\s*"?(?.*)"?', // ], // datasourceTemplate: "conda", // }, From 5d776f1de39e0636515df9039f5639ea6d5289eb Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:48:08 -0600 Subject: [PATCH 02/10] Renovate DRY (#5768) * ci: Use renovate config from ops repo * ci: Add process_updaters * ci: Run test workflow on renovate branches to avoid creating PRs * ci: Enable process updaters * ci: Disable automerge for conda, pypi, and docker --- .github/renovate.json5 | 87 +++++--------------------------------- .github/workflows/test.yml | 4 ++ 2 files changed, 15 insertions(+), 76 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index e19fc0f1dbe..1f9b112d4f6 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,79 +1,14 @@ { - $schema: "https://docs.renovatebot.com/renovate-schema.json", - extends: ["config:best-practices"], - vulnerabilityAlerts: { - labels: ["security"], - schedule: "at any time", - }, - prHourlyLimit: 3, - platform: "github", - onboarding: false, - dependencyDashboard: true, - dependencyDashboardTitle: "Renovate Dashboard 🤖", - suppressNotifications: ["prIgnoreNotification"], - rebaseWhen: "conflicted", - reviewers: ["team:modules"], - assigneesFromCodeOwners: true, - gitAuthor: "nf-core-bot ", - prHeader: "## WARNING\n - [ ] I have made sure to update the singularity and conda images", - "pre-commit": { - enabled: true, - }, - customManagers: [ - // TODO waiting on https://github.com/nf-core/modules/pull/4080 - // { - // customType: "regex", - // description: "Process Containers dependencies", - // fileMatch: ["(^|/)main\\.nf$"], - // matchStrings: [ - // "\\s+'(?\\S+):(?.*\\-.*)' }\"\n", - // ], - // datasourceTemplate: "docker", - // }, - // { - // customType: "regex", - // description: "Upgrade conda dependencies", - // fileMatch: ["(^|/)environment(.*).ya?ml$"], - // matchStrings: [ - // '#\\s*renovate\\s+datasource=conda\\s+depName=(?.*?)\\s+-\\s*[\\w-]+:?:?[\\w-]+\\s*==?\\s*"?(?.*)"?', - // '# renovate: datasource=conda depName=(?.*?)\\s+-\\s*[\\w-]+:?:?[\\w-]+\\s*==?\\s*"?(?.*)"?', - // ], - // datasourceTemplate: "conda", - // }, - // { - // customType: "regex", - // description: "Upgrade pypi dependencies inside conda environment files", - // fileMatch: ["(^|/)environment(.*).ya?ml$"], - // matchStrings: [ - // "# renovate datasource=pypi\\s+-\\s*(?[\\w-]+)\\s*(\\[[\\w,\\s]+\\])?\\s*==?\\s*(?.*)", - // "# renovate: datasource=pypi\\s+-\\s*(?[\\w-]+)\\s*(\\[[\\w,\\s]+\\])?\\s*==?\\s*(?.*)", - // ], - // datasourceTemplate: "pypi", - // }, - ], - packageRules: [ - { - matchDatasources: ["docker"], - registryUrls: ["quay.io"], - }, - { - matchManagers: ["github-actions"], - automerge: true, - pinDigest: { - enabled: false, - }, - matchPackageNames: ["/^nf-core//", "/^seqera/"], - }, - { - matchManagers: ["github-actions"], - groupName: "GitHub Actions", - automerge: true, - schedule: ["before 9am"], - }, - { - automerge: true, - schedule: ["before 9am on Monday"], - matchPackageNames: ["/renovate-config-validator/"], - }, + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "github>nf-core/ops//.github/renovate/default.json5", + "github>nf-core/ops//.github/renovate/process_updaters.json5" ], + "prHeader": "## WARNING\n - [ ] I have made sure to update the singularity and conda images", + "packageRules": [ + { + "matchManagers": ["conda", "pypi", "docker"], + "automerge": false + } + ] } diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a4c49df720..8b0270f5b59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,9 @@ name: Run tests on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates pull_request: branches: [master] merge_group: From bc26bc782d279e7f4adab424123a188856ccd665 Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Wed, 13 Nov 2024 20:58:43 -0600 Subject: [PATCH 03/10] ci(renovate): Managers => Datasources (#6988) --- .github/renovate.json5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 1f9b112d4f6..46af7821e36 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -7,7 +7,7 @@ "prHeader": "## WARNING\n - [ ] I have made sure to update the singularity and conda images", "packageRules": [ { - "matchManagers": ["conda", "pypi", "docker"], + "matchDatasources": ["conda", "pypi", "docker"], "automerge": false } ] From 9900d6d9b5ce115418bfd266616f4664c39477ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 14 Nov 2024 08:15:03 +0100 Subject: [PATCH 04/10] temporarely switch github runners (#6989) --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b0270f5b59..43fe9733d3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -139,7 +139,7 @@ jobs: echo ${{ steps.outputs.outputs.subworkflows }} nf-core-lint-modules: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} name: nf-core-lint-modules needs: [pytest-changes, nf-test-changes] if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} @@ -223,7 +223,7 @@ jobs: run: nf-core subworkflows lint ${{ matrix.tags }} pytest: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} name: pytest needs: [pytest-changes] if: needs.pytest-changes.outputs.tags != '[]' @@ -489,7 +489,7 @@ jobs: !${{ github.workspace }}/.singularity nf-test: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} name: nf-test needs: [nf-test-changes] if: ( needs.nf-test-changes.outputs.paths != '[]' ) From 4265ef4b3b9af8877671715b081f102041c64cfd Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 14 Nov 2024 08:49:27 +0100 Subject: [PATCH 05/10] Update GANON/BUILDCUSTOM (fix support for alternative input type) (#6982) * Change input system to ensure fastas come with (optional) TSV rather than mutually exclusively (missing tests) * Fix ganon build module to work with file input * Update modules/nf-core/ganon/buildcustom/tests/main.nf.test --- modules/nf-core/ganon/buildcustom/main.nf | 6 +- modules/nf-core/ganon/buildcustom/meta.yml | 12 ++-- .../ganon/buildcustom/tests/main.nf.test | 55 +++++++++++++++++-- .../ganon/buildcustom/tests/main.nf.test.snap | 47 +++++++++++----- .../ganon/buildcustom/tests/nextflow.config | 2 +- .../nf-core/ganon/classify/tests/main.nf.test | 2 +- .../nf-core/ganon/report/tests/main.nf.test | 2 +- .../nf-core/ganon/table/tests/main.nf.test | 2 +- 8 files changed, 95 insertions(+), 33 deletions(-) diff --git a/modules/nf-core/ganon/buildcustom/main.nf b/modules/nf-core/ganon/buildcustom/main.nf index 212e49f7f97..41ffd688be1 100644 --- a/modules/nf-core/ganon/buildcustom/main.nf +++ b/modules/nf-core/ganon/buildcustom/main.nf @@ -8,7 +8,7 @@ process GANON_BUILDCUSTOM { input: tuple val(meta), path(input) - val input_type + path input_tsv path taxonomy_files path genome_size_files @@ -23,14 +23,14 @@ process GANON_BUILDCUSTOM { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def input_cmd = input_type == 'fasta' ? "--input ${input}" : input_type == 'tsv' ? "--input-file ${input}" : error("Invalid input type: ${input_type}. Options: fasta, tsv") + def input_cmd = input_tsv ? "--input-file ${input_tsv}" : "--input ${input}" def taxonomy_args = taxonomy_files ? "--taxonomy-files ${taxonomy_files}" : "" def genome_size_args = genome_size_files ? "--genome-size-files ${genome_size_files}" : "" """ ganon \\ build-custom \\ --threads ${task.cpus} \\ - --input ${input} \\ + ${input_cmd} \\ --db-prefix ${prefix} \\ ${taxonomy_args} \\ ${genome_size_args} \\ diff --git a/modules/nf-core/ganon/buildcustom/meta.yml b/modules/nf-core/ganon/buildcustom/meta.yml index 39bc07349ad..5c481ec0f44 100644 --- a/modules/nf-core/ganon/buildcustom/meta.yml +++ b/modules/nf-core/ganon/buildcustom/meta.yml @@ -28,14 +28,14 @@ input: description: | List of input FASTA files, or a directory containing input FASTA files. Note you must supply --input-extension via ext.args if FASTA extensions do not end in the default `fna.gz`. - pattern: "*" - - - input_type: + pattern: "*.{fasta,fna,fa,fa,fasta.gz,fna.gz,fa.gz,fa.gz}" + - - input_tsv: type: string description: | - Specify whether the file(s) given to the input channel are in FASTA format (and will be supplied as --input) - or in TSV format (and will be supplied as --input-file). For TSV format, the 'file' column should be just the - file name so that it's local to the working directory of this process. - pattern: "fasta|tsv" + (Optional) Specify an TSV file containing the paths, and relevant metadata to the input FASTA files to use the `--input-file` option. + The 'file' column should be just the file name of each FASTA file (so that it's local to the working directory of the process). + See ganon documentation for more more information on the other columns. + pattern: "*tsv" - - taxonomy_files: type: file description: Pre-downloaded taxonomy files of input sequences. See ganon docs diff --git a/modules/nf-core/ganon/buildcustom/tests/main.nf.test b/modules/nf-core/ganon/buildcustom/tests/main.nf.test index 8fa42278035..9fe3948955e 100644 --- a/modules/nf-core/ganon/buildcustom/tests/main.nf.test +++ b/modules/nf-core/ganon/buildcustom/tests/main.nf.test @@ -10,16 +10,19 @@ nextflow_process { tag "ganon" tag "ganon/buildcustom" - test("sarscov2 - genome fasta") { + test("sarscov2 - genome - fasta") { when { + params { + module_args = '--input-target sequence' + } process { """ input[0] = [ [ id:'test' ], // meta map file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), ] - input[1] = 'fasta' + input[1] = [] input[2] = [] input[3] = [] """ @@ -31,10 +34,49 @@ nextflow_process { { assert process.success }, { assert snapshot( process.out.db.get(0).get(1).findAll { file(it).name != "test.tax" }, - process.out.versions + process.out.versions, + file(process.out.db.get(0).get(1).find { file(it).name == "test.tax" }).text.contains("MT192765.1") ).match() }, - { assert file(process.out.db.get(0).get(1).find { file(it).name == "test.tax" }).text.contains("MT192765.1") }, + ) + } + } + +test("sarscov2 - genome - tsv") { + + when { + params { + module_args = '--input-target file' + } + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), + ] + input[1] = ch_ganon_input_tsv = Channel.of(["genome.fasta", "Severe_acute_respiratory_syndrome_coronavirus_2", "2697049"]). + map { it.join("\t") } + .collectFile ( + name: "ganon_fasta_input.tsv", + newLine: true + ) + input[2] = [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/metagenome/prot_nodes.dmp', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/metagenome/prot_names.dmp', checkIfExists: true) + ] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.db.get(0).get(1), + process.out.versions + ).match() + } ) } } @@ -44,13 +86,16 @@ nextflow_process { options "-stub" when { + params { + module_args = '' + } process { """ input[0] = [ [ id:'test' ], // meta map file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), ] - input[1] = 'fasta' + input[1] = [] input[2] = [] input[3] = [] """ diff --git a/modules/nf-core/ganon/buildcustom/tests/main.nf.test.snap b/modules/nf-core/ganon/buildcustom/tests/main.nf.test.snap index 2c3243ffbdc..e27a749e38a 100644 --- a/modules/nf-core/ganon/buildcustom/tests/main.nf.test.snap +++ b/modules/nf-core/ganon/buildcustom/tests/main.nf.test.snap @@ -1,4 +1,36 @@ { + "sarscov2 - genome - tsv": { + "content": [ + [ + "test.hibf:md5,9edfe4c3873d621a88ebcad438dca42c", + "test.tax:md5,e15400a1e43cce61545834695da46465" + ], + [ + "versions.yml:md5,9c73293ae36914c6ce3718ad6728ad9e" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-13T13:37:14.320278404" + }, + "sarscov2 - genome - fasta": { + "content": [ + [ + "test.hibf:md5,d10fe6fc6d198696bc15ca85a1459614" + ], + [ + "versions.yml:md5,9c73293ae36914c6ce3718ad6728ad9e" + ], + true + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-13T13:36:52.317157496" + }, "sarscov2 - genome fasta - stub": { "content": [ { @@ -53,20 +85,5 @@ "nextflow": "24.04.4" }, "timestamp": "2024-10-07T17:00:22.98042261" - }, - "sarscov2 - genome fasta": { - "content": [ - [ - "test.hibf:md5,d10fe6fc6d198696bc15ca85a1459614" - ], - [ - "versions.yml:md5,9c73293ae36914c6ce3718ad6728ad9e" - ] - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-10-07T19:03:25.060306554" } } \ No newline at end of file diff --git a/modules/nf-core/ganon/buildcustom/tests/nextflow.config b/modules/nf-core/ganon/buildcustom/tests/nextflow.config index a12988eee89..15c20b612c1 100644 --- a/modules/nf-core/ganon/buildcustom/tests/nextflow.config +++ b/modules/nf-core/ganon/buildcustom/tests/nextflow.config @@ -1,5 +1,5 @@ process { withName: GANON_BUILDCUSTOM { - ext.args = "--input-target sequence" + ext.args = params.module_args } } diff --git a/modules/nf-core/ganon/classify/tests/main.nf.test b/modules/nf-core/ganon/classify/tests/main.nf.test index 40ea069e782..c031e442d8b 100644 --- a/modules/nf-core/ganon/classify/tests/main.nf.test +++ b/modules/nf-core/ganon/classify/tests/main.nf.test @@ -20,7 +20,7 @@ nextflow_process { [ id:'test' ], // meta map file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), ] - input[1] = 'fasta' + input[1] = [] input[2] = [] input[3] = [] """ diff --git a/modules/nf-core/ganon/report/tests/main.nf.test b/modules/nf-core/ganon/report/tests/main.nf.test index e6437b498bc..5804d7af6d0 100644 --- a/modules/nf-core/ganon/report/tests/main.nf.test +++ b/modules/nf-core/ganon/report/tests/main.nf.test @@ -21,7 +21,7 @@ nextflow_process { [ id:'test' ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ] - input[1] = 'fasta' + input[1] = [] input[2] = [] input[3] = [] """ diff --git a/modules/nf-core/ganon/table/tests/main.nf.test b/modules/nf-core/ganon/table/tests/main.nf.test index 9aae4ead0c5..f7b97f66936 100644 --- a/modules/nf-core/ganon/table/tests/main.nf.test +++ b/modules/nf-core/ganon/table/tests/main.nf.test @@ -22,7 +22,7 @@ nextflow_process { [ id:'test' ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), ] - input[1] = 'fasta' + input[1] = [] input[2] = [] input[3] = [] """ From b48e64d5982a587e57e9d3fba585d852c68ac512 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 14 Nov 2024 08:50:23 +0100 Subject: [PATCH 06/10] Add FCSGX_CLEANGENOME (#6975) Co-authored-by: Lauren huet Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Co-authored-by: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> --- .../nf-core/fcsgx/cleangenome/environment.yml | 7 ++ modules/nf-core/fcsgx/cleangenome/main.nf | 51 +++++++++ modules/nf-core/fcsgx/cleangenome/meta.yml | 64 +++++++++++ .../fcsgx/cleangenome/tests/main.nf.test | 85 +++++++++++++++ .../fcsgx/cleangenome/tests/main.nf.test.snap | 100 ++++++++++++++++++ 5 files changed, 307 insertions(+) create mode 100644 modules/nf-core/fcsgx/cleangenome/environment.yml create mode 100644 modules/nf-core/fcsgx/cleangenome/main.nf create mode 100644 modules/nf-core/fcsgx/cleangenome/meta.yml create mode 100644 modules/nf-core/fcsgx/cleangenome/tests/main.nf.test create mode 100644 modules/nf-core/fcsgx/cleangenome/tests/main.nf.test.snap diff --git a/modules/nf-core/fcsgx/cleangenome/environment.yml b/modules/nf-core/fcsgx/cleangenome/environment.yml new file mode 100644 index 00000000000..8fbbad1b543 --- /dev/null +++ b/modules/nf-core/fcsgx/cleangenome/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::ncbi-fcs-gx=0.5.4" diff --git a/modules/nf-core/fcsgx/cleangenome/main.nf b/modules/nf-core/fcsgx/cleangenome/main.nf new file mode 100644 index 00000000000..e46507a6396 --- /dev/null +++ b/modules/nf-core/fcsgx/cleangenome/main.nf @@ -0,0 +1,51 @@ +process FCSGX_CLEANGENOME { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/ncbi-fcs-gx:0.5.4--h4ac6f70_1': + 'biocontainers/ncbi-fcs-gx:0.5.4--h4ac6f70_1' }" + + input: + tuple val(meta), path(fasta), path(fcsgx_report) + + output: + tuple val(meta), path("*.cleaned.fasta") , emit: cleaned + tuple val(meta), path("*.contaminants.fasta"), emit: contaminants + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + gx \\ + clean-genome \\ + --input ${fasta} \\ + --action-report ${fcsgx_report} \\ + --output ${prefix}.cleaned.fasta \\ + --contam-fasta-out ${prefix}.contaminants.fasta \\ + ${args} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fcsgx: \$( gx --help | sed '/build/!d; s/.*:v//; s/-.*//' ) + END_VERSIONS + """ + + stub: + // def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.cleaned.fasta + touch ${prefix}.contaminants.fasta + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fcsgx: \$( gx --help | sed '/build/!d; s/.*:v//; s/-.*//' ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/fcsgx/cleangenome/meta.yml b/modules/nf-core/fcsgx/cleangenome/meta.yml new file mode 100644 index 00000000000..380ec0283fc --- /dev/null +++ b/modules/nf-core/fcsgx/cleangenome/meta.yml @@ -0,0 +1,64 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "fcsgx_cleangenome" +description: Runs FCS-GX (Foreign Contamination Screen - Genome eXtractor) to + remove foreign contamination from genome assemblies +keywords: + - genome + - assembly + - contamination + - screening + - cleaning + - fcs-gx +tools: + - "fcsgx": + description: "The NCBI Foreign Contamination Screen. Genomic cross-species aligner, + for contamination detection." + homepage: "https://github.com/ncbi/fcs-gx" + documentation: "https://github.com/ncbi/fcs/wiki/" + tool_dev_url: "https://github.com/ncbi/fcs-gx" + doi: "10.1186/s13059-024-03198-7" + licence: ["NCBI-PD"] + identifier: "biotools:ncbi_fcs" + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - fasta: + type: file + description: Input genome assembly file in FASTA format + pattern: "*.{fa,fasta,fna}" + - fcsgx_report: + type: file + description: Final contamination report with contaminant cleaning actions. Generated using FCSGX_RUNGX +output: + - cleaned: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.cleaned.fasta": + type: file + description: The fasta file after cleaning, where sequences annotated as ACTION_EXCLUDE or ACTION_TRIM are excluded + - contaminants: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.contaminants.fasta": + type: file + description: Sequences annotated as ACTION_EXCLUDE which are marked as contaminants. + - versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@mahesh-panchal" + - "@LaurenHuet" +maintainers: + - "@mahesh-panchal" diff --git a/modules/nf-core/fcsgx/cleangenome/tests/main.nf.test b/modules/nf-core/fcsgx/cleangenome/tests/main.nf.test new file mode 100644 index 00000000000..8000093d744 --- /dev/null +++ b/modules/nf-core/fcsgx/cleangenome/tests/main.nf.test @@ -0,0 +1,85 @@ +nextflow_process { + + name "Test Process FCSGX_CLEANGENOME" + script "../main.nf" + process "FCSGX_CLEANGENOME" + + tag "modules" + tag "modules_nfcore" + tag "fcsgx" + tag "fcsgx/fetchdb" + tag "fcsgx/rungx" + tag "fcsgx/cleangenome" + + setup { + run("FCSGX_FETCHDB"){ + script "../../fetchdb/main.nf" + process { + """ + input[0] = file('https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/FCS/database/test-only/test-only.manifest', checkIfExists: true) + """ + } + } + run("FCSGX_RUNGX") { + script "../../rungx/main.nf" + process { + """ + input[0] = [ + [ id:'test' ], // meta map + '2697049', // taxid for SARS-CoV-2 + file('https://zenodo.org/records/10932013/files/FCS_combo_test.fa', checkIfExists: true), + ] + input[1] = FCSGX_FETCHDB.out.database + input[2] = [] + """ + } + } + } + + test("sarscov2 - fasta") { + + when { + process { + """ + input[0] = Channel.value([ + [ id:'test' ], // meta map + file('https://zenodo.org/records/10932013/files/FCS_combo_test.fa', checkIfExists: true), + ]).join(FCSGX_RUNGX.out.fcsgx_report) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("sarscov2 - fasta - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.value([ + [ id:'test' ], // meta map + file('https://zenodo.org/records/10932013/files/FCS_combo_test.fa', checkIfExists: true), + ]).join(FCSGX_RUNGX.out.fcsgx_report) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/fcsgx/cleangenome/tests/main.nf.test.snap b/modules/nf-core/fcsgx/cleangenome/tests/main.nf.test.snap new file mode 100644 index 00000000000..a62f95b480b --- /dev/null +++ b/modules/nf-core/fcsgx/cleangenome/tests/main.nf.test.snap @@ -0,0 +1,100 @@ +{ + "sarscov2 - fasta - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.cleaned.fasta:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.contaminants.fasta:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,4235d280269123cbd3e43d920bcf71b9" + ], + "cleaned": [ + [ + { + "id": "test" + }, + "test.cleaned.fasta:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "contaminants": [ + [ + { + "id": "test" + }, + "test.contaminants.fasta:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,4235d280269123cbd3e43d920bcf71b9" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-12T10:02:26.605398704" + }, + "sarscov2 - fasta": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.cleaned.fasta:md5,59ba4b4c773f02a0dc90034762aa1ddf" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.contaminants.fasta:md5,17c11ea7d2d0076be5183b87618ee91e" + ] + ], + "2": [ + "versions.yml:md5,4235d280269123cbd3e43d920bcf71b9" + ], + "cleaned": [ + [ + { + "id": "test" + }, + "test.cleaned.fasta:md5,59ba4b4c773f02a0dc90034762aa1ddf" + ] + ], + "contaminants": [ + [ + { + "id": "test" + }, + "test.contaminants.fasta:md5,17c11ea7d2d0076be5183b87618ee91e" + ] + ], + "versions": [ + "versions.yml:md5,4235d280269123cbd3e43d920bcf71b9" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-12T12:05:43.065207163" + } +} \ No newline at end of file From 03a9f356a1a333923c1177c2912fa7bc61bb46f3 Mon Sep 17 00:00:00 2001 From: Evangelos Karatzas <32259775+vagkaratzas@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:51:04 +0200 Subject: [PATCH 07/10] =?UTF-8?q?hmmalign=20ported=20to=20nf-test,=20versi?= =?UTF-8?q?on=20bump,=20wave=20container,=20standard=20ou=E2=80=A6=20(#699?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hmmalign ported to nf-test, version bump, wave container, standard output format, hmmbuild version bump, wave container, subworkflow updated accordingly --- .../nf-core/hmmer/hmmalign/environment.yml | 2 +- modules/nf-core/hmmer/hmmalign/main.nf | 21 ++- modules/nf-core/hmmer/hmmalign/meta.yml | 7 +- .../nf-core/hmmer/hmmalign/tests/main.nf.test | 58 +++++++ .../hmmer/hmmalign/tests/main.nf.test.snap | 68 ++++++++ .../nf-core/hmmer/hmmbuild/environment.yml | 2 +- modules/nf-core/hmmer/hmmbuild/main.nf | 4 +- .../nf-core/hmmer/hmmbuild/tests/main.nf.test | 12 +- .../hmmer/hmmbuild/tests/main.nf.test.snap | 14 +- .../nf-core/fasta_newick_epang_gappa/main.nf | 4 +- .../tests/main.nf.test.snap | 164 +++++++++--------- tests/config/pytest_modules.yml | 3 - tests/modules/nf-core/hmmer/hmmalign/main.nf | 30 ---- .../nf-core/hmmer/hmmalign/nextflow.config | 8 - tests/modules/nf-core/hmmer/hmmalign/test.yml | 16 -- 15 files changed, 247 insertions(+), 166 deletions(-) create mode 100644 modules/nf-core/hmmer/hmmalign/tests/main.nf.test create mode 100644 modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap delete mode 100644 tests/modules/nf-core/hmmer/hmmalign/main.nf delete mode 100644 tests/modules/nf-core/hmmer/hmmalign/nextflow.config delete mode 100644 tests/modules/nf-core/hmmer/hmmalign/test.yml diff --git a/modules/nf-core/hmmer/hmmalign/environment.yml b/modules/nf-core/hmmer/hmmalign/environment.yml index 7c62eac7e3a..c5ddec5d912 100644 --- a/modules/nf-core/hmmer/hmmalign/environment.yml +++ b/modules/nf-core/hmmer/hmmalign/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::hmmer=3.3.2 + - bioconda::hmmer=3.4 diff --git a/modules/nf-core/hmmer/hmmalign/main.nf b/modules/nf-core/hmmer/hmmalign/main.nf index 00ae8da2b95..39b17c787d8 100644 --- a/modules/nf-core/hmmer/hmmalign/main.nf +++ b/modules/nf-core/hmmer/hmmalign/main.nf @@ -4,16 +4,16 @@ process HMMER_HMMALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/hmmer:3.3.2--h1b792b2_1' : - 'biocontainers/hmmer:3.3.2--h1b792b2_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/07/07c4cbd91c4459dc86b13b5cd799cacba96b27d66c276485550d299c7a4c6f8a/data' : + 'community.wave.seqera.io/library/hmmer:3.4--cb5d2dd2e85974ca' }" input: tuple val(meta), path(fasta) path hmm output: - tuple val(meta), path("*.sthlm.gz"), emit: sthlm - path "versions.yml" , emit: versions + tuple val(meta), path("*.sto.gz"), emit: sto + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -25,7 +25,18 @@ process HMMER_HMMALIGN { hmmalign \\ $args \\ $hmm \\ - $fasta | gzip -c > ${prefix}.sthlm.gz + $fasta | gzip -c > ${prefix}.sto.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + hmmer: \$(hmmalign -h | grep -o '^# HMMER [0-9.]*' | sed 's/^# HMMER *//') + END_VERSIONS + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + echo | gzip > ${prefix}.sto.gz cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/hmmer/hmmalign/meta.yml b/modules/nf-core/hmmer/hmmalign/meta.yml index 366cf4f5a58..fb16ba6822e 100644 --- a/modules/nf-core/hmmer/hmmalign/meta.yml +++ b/modules/nf-core/hmmer/hmmalign/meta.yml @@ -30,16 +30,16 @@ input: description: A gzipped HMM file pattern: "*.hmm.gz" output: - - sthlm: + - sto: - meta: type: map description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - "*.sthlm.gz": + - "*.sto.gz": type: file description: Multiple alignment in gzipped Stockholm format - pattern: "*.sthlm.gz" + pattern: "*.sto.gz" - versions: - versions.yml: type: file @@ -51,3 +51,4 @@ authors: maintainers: - "@erikrikarddaniel" - "@jfy133" + - "@vagkaratzas" diff --git a/modules/nf-core/hmmer/hmmalign/tests/main.nf.test b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test new file mode 100644 index 00000000000..7b0368fc630 --- /dev/null +++ b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test @@ -0,0 +1,58 @@ + +nextflow_process { + + name "Test Process HMMER_HMMALIGN" + script "../main.nf" + process "HMMER_HMMALIGN" + + tag "modules" + tag "modules_nfcore" + tag "hmmer" + tag "hmmer/hmmalign" + + test("test-hmmer-hmmalign") { + + when { + process { + """ + input[0] = [ + [ id: 'test' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/e_coli_k12_16s.fna.gz', checkIfExists: true) + ] + input[1] = file(params.modules_testdata_base_path + 'delete_me/hmmer/bac.16S_rRNA.hmm.gz', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("test-hmmer-hmmalign-stub") { + options '-stub' + + when { + process { + """ + input[0] = [ + [ id: 'test' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/e_coli_k12_16s.fna.gz', checkIfExists: true) + ] + input[1] = file(params.modules_testdata_base_path + 'delete_me/hmmer/bac.16S_rRNA.hmm.gz', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + +} diff --git a/modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap new file mode 100644 index 00000000000..ae65cacb18b --- /dev/null +++ b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap @@ -0,0 +1,68 @@ +{ + "test-hmmer-hmmalign": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,4ae989d5ade2aaae9578cb88ba031e8f" + ] + ], + "1": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ], + "sto": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,4ae989d5ade2aaae9578cb88ba031e8f" + ] + ], + "versions": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-13T12:37:26.551497424" + }, + "test-hmmer-hmmalign-stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "1": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ], + "sto": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-13T12:37:32.244343836" + } +} \ No newline at end of file diff --git a/modules/nf-core/hmmer/hmmbuild/environment.yml b/modules/nf-core/hmmer/hmmbuild/environment.yml index 7c62eac7e3a..c5ddec5d912 100644 --- a/modules/nf-core/hmmer/hmmbuild/environment.yml +++ b/modules/nf-core/hmmer/hmmbuild/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::hmmer=3.3.2 + - bioconda::hmmer=3.4 diff --git a/modules/nf-core/hmmer/hmmbuild/main.nf b/modules/nf-core/hmmer/hmmbuild/main.nf index b4629cb47a6..8eed7fcd7ad 100644 --- a/modules/nf-core/hmmer/hmmbuild/main.nf +++ b/modules/nf-core/hmmer/hmmbuild/main.nf @@ -4,8 +4,8 @@ process HMMER_HMMBUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/hmmer:3.3.2--h87f3376_2': - 'biocontainers/hmmer:3.3.2--h1b792b2_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/07/07c4cbd91c4459dc86b13b5cd799cacba96b27d66c276485550d299c7a4c6f8a/data' : + 'community.wave.seqera.io/library/hmmer:3.4--cb5d2dd2e85974ca' }" input: tuple val(meta), path(alignment) diff --git a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test index d7c058255ea..635f5b661f1 100644 --- a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test +++ b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test @@ -16,9 +16,9 @@ nextflow_process { process { """ input[0] = [ - [ id: 'PF14720' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) - ] + [ id: 'PF14720' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) + ] input[1] = [] """ @@ -46,9 +46,9 @@ nextflow_process { process { """ input[0] = [ - [ id: 'PF14720' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) - ] + [ id: 'PF14720' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) + ] input[1] = [] """ diff --git a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap index 8cdff17c142..79317b89ebd 100644 --- a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap +++ b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap @@ -14,7 +14,7 @@ "PF14720.hmmbuild.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "2": [ - "versions.yml:md5,6f4796b714cb0d5981cd12d9e89cb599" + "versions.yml:md5,f8a0bffcbbc58404752849403812905b" ], "hmm": [ [ @@ -28,28 +28,28 @@ "PF14720.hmmbuild.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "versions": [ - "versions.yml:md5,6f4796b714cb0d5981cd12d9e89cb599" + "versions.yml:md5,f8a0bffcbbc58404752849403812905b" ] } ], "meta": { "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nextflow": "23.04.5" }, - "timestamp": "2024-09-05T21:38:44.751216" + "timestamp": "2024-11-13T12:52:29.794123574" }, "test-hmmer-hmmbuild": { "content": [ "PF14720.hmm.gz", "PF14720.hmmbuild.txt", [ - "versions.yml:md5,6f4796b714cb0d5981cd12d9e89cb599" + "versions.yml:md5,f8a0bffcbbc58404752849403812905b" ] ], "meta": { "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nextflow": "23.04.5" }, - "timestamp": "2024-09-05T21:49:25.772308" + "timestamp": "2024-11-13T12:52:23.95935055" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf b/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf index 28016014f94..723f4a2d91a 100644 --- a/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf +++ b/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf @@ -82,10 +82,10 @@ workflow FASTA_NEWICK_EPANG_GAPPA { ch_versions = ch_versions.mix(HMMER_HMMALIGNQUERY.out.versions) // 1.d Mask the alignments (Add '--rf-is-mask' ext.args in config for the process.) - HMMER_MASKREF ( HMMER_HMMALIGNREF.out.sthlm.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) + HMMER_MASKREF ( HMMER_HMMALIGNREF.out.sto.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) ch_versions = ch_versions.mix(HMMER_MASKREF.out.versions) - HMMER_MASKQUERY ( HMMER_HMMALIGNQUERY.out.sthlm.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) + HMMER_MASKQUERY ( HMMER_HMMALIGNQUERY.out.sto.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) ch_versions = ch_versions.mix(HMMER_MASKQUERY.out.versions) // 1.e Reformat alignments to "afa" (aligned fasta) diff --git a/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap index e6627edbe99..207d28e5013 100644 --- a/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap @@ -4,201 +4,201 @@ "hmmer.taxonomy.per_query.tsv" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:45:46.123551197" + "timestamp": "2024-11-14T10:16:44.874521222" }, "mafft.taxonomy.profile.tsv_lines": { "content": "eb4ae24c330c4b4ab768f7d1c26c7215", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:24:06.138041" + "timestamp": "2024-11-14T10:16:59.978336429" + }, + "hmmer.graft.placement.epa_result.newick_lines": { + "content": "8533b159d543f75b92efcfb4fd330280", + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-14T10:16:44.866852673" }, "hmmer.taxonomy.profile.tsv": { "content": [ "hmmer.taxonomy.profile.tsv" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-04-05T20:45:46.115530176" - }, - "hmmer.graft.placement.epa_result.newick_lines": { - "content": "8533b159d543f75b92efcfb4fd330280", - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T10:55:06.361779" + "timestamp": "2024-11-14T10:16:44.868720482" }, "hmmer.taxonomy.profile.tsv_lines": { "content": "8533b159d543f75b92efcfb4fd330280", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T10:59:31.223864" + "timestamp": "2024-11-14T10:16:44.871385161" }, "mafft.graft.placement.epa_result.newick_lines": { "content": "eb4ae24c330c4b4ab768f7d1c26c7215", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:24:06.117996" + "timestamp": "2024-11-14T10:16:59.973967772" }, "mafft.taxonomy.per_query.tsv_lines": { "content": "eb4ae24c330c4b4ab768f7d1c26c7215", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:24:06.175662" + "timestamp": "2024-11-14T10:16:59.98411855" }, "nucl_hmmer.versions.yml": { "content": [ "versions.yml" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:42.212896" + "timestamp": "2024-11-14T10:17:41.730837855" }, "hmmer.graft.placement.epa_result.newick": { "content": [ "hmmer.graft.hmmer.epa_result.newick" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:45:46.105219765" + "timestamp": "2024-11-14T10:16:44.863462591" }, "hmmer.versions.yml": { "content": [ "versions.yml" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:05.782093" + "timestamp": "2024-11-14T10:16:44.880743216" }, - "nucl_hmmer.graft.placement.epa_result.newick": { + "hmmer.epa_result.jplace.gz": { "content": [ - "nucl_hmmer.graft.nucl_hmmer.epa_result.newick" + "hmmer.epa_result.jplace.gz" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:37:46.19414" + "timestamp": "2024-11-14T10:16:44.857734455" }, - "hmmer.epa_result.jplace.gz": { + "nucl_hmmer.graft.placement.epa_result.newick": { "content": [ - "hmmer.epa_result.jplace.gz" + "nucl_hmmer.graft.nucl_hmmer.epa_result.newick" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:56:49.564967" + "timestamp": "2024-11-14T10:17:41.728034651" }, "hmmer.taxonomy.per_query.tsv_lines": { "content": "8533b159d543f75b92efcfb4fd330280", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T10:59:31.236689" + "timestamp": "2024-11-14T10:16:44.878231148" }, "mafft.taxonomy.per_query.tsv": { "content": [ "mafft.taxonomy.per_query.tsv" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:57:13.505168194" + "timestamp": "2024-11-14T10:16:59.980370338" }, "mafft.graft.placement.epa_result.newick": { "content": [ "mafft.graft.mafft.epa_result.newick" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:57:13.49856233" + "timestamp": "2024-11-14T10:16:59.972095017" }, "nucl_hmmer.epa_result.jplace.gz": { "content": [ "nucl_hmmer.epa_result.jplace.gz" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:44:49.494246" + "timestamp": "2024-11-14T10:17:41.725684052" }, "nucl_hmmer.versions.yml_lines": { - "content": "664dc00d47aecd41a420dc6ba7ef4eea", + "content": "eafb4184937621216ff03df17c4245a6", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:42.217138" + "timestamp": "2024-11-14T10:17:41.733022413" }, - "mafft.taxonomy.profile.tsv": { - "content": [ - "mafft.taxonomy.profile.tsv" - ], + "hmmer.versions.yml_lines": { + "content": "eafb4184937621216ff03df17c4245a6", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:57:13.501534227" + "timestamp": "2024-11-14T10:16:44.884598465" }, "mafft.epa_result.jplace.gz": { "content": [ "mafft.epa_result.jplace.gz" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:57:06.815658" + "timestamp": "2024-11-14T10:16:59.970209461" }, - "hmmer.versions.yml_lines": { - "content": "664dc00d47aecd41a420dc6ba7ef4eea", + "mafft.taxonomy.profile.tsv": { + "content": [ + "mafft.taxonomy.profile.tsv" + ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:05.849215" + "timestamp": "2024-11-14T10:16:59.975738553" }, "mafft.versions.yml": { "content": [ "versions.yml" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:20.88609" + "timestamp": "2024-11-14T10:16:59.986610634" }, "mafft.versions.yml_lines": { "content": "f5957093def0191f32bb294000fb9242", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:20.890963" + "timestamp": "2024-11-14T10:16:59.988948939" } } \ No newline at end of file diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 55984e2de9e..2e0e6420c33 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -299,9 +299,6 @@ hmmer/eslalimask: hmmer/eslreformat: - modules/nf-core/hmmer/eslreformat/** - tests/modules/nf-core/hmmer/eslreformat/** -hmmer/hmmalign: - - modules/nf-core/hmmer/hmmalign/** - - tests/modules/nf-core/hmmer/hmmalign/** hmtnote/annotate: - modules/nf-core/hmtnote/annotate/** - tests/modules/nf-core/hmtnote/annotate/** diff --git a/tests/modules/nf-core/hmmer/hmmalign/main.nf b/tests/modules/nf-core/hmmer/hmmalign/main.nf deleted file mode 100644 index 64814641c3d..00000000000 --- a/tests/modules/nf-core/hmmer/hmmalign/main.nf +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { HMMER_HMMALIGN as HMMER_HMMALIGN } from '../../../../../modules/nf-core/hmmer/hmmalign/main.nf' -include { HMMER_HMMALIGN as HMMER_HMMALIGN_PREFIX } from '../../../../../modules/nf-core/hmmer/hmmalign/main.nf' - -workflow test_hmmer_hmmalign { - - input = [ - [ id:'test' ], // meta map - file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/e_coli_k12_16s.fna.gz') // Change to params.test_data syntax after the data is included in ./tests/config/test_data.config - ] - - hmm = file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/bac.16S_rRNA.hmm.gz') - - HMMER_HMMALIGN ( input, hmm ) -} - -workflow test_hmmer_hmmalign_prefix { - - input = [ - [ id:'test' ], // meta map - file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/e_coli_k12_16s.fna.gz') // Change to params.test_data syntax after the data is included in ./tests/config/test_data.config - ] - - hmm = file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/bac.16S_rRNA.hmm.gz') - - HMMER_HMMALIGN_PREFIX ( input, hmm ) -} diff --git a/tests/modules/nf-core/hmmer/hmmalign/nextflow.config b/tests/modules/nf-core/hmmer/hmmalign/nextflow.config deleted file mode 100644 index b2115af55d9..00000000000 --- a/tests/modules/nf-core/hmmer/hmmalign/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: HMMER_HMMALIGN_PREFIX { - ext.prefix = { "${meta.id}.prefix" } - } -} diff --git a/tests/modules/nf-core/hmmer/hmmalign/test.yml b/tests/modules/nf-core/hmmer/hmmalign/test.yml deleted file mode 100644 index 2f483956c98..00000000000 --- a/tests/modules/nf-core/hmmer/hmmalign/test.yml +++ /dev/null @@ -1,16 +0,0 @@ -- name: hmmer hmmalign test_hmmer_hmmalign - command: nextflow run ./tests/modules/nf-core/hmmer/hmmalign -entry test_hmmer_hmmalign -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/hmmer/hmmalign/nextflow.config - tags: - - hmmer - - hmmer/hmmalign - files: - - path: output/hmmer/test.sthlm.gz - md5sum: ddaa8b96291edf4e1a929a224329161b -- name: hmmer hmmalign test_hmmer_hmmalign_prefix - command: nextflow run ./tests/modules/nf-core/hmmer/hmmalign -entry test_hmmer_hmmalign_prefix -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/hmmer/hmmalign/nextflow.config - tags: - - hmmer - - hmmer/hmmalign - files: - - path: output/hmmer/test.prefix.sthlm.gz - md5sum: ddaa8b96291edf4e1a929a224329161b From 920895254d42e874c035d07d783d40da7395c2da Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Thu, 14 Nov 2024 18:12:33 +0530 Subject: [PATCH 08/10] Fix/biscuit apps (#6979) * init update biscuit apps * update index module + test; use seqera container * update align module + test; use seqera container * update blaster container + test * update container + test * update container + test * update mergecg container + test + rm tags.yml * update pileup container + test * update index container * update qc container + test * update vcf2bed container + test * fix biscuit, samtools cpu determination --- modules/nf-core/biscuit/align/environment.yml | 4 +- modules/nf-core/biscuit/align/main.nf | 33 ++++-- modules/nf-core/biscuit/align/meta.yml | 21 +++- .../nf-core/biscuit/align/tests/main.nf.test | 70 ++++++++++-- .../biscuit/align/tests/main.nf.test.snap | 87 ++++++++++++-- modules/nf-core/biscuit/align/tests/tags.yml | 3 - .../biscuit/biscuitblaster/environment.yml | 4 +- .../nf-core/biscuit/biscuitblaster/main.nf | 32 +++--- .../nf-core/biscuit/biscuitblaster/meta.yml | 24 +++- .../biscuit/biscuitblaster/tests/main.nf.test | 58 ++++++---- .../biscuitblaster/tests/main.nf.test.snap | 53 ++++----- .../biscuit/biscuitblaster/tests/tags.yml | 2 - .../nf-core/biscuit/bsconv/environment.yml | 2 +- modules/nf-core/biscuit/bsconv/main.nf | 19 ++- modules/nf-core/biscuit/bsconv/meta.yml | 26 ++++- .../nf-core/biscuit/bsconv/tests/main.nf.test | 49 +++++--- .../biscuit/bsconv/tests/main.nf.test.snap | 63 ++-------- modules/nf-core/biscuit/bsconv/tests/tags.yml | 2 - .../nf-core/biscuit/epiread/environment.yml | 4 +- modules/nf-core/biscuit/epiread/main.nf | 31 +++-- modules/nf-core/biscuit/epiread/meta.yml | 35 +++++- .../biscuit/epiread/tests/main.nf.test | 92 +++++++++++---- .../biscuit/epiread/tests/main.nf.test.snap | 92 +++++++-------- modules/nf-core/biscuit/index/environment.yml | 2 +- modules/nf-core/biscuit/index/main.nf | 11 +- modules/nf-core/biscuit/index/meta.yml | 18 ++- .../nf-core/biscuit/index/tests/main.nf.test | 17 +-- .../biscuit/index/tests/main.nf.test.snap | 108 +++++++++++------- .../nf-core/biscuit/mergecg/environment.yml | 4 +- modules/nf-core/biscuit/mergecg/main.nf | 21 ++-- modules/nf-core/biscuit/mergecg/meta.yml | 21 +++- .../biscuit/mergecg/tests/main.nf.test | 47 ++++---- .../biscuit/mergecg/tests/main.nf.test.snap | 36 +++--- .../nf-core/biscuit/mergecg/tests/tags.yml | 2 - .../nf-core/biscuit/pileup/environment.yml | 4 +- modules/nf-core/biscuit/pileup/main.nf | 15 +-- modules/nf-core/biscuit/pileup/meta.yml | 19 ++- .../nf-core/biscuit/pileup/tests/main.nf.test | 36 ++++-- .../biscuit/pileup/tests/main.nf.test.snap | 64 +++++------ modules/nf-core/biscuit/qc/environment.yml | 2 +- modules/nf-core/biscuit/qc/main.nf | 13 ++- modules/nf-core/biscuit/qc/meta.yml | 20 +++- modules/nf-core/biscuit/qc/tests/main.nf.test | 21 +++- .../biscuit/qc/tests/main.nf.test.snap | 28 ++--- .../nf-core/biscuit/vcf2bed/environment.yml | 4 +- modules/nf-core/biscuit/vcf2bed/main.nf | 12 +- modules/nf-core/biscuit/vcf2bed/meta.yml | 1 + .../biscuit/vcf2bed/tests/main.nf.test | 14 +-- .../biscuit/vcf2bed/tests/main.nf.test.snap | 24 ++-- 49 files changed, 839 insertions(+), 531 deletions(-) delete mode 100644 modules/nf-core/biscuit/align/tests/tags.yml delete mode 100644 modules/nf-core/biscuit/biscuitblaster/tests/tags.yml delete mode 100644 modules/nf-core/biscuit/bsconv/tests/tags.yml delete mode 100644 modules/nf-core/biscuit/mergecg/tests/tags.yml diff --git a/modules/nf-core/biscuit/align/environment.yml b/modules/nf-core/biscuit/align/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/align/environment.yml +++ b/modules/nf-core/biscuit/align/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/align/main.nf b/modules/nf-core/biscuit/align/main.nf index 5154748725d..0d43dc05183 100644 --- a/modules/nf-core/biscuit/align/main.nf +++ b/modules/nf-core/biscuit/align/main.nf @@ -4,12 +4,13 @@ process BISCUIT_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(reads) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: tuple val(meta), path("*.bam"), emit: bam @@ -23,18 +24,34 @@ process BISCUIT_ALIGN { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.9),1) - def samtools_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def samtools_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit align \\ $args \\ -@ $biscuit_cpus \\ - \$INDEX \\ + $index/$fasta \\ $reads \\ - | samtools sort $args2 --threads $samtools_cpus --write-index -o ${prefix}.bam##idx##${prefix}.bam.bai - + | samtools sort $args2 --threads $samtools_cpus --write-index -o ${prefix}.bam -O BAM - + samtools index ${prefix}.bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + biscuit: \$( biscuit version |& sed '1!d; s/^.*BISCUIT Version: //' ) + samtools: \$( samtools --version |& sed '1!d; s/^.*samtools //' ) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.bam + touch ${prefix}.bam.bai cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/align/meta.yml b/modules/nf-core/biscuit/align/meta.yml index 645fc2c98a3..464e435bfa9 100644 --- a/modules/nf-core/biscuit/align/meta.yml +++ b/modules/nf-core/biscuit/align/meta.yml @@ -27,12 +27,24 @@ input: - reads: type: file description: | - List of input fastq files of size 1 and 2 for single-end and paired-end data, + List of input FastQ files of size 1 and 2 for single-end and paired-end data, respectively. - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - bam: - meta: @@ -63,3 +75,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/align/tests/main.nf.test b/modules/nf-core/biscuit/align/tests/main.nf.test index 1dc3eb3faca..84feb7a2249 100644 --- a/modules/nf-core/biscuit/align/tests/main.nf.test +++ b/modules/nf-core/biscuit/align/tests/main.nf.test @@ -12,16 +12,19 @@ nextflow_process { setup { run("BISCUIT_INDEX") { - script "../../index/main.nf" + script "../../../biscuit/index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("Single-End sarscov2 test_methylated_1 [fastq_gz]") { + test("sarscov2 single-end [fastq_gz]") { when { process { @@ -30,7 +33,11 @@ nextflow_process { [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz', checkIfExists: true) ] ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -38,14 +45,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert file(process.out.bam[0][1]).name == "test.bam" }, - { assert file(process.out.bai[0][1]).name == "test.bam.bai" }, - { assert snapshot(process.out.versions).match("single-end-versions") } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } ) } } - test("Paired-End sarscov2 test_methylated_1 [fastq_gz]") { + test("sarscov2 paired-end [fastq_gz]") { when { process { @@ -55,7 +65,11 @@ nextflow_process { [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz', checkIfExists: true) ] ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -63,9 +77,41 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert file(process.out.bam[0][1]).name == "test.bam" }, - { assert file(process.out.bai[0][1]).name == "test.bam.bai" }, - { assert snapshot(process.out.versions).match("paired-end-versions") } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } + ) + } + } + + test("sarscov2 paired-end [fastq_gz] - stub") { + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz', checkIfExists: true) ] + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() + } ) } } diff --git a/modules/nf-core/biscuit/align/tests/main.nf.test.snap b/modules/nf-core/biscuit/align/tests/main.nf.test.snap index 5776f87c62f..b6aeda38f7b 100644 --- a/modules/nf-core/biscuit/align/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/align/tests/main.nf.test.snap @@ -1,26 +1,93 @@ { - "single-end-versions": { + "sarscov2 single-end [fastq_gz]": { "content": [ + "499055250804bb3674c724bd9adf9d02", [ - "versions.yml:md5,0b2a8c4c438a785a890110a458399bfd" + [ + { + "id": "test" + }, + "test.bam.bai:md5,a748afc3cec6c27906ae80638e2024f5" + ] + ], + [ + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-22T20:06:02.295706212" + "timestamp": "2024-11-14T10:03:51.461022384" }, - "paired-end-versions": { + "sarscov2 paired-end [fastq_gz]": { "content": [ + "6b5afc62da5533472e1479950bb5f09a", + [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,ba60b802f583496e330c555bca9f0b97" + ] + ], [ - "versions.yml:md5,0b2a8c4c438a785a890110a458399bfd" + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T10:04:02.88266119" + }, + "sarscov2 paired-end [fastq_gz] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" + ], + "bai": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-22T20:06:17.879689395" + "timestamp": "2024-11-13T06:48:19.558887499" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/align/tests/tags.yml b/modules/nf-core/biscuit/align/tests/tags.yml deleted file mode 100644 index d73b8183f7f..00000000000 --- a/modules/nf-core/biscuit/align/tests/tags.yml +++ /dev/null @@ -1,3 +0,0 @@ -biscuit/align: - - modules/nf-core/biscuit/index/** - - modules/nf-core/biscuit/align/** diff --git a/modules/nf-core/biscuit/biscuitblaster/environment.yml b/modules/nf-core/biscuit/biscuitblaster/environment.yml index ba50e31d11c..74a7b2fb908 100644 --- a/modules/nf-core/biscuit/biscuitblaster/environment.yml +++ b/modules/nf-core/biscuit/biscuitblaster/environment.yml @@ -2,6 +2,6 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 - bioconda::samblaster=0.1.26 - - bioconda::samtools=1.16.1 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/biscuitblaster/main.nf b/modules/nf-core/biscuit/biscuitblaster/main.nf index c4f7e1a52c5..f7876cb2fb5 100644 --- a/modules/nf-core/biscuit/biscuitblaster/main.nf +++ b/modules/nf-core/biscuit/biscuitblaster/main.nf @@ -4,12 +4,13 @@ process BISCUIT_BLASTER { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/22/22f141a92e12f98040054b9a47a51fb4de9c544174bbfa6d251a110f5e26fddc/data': + 'community.wave.seqera.io/library/biscuit_samblaster_samtools:a683d0887e7a91bf' }" input: tuple val(meta), path(reads) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: tuple val(meta), path("*.bam"), emit: bam @@ -24,23 +25,24 @@ process BISCUIT_BLASTER { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def args3 = task.ext.args3 ?: '' - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.95),1) - def samtools_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def samtools_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit align \\ -@ $biscuit_cpus \\ $args \\ - \$INDEX \\ - $reads | \\ - samblaster \\ - $args2 | \\ - samtools sort \\ - -@ $samtools_cpus \\ - $args3 \\ - --write-index \\ - -o ${prefix}.bam##idx##${prefix}.bam.bai + $index/$fasta\\ + $reads \\ + | samblaster $args2 \\ + | samtools sort \\ + -@ $samtools_cpus \\ + $args3 \\ + --write-index \\ + -o ${prefix}.bam -O BAM - + + samtools index ${prefix}.bam cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/biscuitblaster/meta.yml b/modules/nf-core/biscuit/biscuitblaster/meta.yml index d64cc1c2694..f2bed154053 100644 --- a/modules/nf-core/biscuit/biscuitblaster/meta.yml +++ b/modules/nf-core/biscuit/biscuitblaster/meta.yml @@ -1,6 +1,5 @@ name: biscuit_blaster -description: A fast, compact one-liner to produce duplicate-marked, sorted, and indexed - BAM files using Biscuit +description: A fast, compact one-liner to produce duplicate-marked, sorted, and indexed BAM files using Biscuit keywords: - biscuit - DNA methylation @@ -48,12 +47,24 @@ input: - reads: type: file description: | - List of input fastq files of size 1 and 2 for single-end and paired-end data, + List of input FastQ files of size 1 and 2 for single-end and paired-end data, respectively. - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - bam: - meta: @@ -84,3 +95,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test index ccea7f5b85e..9b7f7562ed8 100644 --- a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test +++ b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test @@ -14,21 +14,27 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.fasta", checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2 - single") { + test("sarscov2 single-end [fastq_gz]") { when { process { """ input[0] = [ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz", checkIfExists: true) ] - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -36,25 +42,30 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(file(process.out.bai[0][1]).name, - process.out.bam.collect { bam(it[1]).getHeaderMD5() }, - process.out.bam.collect { bam(it[1]).getReadsMD5() }, - process.out.versions).match() } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } ) } } - test("sarscov2 - paired-end") { + test("sarscov2 paired-end [fastq_gz]") { when { process { """ input[0] = [ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz", checkIfExists: true), file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz", checkIfExists: true) ] - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -62,15 +73,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.bai, - process.out.bam.collect { bam(it[1]).getHeaderMD5() }, - process.out.bam.collect { bam(it[1]).getReadsMD5() }, - process.out.versions).match() } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } ) } } - test("sarscov2 - paired-end -stub") { + test("sarscov2 paired-end [fastq_gz] - stub") { options "-stub" when { process { @@ -78,9 +91,12 @@ nextflow_process { input[0] = [ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz", checkIfExists: true), file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz", checkIfExists: true) ] - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap index e537de7d8ca..72f3eea613f 100644 --- a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap @@ -1,50 +1,47 @@ { - "sarscov2 - single": { + "sarscov2 single-end [fastq_gz]": { "content": [ - "test.bam.bai", + "499055250804bb3674c724bd9adf9d02", [ - "a3b85097046b61879ab72293c38daea0" - ], - [ - "499055250804bb3674c724bd9adf9d02" + [ + { + "id": "test" + }, + "test.bam.bai:md5,56efb7d64af887266e8490a4afd6171f" + ] ], [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-10T11:20:35.533081588" + "timestamp": "2024-11-14T06:39:47.556767238" }, - "sarscov2 - paired-end": { + "sarscov2 paired-end [fastq_gz]": { "content": [ + "6b5afc62da5533472e1479950bb5f09a", [ [ { "id": "test" }, - "test.bam.bai:md5,bc235b43786886957bd5bb5b6623d4dd" + "test.bam.bai:md5,128302694afc1cf9e04b45c41860785f" ] ], [ - "d66c238598e02630bba7595baa2a6e71" - ], - [ - "6b5afc62da5533472e1479950bb5f09a" - ], - [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-10T10:57:59.954913657" + "timestamp": "2024-11-14T06:40:01.376002655" }, - "sarscov2 - paired-end -stub": { + "sarscov2 paired-end [fastq_gz] - stub": { "content": [ { "0": [ @@ -64,7 +61,7 @@ ] ], "2": [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ], "bai": [ [ @@ -83,14 +80,14 @@ ] ], "versions": [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-10T10:58:19.26950429" + "timestamp": "2024-11-14T06:40:14.345664344" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/biscuitblaster/tests/tags.yml b/modules/nf-core/biscuit/biscuitblaster/tests/tags.yml deleted file mode 100644 index 13adde05efb..00000000000 --- a/modules/nf-core/biscuit/biscuitblaster/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -biscuit/biscuitblaster: - - "modules/nf-core/biscuit/biscuitblaster/**" diff --git a/modules/nf-core/biscuit/bsconv/environment.yml b/modules/nf-core/biscuit/bsconv/environment.yml index 56b7e967693..123215dc20f 100644 --- a/modules/nf-core/biscuit/bsconv/environment.yml +++ b/modules/nf-core/biscuit/bsconv/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 diff --git a/modules/nf-core/biscuit/bsconv/main.nf b/modules/nf-core/biscuit/bsconv/main.nf index 881fa0be813..5fb47439d1c 100644 --- a/modules/nf-core/biscuit/bsconv/main.nf +++ b/modules/nf-core/biscuit/bsconv/main.nf @@ -2,18 +2,19 @@ process BISCUIT_BSCONV { tag "$meta.id" label 'process_long' - conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/biscuit:1.1.0.20220707--he272189_1': - 'biocontainers/biscuit:1.1.0.20220707--he272189_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/33/33a9ca30b4154f11253c8d91a75382065dcb8282ba99b74dbee59ed8faceabd7/data': + 'community.wave.seqera.io/library/biscuit:1.5.0.20240506--ca92d9d0a37b5fa8' }" input: - tuple val(meta), path(bam), path(bai) - path(index) + tuple val(meta), path(bam) + tuple val(meta2), path(bai) + tuple val(meta3), path(fasta) + tuple val(meta4), path(index) output: - tuple val(meta), path("*.bam"), emit: bsconv_bam + tuple val(meta), path("*.bam"), emit: bam path "versions.yml" , emit: versions when: @@ -24,11 +25,11 @@ process BISCUIT_BSCONV { def prefix = task.ext.prefix ?: "${meta.id}" if ("$bam" == "${prefix}.bam") error "Input and output names are the same, set prefix in module configuration to disambiguate!" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit bsconv \\ $args \\ - \$INDEX \\ + $index/$fasta \\ $bam \\ ${prefix}.bam @@ -41,8 +42,6 @@ process BISCUIT_BSCONV { stub: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - if ("$bam" == "${prefix}.bam") error "Input and output names are the same, set prefix in module configuration to disambiguate!" - """ touch ${prefix}.bam diff --git a/modules/nf-core/biscuit/bsconv/meta.yml b/modules/nf-core/biscuit/bsconv/meta.yml index 75d9b2b7939..e162e175af0 100644 --- a/modules/nf-core/biscuit/bsconv/meta.yml +++ b/modules/nf-core/biscuit/bsconv/meta.yml @@ -27,15 +27,32 @@ input: - bam: type: file description: BAM file contained mapped reads + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - bai: type: file description: BAM file index - - - index: + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta4: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - - bsconv_bam: + - bam: - meta: type: map description: | @@ -54,3 +71,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/bsconv/tests/main.nf.test b/modules/nf-core/biscuit/bsconv/tests/main.nf.test index 1cbf306a88b..331c3df01f8 100644 --- a/modules/nf-core/biscuit/bsconv/tests/main.nf.test +++ b/modules/nf-core/biscuit/bsconv/tests/main.nf.test @@ -9,29 +9,36 @@ nextflow_process { tag "biscuit/bsconv" tag "biscuit/index" - setup { run("BISCUIT_INDEX") { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2-pe - bam") { + test("sarscov2 paired-end [bam]") { when { process { """ input[0] = [ [id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ [id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[2] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[3] = BISCUIT_INDEX.out.index """ } } @@ -39,14 +46,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.bsconv_bam).match("bsconv_bam") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.versions + ).match() + } ) } } - test("sarscov2-pe - bam - stub") { + test("sarscov2 paired-end [bam] - stub") { options "-stub" @@ -54,11 +64,16 @@ nextflow_process { process { """ input[0] = [ [id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ [id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[2] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[3] = BISCUIT_INDEX.out.index """ } } @@ -66,7 +81,11 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.versions + ).match() + } ) } diff --git a/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap b/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap index 96dc77c097b..7796ebbb5ad 100644 --- a/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap @@ -1,67 +1,28 @@ { - "versions": { + "sarscov2 paired-end [bam]": { "content": [ + "1608a01cabbbaba635e2905b7049f487", [ - "versions.yml:md5,45455cc9522a3235fa482ad065a242b3" + "versions.yml:md5,f8bba061ed5d28794abb4ffc2898aa02" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-07T15:28:04.601734997" + "timestamp": "2024-11-14T08:18:08.129627771" }, - "sarscov2-pe - bam - stub": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "1": [ - "versions.yml:md5,45455cc9522a3235fa482ad065a242b3" - ], - "bsconv_bam": [ - [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "versions": [ - "versions.yml:md5,45455cc9522a3235fa482ad065a242b3" - ] - } - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-05-07T15:28:14.729945289" - }, - "bsconv_bam": { + "sarscov2 paired-end [bam] - stub": { "content": [ + "d41d8cd98f00b204e9800998ecf8427e", [ - [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,0d425851bd18963ebf1b6d727a7d02c7" - ] + "versions.yml:md5,f8bba061ed5d28794abb4ffc2898aa02" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-07T15:28:04.580771449" + "timestamp": "2024-11-14T08:18:18.658974235" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/bsconv/tests/tags.yml b/modules/nf-core/biscuit/bsconv/tests/tags.yml deleted file mode 100644 index 5b890ac3597..00000000000 --- a/modules/nf-core/biscuit/bsconv/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -biscuit/bsconv: - - "modules/nf-core/biscuit/bsconv/**" diff --git a/modules/nf-core/biscuit/epiread/environment.yml b/modules/nf-core/biscuit/epiread/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/epiread/environment.yml +++ b/modules/nf-core/biscuit/epiread/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/epiread/main.nf b/modules/nf-core/biscuit/epiread/main.nf index 8eeb675c09c..110177c000c 100644 --- a/modules/nf-core/biscuit/epiread/main.nf +++ b/modules/nf-core/biscuit/epiread/main.nf @@ -4,15 +4,18 @@ process BISCUIT_EPIREAD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: - tuple val(meta), path(bam), path(bai), path(snp_bed) - path(index) + tuple val(meta), path(bam) + tuple val(meta2), path(bai) + tuple val(meta3), path(snp_bed) + tuple val(meta4), path(fasta) + tuple val(meta5), path(index) output: - tuple val(meta), path("*.bed.gz"), emit: epiread_bed + tuple val(meta), path("*.bed.gz"), emit: bed path "versions.yml" , emit: versions when: @@ -22,8 +25,8 @@ process BISCUIT_EPIREAD { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.9),1) - def samtools_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def samtools_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) // As of 2/25/22, epiread does not support reading a gzipped SNP BED file. // This is a bit hacky but allows the user to supply a gzipped OR uncompressed bed file def unzip_snp_bed = snp_bed && (snp_bed.toString() =~ /\.gz$/) ? "bgzip -d ${snp_bed}" : "" @@ -32,7 +35,7 @@ process BISCUIT_EPIREAD { def options_snp_bed = snp_bed ? "-B ${unzipped_snp_bed}" : "" if ("$options_snp_bed" == "${prefix}.bed.gz") error "Input and output names for biscuit epiread are the same, set prefix in module configuration to disambiguate!" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta $unzip_snp_bed @@ -40,11 +43,10 @@ process BISCUIT_EPIREAD { -@ $biscuit_cpus \\ $args \\ $options_snp_bed \\ - \$INDEX \\ - $bam | \\ - LC_ALL=C sort -k1,1 -k2,2n | \\ - bgzip \\ - -@ $samtools_cpus \\ + $index/$fasta \\ + $bam \\ + | LC_ALL=C sort -k1,1 -k2,2n \\ + | bgzip -@ $samtools_cpus \\ $args2 \\ -c > ${prefix}.bed.gz @@ -57,9 +59,6 @@ process BISCUIT_EPIREAD { stub: def prefix = task.ext.prefix ?: "${meta.id}" - def unzipped_snp_bed = snp_bed ? snp_bed.toString() - ~/\.gz$/: "" - def options_snp_bed = snp_bed ? "-B ${unzipped_snp_bed}" : "" - if ("$options_snp_bed" == "${prefix}.bed.gz") error "Input and output names for biscuit epiread are the same, set prefix in module configuration to disambiguate!" """ echo | gzip > ${prefix}.bed.gz diff --git a/modules/nf-core/biscuit/epiread/meta.yml b/modules/nf-core/biscuit/epiread/meta.yml index 8974f1d952a..b695be1cd8f 100644 --- a/modules/nf-core/biscuit/epiread/meta.yml +++ b/modules/nf-core/biscuit/epiread/meta.yml @@ -27,19 +27,41 @@ input: e.g. [ id:'test', single_end:false ] - bam: type: file - description: Biscuit BAM file + description: BAM file contained mapped reads + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - bai: type: file - description: BAM index + description: BAM file index + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - snp_bed: type: file description: BED file containing SNP information (optional) - - - index: + - - meta4: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta5: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - - epiread_bed: + - bed: - meta: type: map description: | @@ -58,3 +80,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/epiread/tests/main.nf.test b/modules/nf-core/biscuit/epiread/tests/main.nf.test index 5a151008afa..ceb1bff5b6f 100644 --- a/modules/nf-core/biscuit/epiread/tests/main.nf.test +++ b/modules/nf-core/biscuit/epiread/tests/main.nf.test @@ -15,24 +15,36 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2 - test_paired_end_methylated_sorted_bam") { + test("sarscov2 paired-end [bam]") { when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - [] //SNP BED file + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], [] //SNP BED file + ] + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } @@ -46,18 +58,28 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - snp") { + test("sarscov2 paired-end [bam] [snp]") { when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed') + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], + file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } @@ -71,18 +93,28 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz") { + test("sarscov2 paired-end [bam] [snp_gz]") { when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz') + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], + file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz', checkIfExists: true) + ] + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } @@ -96,19 +128,29 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz - stub") { + test("sarscov2 paired-end [bam] [snp_gz] - stub") { options '-stub' when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz') + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], + file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap b/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap index 1a64679d465..19a1e069912 100644 --- a/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap @@ -1,142 +1,134 @@ { - "sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz": { + "sarscov2 paired-end [bam] [snp_gz]": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:24.162952" + "timestamp": "2024-11-14T08:58:54.313858378" }, - "sarscov2 - test_paired_end_methylated_sorted_bam - snp": { + "sarscov2 paired-end [bam]": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,f75d8fe928fe2dc1a3ce6178d8963602" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,f75d8fe928fe2dc1a3ce6178d8963602" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:17.32731" + "timestamp": "2024-11-14T08:58:32.745357698" }, - "sarscov2 - test_paired_end_methylated_sorted_bam": { + "sarscov2 paired-end [bam] [snp]": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,3f60061464bd1c282bce0dc6478f8738" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,3f60061464bd1c282bce0dc6478f8738" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:09.919651" + "timestamp": "2024-11-14T08:58:43.662473332" }, - "sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz - stub": { + "sarscov2 paired-end [bam] [snp_gz] - stub": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:29.932067" + "timestamp": "2024-11-14T08:59:04.687765786" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/index/environment.yml b/modules/nf-core/biscuit/index/environment.yml index 56b7e967693..123215dc20f 100644 --- a/modules/nf-core/biscuit/index/environment.yml +++ b/modules/nf-core/biscuit/index/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 diff --git a/modules/nf-core/biscuit/index/main.nf b/modules/nf-core/biscuit/index/main.nf index 43566fc4b50..fb5ab0d40c5 100644 --- a/modules/nf-core/biscuit/index/main.nf +++ b/modules/nf-core/biscuit/index/main.nf @@ -4,15 +4,15 @@ process BISCUIT_INDEX { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/biscuit:1.1.0.20220707--he272189_1': - 'biocontainers/biscuit:1.1.0.20220707--he272189_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/33/33a9ca30b4154f11253c8d91a75382065dcb8282ba99b74dbee59ed8faceabd7/data': + 'community.wave.seqera.io/library/biscuit:1.5.0.20240506--ca92d9d0a37b5fa8' }" input: - path fasta, stageAs: "BiscuitIndex/*" + tuple val(meta), path(fasta, name:"BiscuitIndex/") output: - path "BiscuitIndex/*.fa*", emit: index, includeInputs: true - path "versions.yml" , emit: versions + tuple val(meta), path("BiscuitIndex"), emit: index + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -34,7 +34,6 @@ process BISCUIT_INDEX { stub: def args = task.ext.args ?: '' """ - touch ${fasta}.bis.amb touch ${fasta}.bis.ann touch ${fasta}.bis.pac diff --git a/modules/nf-core/biscuit/index/meta.yml b/modules/nf-core/biscuit/index/meta.yml index 39a0bcf8d94..cdf51a8b4d6 100644 --- a/modules/nf-core/biscuit/index/meta.yml +++ b/modules/nf-core/biscuit/index/meta.yml @@ -19,15 +19,24 @@ tools: licence: ["MIT"] identifier: biotools:biscuit input: - - - fasta: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: type: file description: Input genome fasta file output: - index: - - BiscuitIndex/*.fa*: + - meta: type: directory - description: Biscuit genome index directory - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index + pattern: "index" + - BiscuitIndex: + type: directory + description: Directory containing biscuit genome index + pattern: "index" - versions: - versions.yml: type: file @@ -37,3 +46,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/index/tests/main.nf.test b/modules/nf-core/biscuit/index/tests/main.nf.test index 0402a262200..3c99b27bb08 100644 --- a/modules/nf-core/biscuit/index/tests/main.nf.test +++ b/modules/nf-core/biscuit/index/tests/main.nf.test @@ -1,4 +1,3 @@ - nextflow_process { name "Test Process BISCUIT_INDEX" @@ -10,13 +9,15 @@ nextflow_process { tag "biscuit" tag "biscuit/index" - test("test-biscuit-index") { + test("sarscov2 genome [fasta]") { when { process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } @@ -29,14 +30,16 @@ nextflow_process { } } - test("test-biscuit-index-stub") { + test("sarscov2 genome [fasta] - stub") { options '-stub' when { process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } diff --git a/modules/nf-core/biscuit/index/tests/main.nf.test.snap b/modules/nf-core/biscuit/index/tests/main.nf.test.snap index 698fc7e4bd8..715875012bd 100644 --- a/modules/nf-core/biscuit/index/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/index/tests/main.nf.test.snap @@ -1,84 +1,104 @@ { - "test-biscuit-index": { + "sarscov2 genome [fasta] - stub": { "content": [ { "0": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", - "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", - "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", - "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", - "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", - "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", - "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ] ], "1": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ], "index": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", - "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", - "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", - "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", - "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", - "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", - "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ] ], "versions": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-28T10:33:50.146795" + "timestamp": "2024-11-13T06:48:54.473235719" }, - "test-biscuit-index-stub": { + "sarscov2 genome [fasta]": { "content": [ { "0": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", + "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", + "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", + "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", + "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + ] ] ], "1": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ], "index": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", + "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", + "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", + "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", + "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + ] ] ], "versions": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T10:49:18.069711" + "timestamp": "2024-11-13T06:48:44.596263127" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/mergecg/environment.yml b/modules/nf-core/biscuit/mergecg/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/mergecg/environment.yml +++ b/modules/nf-core/biscuit/mergecg/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/mergecg/main.nf b/modules/nf-core/biscuit/mergecg/main.nf index 93094b2a6cd..14efa0b7d81 100644 --- a/modules/nf-core/biscuit/mergecg/main.nf +++ b/modules/nf-core/biscuit/mergecg/main.nf @@ -4,16 +4,17 @@ process BISCUIT_MERGECG { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(bed) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: - tuple val(meta), path("*.bed.gz"), emit: mergecg_bed + tuple val(meta), path("*.bed.gz"), emit: bed path "versions.yml" , emit: versions when: @@ -24,16 +25,14 @@ process BISCUIT_MERGECG { def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit mergecg \\ $args \\ - \$INDEX \\ - $bed | \\ - LC_ALL=C sort -k1,1 -k2,2n | \\ - bgzip \\ - $args2 \\ - -c > ${prefix}.bed.gz + $index/$fasta\\ + $bed \\ + | LC_ALL=C sort -k1,1 -k2,2n \\ + | bgzip $args2 -c > ${prefix}.bed.gz cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/mergecg/meta.yml b/modules/nf-core/biscuit/mergecg/meta.yml index 2084f43b9d6..29b0889a4a1 100644 --- a/modules/nf-core/biscuit/mergecg/meta.yml +++ b/modules/nf-core/biscuit/mergecg/meta.yml @@ -27,12 +27,24 @@ input: type: file description: | Biscuit BED file (output of biscuit vcf2bed) - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - - mergecg_bed: + - bed: - meta: type: map description: | @@ -51,3 +63,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/mergecg/tests/main.nf.test b/modules/nf-core/biscuit/mergecg/tests/main.nf.test index 0b593f76842..cfa29b719d9 100644 --- a/modules/nf-core/biscuit/mergecg/tests/main.nf.test +++ b/modules/nf-core/biscuit/mergecg/tests/main.nf.test @@ -9,27 +9,34 @@ nextflow_process { tag "biscuit/index" tag "biscuit/mergecg" - test("sars_cov_bed") { - - setup { + setup { run("BISCUIT_INDEX") { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.fasta", checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } + + test("sarscov2 cov [bed]") { + when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + "delete_me/biscuit/test-cg.bed.gz", checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -43,27 +50,21 @@ nextflow_process { } - test("sars_cov_bed -stub") { + test("sarscov2 cov [bed] - stub") { options "-stub" - setup { - run("BISCUIT_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.fasta", checkIfExists: true) - """ - } - } - } + when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + "delete_me/biscuit/test-cg.bed.gz", checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap b/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap index 148a1f947bd..f003dee7b8d 100644 --- a/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "sars_cov_bed": { + "sarscov2 cov [bed] - stub": { "content": [ { "0": [ @@ -8,33 +8,33 @@ "id": "test", "single_end": false }, - "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" + "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "1": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ], - "mergecg_bed": [ + "bed": [ [ { "id": "test", "single_end": false }, - "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" + "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "versions": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-11T17:31:48.698330905" + "timestamp": "2024-11-14T09:09:45.257367761" }, - "sars_cov_bed -stub": { + "sarscov2 cov [bed]": { "content": [ { "0": [ @@ -43,30 +43,30 @@ "id": "test", "single_end": false }, - "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" ] ], "1": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ], - "mergecg_bed": [ + "bed": [ [ { "id": "test", "single_end": false }, - "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" ] ], "versions": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-12T09:59:25.605969084" + "timestamp": "2024-11-14T09:09:30.544664794" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/mergecg/tests/tags.yml b/modules/nf-core/biscuit/mergecg/tests/tags.yml deleted file mode 100644 index d7140b07550..00000000000 --- a/modules/nf-core/biscuit/mergecg/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -biscuit/mergecg: - - "modules/nf-core/biscuit/mergecg/**" diff --git a/modules/nf-core/biscuit/pileup/environment.yml b/modules/nf-core/biscuit/pileup/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/pileup/environment.yml +++ b/modules/nf-core/biscuit/pileup/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/pileup/main.nf b/modules/nf-core/biscuit/pileup/main.nf index dbbd6ffa2d5..db51aae2c09 100644 --- a/modules/nf-core/biscuit/pileup/main.nf +++ b/modules/nf-core/biscuit/pileup/main.nf @@ -4,12 +4,13 @@ process BISCUIT_PILEUP { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(normal_bams), path(normal_bais), path(tumor_bam), path(tumor_bai) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: tuple val(meta), path("*.vcf.gz"), emit: vcf @@ -22,18 +23,18 @@ process BISCUIT_PILEUP { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.9),1) - def bgzip_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def bgzip_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) if ( tumor_bam != [] && normal_bams.toList().size() > 1 ) error "[BISCUIT_PILEUP] error: Tumor BAM provided with more than one normal BAM" if ( tumor_bam.toList().size() > 1 ) error "[BISCUIT_PILEUP] error: more than one tumor BAM provided" input = ( tumor_bam==[] ) ? "${normal_bams}" : "-S -T ${tumor_bam} -I ${normal_bams}" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit pileup \\ -@ $biscuit_cpus \\ $args \\ - \$INDEX \\ + $index/$fasta \\ $input \\ | bgzip -@ $bgzip_cpus $args2 > ${prefix}.vcf.gz diff --git a/modules/nf-core/biscuit/pileup/meta.yml b/modules/nf-core/biscuit/pileup/meta.yml index 9f532797e43..0ab31cc549b 100644 --- a/modules/nf-core/biscuit/pileup/meta.yml +++ b/modules/nf-core/biscuit/pileup/meta.yml @@ -47,10 +47,22 @@ input: type: file description: Optional. BAM index file corresponding to provided tumor_bam pattern: "*.{bai}" - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - vcf: - meta: @@ -71,3 +83,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/pileup/tests/main.nf.test b/modules/nf-core/biscuit/pileup/tests/main.nf.test index b188103a06f..74ebff4d271 100644 --- a/modules/nf-core/biscuit/pileup/tests/main.nf.test +++ b/modules/nf-core/biscuit/pileup/tests/main.nf.test @@ -15,18 +15,21 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("homo_sapiens - test_paired_end_sorted_bam") { + test("homo_sapiens - paired-end [bam]") { when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), @@ -38,9 +41,12 @@ nextflow_process { ], [], // tumor bam [] // tumor bai - ] - - input[1] = BISCUIT_INDEX.out.index + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -58,7 +64,7 @@ nextflow_process { } - test("homo_sapiens - test_paired_end_sorted_bam - somatic") { + test("homo_sapiens - paired-end [bam] - somatic") { when { process { @@ -70,8 +76,11 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai', checkIfExists: true) ] - - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -89,7 +98,7 @@ nextflow_process { } - test("homo_sapiens - test_paired_end_sorted_bam - somatic - stub") { + test("homo_sapiens - paired-end [bam] - somatic - stub") { options '-stub' when { @@ -102,8 +111,11 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai', checkIfExists: true) ] - - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap b/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap index ca99df65997..14c40aa1a4d 100644 --- a/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap @@ -1,31 +1,5 @@ { - "homo_sapiens - test_paired_end_sorted_bam - somatic": { - "content": [ - "12357b78693a704403196644b34d69a6", - [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" - }, - "timestamp": "2024-08-29T11:43:13.955407" - }, - "homo_sapiens - test_paired_end_sorted_bam": { - "content": [ - "4ae9e1f5d3dbe07aca279680840bbe20", - [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" - }, - "timestamp": "2024-08-29T11:41:00.666921" - }, - "homo_sapiens - test_paired_end_sorted_bam - somatic - stub": { + "homo_sapiens - paired-end [bam] - somatic - stub": { "content": [ { "0": [ @@ -37,7 +11,7 @@ ] ], "1": [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" ], "vcf": [ [ @@ -48,14 +22,40 @@ ] ], "versions": [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T09:24:21.352967241" + }, + "homo_sapiens - paired-end [bam] - somatic": { + "content": [ + "5e1157703410c1a6b7cc7eded1436425", + [ + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T09:27:58.885274336" + }, + "homo_sapiens - paired-end [bam]": { + "content": [ + "6bea182d0b89d10246cc0ac711f6507", + [ + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:43:19.616012" + "timestamp": "2024-11-14T09:27:47.811858126" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/qc/environment.yml b/modules/nf-core/biscuit/qc/environment.yml index 56b7e967693..123215dc20f 100644 --- a/modules/nf-core/biscuit/qc/environment.yml +++ b/modules/nf-core/biscuit/qc/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 diff --git a/modules/nf-core/biscuit/qc/main.nf b/modules/nf-core/biscuit/qc/main.nf index 0ac7bd9eeab..4e63292ebd9 100644 --- a/modules/nf-core/biscuit/qc/main.nf +++ b/modules/nf-core/biscuit/qc/main.nf @@ -4,15 +4,16 @@ process BISCUIT_QC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/biscuit:1.1.0.20220707--he272189_1': - 'biocontainers/biscuit:1.1.0.20220707--he272189_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/33/33a9ca30b4154f11253c8d91a75382065dcb8282ba99b74dbee59ed8faceabd7/data': + 'community.wave.seqera.io/library/biscuit:1.5.0.20240506--ca92d9d0a37b5fa8' }" input: tuple val(meta), path(bam) - path(index) + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: - tuple val(meta), path("*.txt"), emit: biscuit_qc_reports + tuple val(meta), path("*.txt"), emit: reports path "versions.yml" , emit: versions when: @@ -23,12 +24,12 @@ process BISCUIT_QC { def prefix = task.ext.prefix ?: "${meta.id}" def se = meta.single_end ? "-s" : "" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit qc \\ $args \\ $se \\ - \$INDEX \\ + $index/$fasta \\ $bam \\ $prefix diff --git a/modules/nf-core/biscuit/qc/meta.yml b/modules/nf-core/biscuit/qc/meta.yml index ab7500ba823..379f46bbf3e 100644 --- a/modules/nf-core/biscuit/qc/meta.yml +++ b/modules/nf-core/biscuit/qc/meta.yml @@ -27,11 +27,24 @@ input: - bam: type: file description: BAM file produced using Biscuit - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: type: file - description: index file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: + type: directory + description: Directory containing biscuit genome index output: - - biscuit_qc_reports: + - reports: - meta: type: file description: | @@ -65,3 +78,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/qc/tests/main.nf.test b/modules/nf-core/biscuit/qc/tests/main.nf.test index d4b4e57ad47..24d0dc23729 100644 --- a/modules/nf-core/biscuit/qc/tests/main.nf.test +++ b/modules/nf-core/biscuit/qc/tests/main.nf.test @@ -15,13 +15,16 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2 - test_paired_end_methylated_sorted_bam") { + test("sarscov2 paired-end [bam]") { when { process { @@ -30,7 +33,11 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -44,7 +51,7 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - stub") { + test("sarscov2 paired-end [bam] - stub") { options '-stub' when { @@ -54,7 +61,11 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/qc/tests/main.nf.test.snap b/modules/nf-core/biscuit/qc/tests/main.nf.test.snap index 51c995d9e5b..4c5deb3fd8d 100644 --- a/modules/nf-core/biscuit/qc/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/qc/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "sarscov2 - test_paired_end_methylated_sorted_bam": { + "sarscov2 paired-end [bam]": { "content": [ { "0": [ @@ -20,9 +20,9 @@ ] ], "1": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ], - "biscuit_qc_reports": [ + "reports": [ [ { "id": "test", @@ -40,17 +40,17 @@ ] ], "versions": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:27:13.208945" + "timestamp": "2024-11-14T09:49:31.146030396" }, - "sarscov2 - test_paired_end_methylated_sorted_bam - stub": { + "sarscov2 paired-end [bam] - stub": { "content": [ { "0": [ @@ -71,9 +71,9 @@ ] ], "1": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ], - "biscuit_qc_reports": [ + "reports": [ [ { "id": "test", @@ -91,14 +91,14 @@ ] ], "versions": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:30:49.37928" + "timestamp": "2024-11-14T09:49:41.278929227" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/vcf2bed/environment.yml b/modules/nf-core/biscuit/vcf2bed/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/vcf2bed/environment.yml +++ b/modules/nf-core/biscuit/vcf2bed/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/vcf2bed/main.nf b/modules/nf-core/biscuit/vcf2bed/main.nf index f2ba512ef63..d53c8ee342b 100644 --- a/modules/nf-core/biscuit/vcf2bed/main.nf +++ b/modules/nf-core/biscuit/vcf2bed/main.nf @@ -4,8 +4,8 @@ process BISCUIT_VCF2BED { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(vcf) @@ -24,11 +24,9 @@ process BISCUIT_VCF2BED { """ biscuit vcf2bed \\ $args \\ - $vcf | \\ - LC_ALL=C sort -k1,1 -k2,2n | \\ - bgzip \\ - $args2 \\ - -c > ${prefix}.bed.gz + $vcf \\ + | LC_ALL=C sort -k1,1 -k2,2n \\ + | bgzip $args2 -c > ${prefix}.bed.gz cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/vcf2bed/meta.yml b/modules/nf-core/biscuit/vcf2bed/meta.yml index 76ee1cad11c..b50fa56d44a 100644 --- a/modules/nf-core/biscuit/vcf2bed/meta.yml +++ b/modules/nf-core/biscuit/vcf2bed/meta.yml @@ -48,3 +48,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test index 527f0c0ff4f..b9097a07e02 100644 --- a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test +++ b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test @@ -10,16 +10,15 @@ nextflow_process { tag "biscuit" tag "biscuit/vcf2bed" - test("test-biscuit-vcf2bed") { + test("test [vcf]") { when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test.vcf.gz', checkIfExists: true) - ] - + ]) """ } } @@ -32,17 +31,16 @@ nextflow_process { } } - test("test-biscuit-vcf2bed-stub") { + test("test [vcf] - stub") { options '-stub' when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test.vcf.gz', checkIfExists: true) - ] - + ]) """ } } diff --git a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap index fd232298b85..816bd1cceaa 100644 --- a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "test-biscuit-vcf2bed": { + "test [vcf]": { "content": [ { "0": [ @@ -12,7 +12,7 @@ ] ], "1": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ], "bed": [ [ @@ -24,17 +24,17 @@ ] ], "versions": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:02:38.571195" + "timestamp": "2024-11-14T09:55:25.366750014" }, - "test-biscuit-vcf2bed-stub": { + "test [vcf] - stub": { "content": [ { "0": [ @@ -47,7 +47,7 @@ ] ], "1": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ], "bed": [ [ @@ -59,14 +59,14 @@ ] ], "versions": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:02:43.49219" + "timestamp": "2024-11-14T09:55:34.060320551" } } \ No newline at end of file From 3ef36024bb95d306c63c7bf9014132e62c7b4755 Mon Sep 17 00:00:00 2001 From: Evangelos Karatzas <32259775+vagkaratzas@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:03:39 +0200 Subject: [PATCH 09/10] clipkit - new module (#6993) * clipkit - new module * standard prefix.clipkit output + better meta description for the alignment file --- modules/nf-core/clipkit/environment.yml | 7 ++ modules/nf-core/clipkit/main.nf | 46 +++++++++++++ modules/nf-core/clipkit/meta.yml | 50 ++++++++++++++ modules/nf-core/clipkit/tests/main.nf.test | 66 ++++++++++++++++++ .../nf-core/clipkit/tests/main.nf.test.snap | 68 +++++++++++++++++++ 5 files changed, 237 insertions(+) create mode 100644 modules/nf-core/clipkit/environment.yml create mode 100644 modules/nf-core/clipkit/main.nf create mode 100644 modules/nf-core/clipkit/meta.yml create mode 100644 modules/nf-core/clipkit/tests/main.nf.test create mode 100644 modules/nf-core/clipkit/tests/main.nf.test.snap diff --git a/modules/nf-core/clipkit/environment.yml b/modules/nf-core/clipkit/environment.yml new file mode 100644 index 00000000000..65c451ffb1f --- /dev/null +++ b/modules/nf-core/clipkit/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::clipkit=2.3.0" diff --git a/modules/nf-core/clipkit/main.nf b/modules/nf-core/clipkit/main.nf new file mode 100644 index 00000000000..2a9ebe3f851 --- /dev/null +++ b/modules/nf-core/clipkit/main.nf @@ -0,0 +1,46 @@ +process CLIPKIT { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/clipkit:2.3.0--pyhdfd78af_0': + 'biocontainers/clipkit:2.3.0--pyhdfd78af_0' }" + + input: + tuple val(meta), path(aln) + + output: + tuple val(meta), path("*.clipkit"), emit: clipkit + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + clipkit \\ + $args \\ + $aln \\ + -o ${prefix}.clipkit + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + clipkit: \$(clipkit --version |& sed '1!d ; s/clipkit //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.clipkit + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + clipkit: \$(clipkit --version |& sed '1!d ; s/clipkit //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/clipkit/meta.yml b/modules/nf-core/clipkit/meta.yml new file mode 100644 index 00000000000..5347ad5923a --- /dev/null +++ b/modules/nf-core/clipkit/meta.yml @@ -0,0 +1,50 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "clipkit" +description: ClipKIT is a fast and flexible alignment trimming tool that keeps phylogenetically informative sites and removes those that display characteristics poor phylogenetic signal. +keywords: + - sort + - example + - genomics +tools: + - "clipkit": + description: "Alignment trimming software for phylogenetics." + homepage: "https://jlsteenwyk.com/ClipKIT/" + documentation: "https://jlsteenwyk.com/ClipKIT/" + tool_dev_url: "https://github.com/JLSteenwyk/ClipKIT" + doi: "10.1371/journal.pbio.3001007" + licence: ["MIT"] + identifier: "" + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - aln: + type: file + description: Multiple sequence alignment file in various supported formats. + pattern: "*.{fa,fasta,fa,fna,faa,alnfaa,aln,sto,stk,mauve,alignment,clustal}" + +output: + - clipkit: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.clipkit": + type: file + description: Trimmed multiple sequence alignment file + pattern: "*.clipkit" + - versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@vagkaratzas" +maintainers: + - "@vagkaratzas" diff --git a/modules/nf-core/clipkit/tests/main.nf.test b/modules/nf-core/clipkit/tests/main.nf.test new file mode 100644 index 00000000000..46cc8a00716 --- /dev/null +++ b/modules/nf-core/clipkit/tests/main.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test Process CLIPKIT" + script "../main.nf" + process "CLIPKIT" + + tag "modules" + tag "modules_nfcore" + tag "clipkit" + tag "gunzip" + + setup { + run("GUNZIP") { + script "../../gunzip/main.nf" + process { + """ + input[0] = [ + [ id:'PF14720' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) + ] + """ + } + } + } + + test("PF14720 - aln") { + + when { + process { + """ + input[0] = GUNZIP.out.gunzip + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("PF14720 - aln - stub") { + + options "-stub" + + when { + process { + """ + input[0] = GUNZIP.out.gunzip + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/clipkit/tests/main.nf.test.snap b/modules/nf-core/clipkit/tests/main.nf.test.snap new file mode 100644 index 00000000000..55f837d9e52 --- /dev/null +++ b/modules/nf-core/clipkit/tests/main.nf.test.snap @@ -0,0 +1,68 @@ +{ + "PF14720 - aln - stub": { + "content": [ + { + "0": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ], + "clipkit": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-14T13:11:21.109400968" + }, + "PF14720 - aln": { + "content": [ + { + "0": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,97e4328b5990f5e5a6241c225ed6d922" + ] + ], + "1": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ], + "clipkit": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,97e4328b5990f5e5a6241c225ed6d922" + ] + ], + "versions": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-14T13:11:14.278212183" + } +} \ No newline at end of file From 0e2d0c6d94f048eb1ba88924a0c87edc2c1bd6ef Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:25:15 -0600 Subject: [PATCH 10/10] Wave with Dockerfiles (#4940) * ci(renovate): Add Dockerfile building * build: Bump quarto to 24.04 * ci: Get clever with singularity flag * build: Try a better Dockerfile * ci(wave): Better names * ci(wave): Properly skip environment.yml and Dockerfiles * style(wittyer): Remove environment.yml * ci(wave): Let Dockerfiles fail and request @edmundmiller and @maxulysse --- .github/CODEOWNERS | 1 + .github/workflows/wave.yml | 101 ++++++++++++++-------- modules/nf-core/quartonotebook/Dockerfile | 2 +- modules/nf-core/wittyer/Dockerfile | 2 + modules/nf-core/wittyer/environment.yml | 5 -- 5 files changed, 71 insertions(+), 40 deletions(-) delete mode 100644 modules/nf-core/wittyer/environment.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3893bf75737..9552f7c99ad 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,3 +12,4 @@ modules/nf-core/pints/caller/** @edmundmiller modules/nf-core/preseq/ccurve/** @edmundmiller modules/nf-core/preseq/lcextrap/** @edmundmiller subworkflows/nf-core/homer/** @edmundmiller +modules/nf-core/**/Dockerfile @edmundmiller @maxulysse diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 4f36575b6d1..a86fd9bb10a 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - "**/environment.yml" + - "**/Dockerfile" # TODO On complete call testing CI # TODO Skip testing CI if any changes to environment.yml @@ -18,38 +19,44 @@ jobs: runs-on: ubuntu-latest # Only run on Pull Requests within the same repository, and not from forks if: github.event.pull_request.head.repo.full_name == github.repository - + outputs: + conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }} + docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - name: Calculate file differences - id: diff + - name: Find conda differences + id: conda-diff uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45 with: json: true - quotepath: false - # TODO Add Dockerfiles + escape_json: false files: | modules/**/environment.yml - - name: Debug - run: echo ${{ steps.diff.outputs.all_changed_files }} - - id: set-matrix - run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT" - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + - name: Find Dockerfile differences + id: docker-diff + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45 + with: + json: true + escape_json: false + files: | + modules/**/Dockerfile - build: + conda-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}" + if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }} needs: generate-matrix - name: build + name: Build Conda-based ${{ matrix.profile }} Container runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false - matrix: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}" + max-parallel: 4 + matrix: + files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}" + profile: [docker, singularity] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -59,38 +66,64 @@ jobs: sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave - - name: Create a registry name - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - id: registry-name - with: - result-encoding: string - script: | - return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_'); - - - name: Build container + - name: Build ${{ matrix.profile }} container # FIXME Hack while iron out the CI continue-on-error: true - if: matrix.profile == 'docker' + env: + PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} run: | wave --conda-file "${{ matrix.files }}" \ + $PROFILE \ --freeze \ --await \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} - - name: Build Singularity - # FIXME Hack while iron out the CI - continue-on-error: true - if: matrix.profile == 'singularity' + docker-wave: + # NOTE This should get skipped because generate-matrix won't run + # if: github.repository == 'nf-core/modules' + if: ${{ needs.generate-matrix.outputs.docker-matrix != '[]' }} + needs: generate-matrix + name: Build Dockerfile-based ${{ matrix.profile }} Container + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + files: "${{ fromJson(needs.generate-matrix.outputs.docker-matrix) }}" + profile: [docker, singularity] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Install wave-cli run: | - wave --conda-file "${{ matrix.files }}" \ + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 + sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + chmod +x /usr/local/bin/wave + + - name: Create a registry name + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: registry-name + with: + result-encoding: string + script: | + return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/Dockerfile', '').replace('/', '_'); + + - name: Build ${{ matrix.profile }} container + # NOTE If you're changing a Dockerfile and this is running, try to update the Dockerfile to build with wave + continue-on-error: false + env: + PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} + run: | + wave -f "${{ matrix.files }}" \ + $PROFILE \ --freeze \ --await \ + --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \ + --cache-repository quay.io/nf-core/wave-cache \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ - --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ - --singularity - - # TODO Build from Dockerfiles + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} # bump-versions: # needs: generate-matrix diff --git a/modules/nf-core/quartonotebook/Dockerfile b/modules/nf-core/quartonotebook/Dockerfile index 78d2ab20c23..26624b7ea6d 100644 --- a/modules/nf-core/quartonotebook/Dockerfile +++ b/modules/nf-core/quartonotebook/Dockerfile @@ -1,7 +1,7 @@ # # First stage: Quarto installation # -FROM ubuntu:20.04 as quarto +FROM ubuntu:24.04 as quarto ARG QUARTO_VERSION=1.3.433 ARG TARGETARCH RUN apt-get update \ diff --git a/modules/nf-core/wittyer/Dockerfile b/modules/nf-core/wittyer/Dockerfile index fae697ab9dc..fbf732f32bd 100644 --- a/modules/nf-core/wittyer/Dockerfile +++ b/modules/nf-core/wittyer/Dockerfile @@ -1,4 +1,5 @@ FROM mcr.microsoft.com/dotnet/sdk:6.0@sha256:6df1177e48b55272316d08f19cb383483af82aca5cdc67a76c414bc200847624 as builder + WORKDIR /src COPY . /src RUN cd Wittyer \ @@ -6,6 +7,7 @@ RUN cd Wittyer \ && chmod +x /output/Wittyer FROM mcr.microsoft.com/dotnet/runtime:6.0@sha256:437cda84bdce26ce074d88b63abeec567c7226d73e8b911605077468e1d5c8d5 + LABEL git_repository=https://git.illumina.com/DASTE/Ilmn.Das.App.Wittyer.git WORKDIR /opt/Wittyer RUN apt-get -y update && apt-get -y install tabix libunwind8 openssl procps diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml deleted file mode 100644 index c90cb8dccbf..00000000000 --- a/modules/nf-core/wittyer/environment.yml +++ /dev/null @@ -1,5 +0,0 @@ -channels: - - conda-forge - - bioconda -dependencies: - - tabix