From da19f5ea33a45f308d93f1357d4a44cc891adb49 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Mon, 6 May 2024 07:27:36 +0000 Subject: [PATCH 01/12] remove 'output' default --- conf/defaults.config | 2 +- nextflow_schema.json | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/defaults.config b/conf/defaults.config index 694f066..3680e59 100644 --- a/conf/defaults.config +++ b/conf/defaults.config @@ -23,7 +23,7 @@ params { // Sets output directory - output = "output" + output = null // inputs from SRA. A file containing on SRA ID per line. sra_ids = null diff --git a/nextflow_schema.json b/nextflow_schema.json index ff2ca34..2c902a1 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -13,7 +13,6 @@ "properties": { "output": { "type": "string", - "default": "output", "help_text": "Where to place results?", "fa_icon": "fas fa-folder" }, From 6489f87b15027de95940171fd3d708a66448e22e Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Mon, 6 May 2024 07:38:23 +0000 Subject: [PATCH 02/12] fixed loading order --- nextflow.config | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/nextflow.config b/nextflow.config index f0c9def..11eacad 100644 --- a/nextflow.config +++ b/nextflow.config @@ -37,6 +37,22 @@ try { System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") } +/* + Setting up NF profiles + To use different profiles and executors + please read more at: https://www.nextflow.io/docs/latest/config.html#config-profiles + */ +profiles { + + // load profiles + standard { includeConfig 'conf/standard.config' } + conda { includeConfig 'conf/conda.config' } + docker { includeConfig 'conf/docker.config' } + singularity { includeConfig 'conf/singularity.config' } + test { includeConfig 'conf/test.config' } + +} + /* @@ -66,22 +82,6 @@ report { file = "${params.output}/pipeline_info/ngs_preprocess_report_${trace_timestamp}.html" } -/* - Setting up NF profiles - To use different profiles and executors - please read more at: https://www.nextflow.io/docs/latest/config.html#config-profiles - */ -profiles { - - // load profiles - standard { includeConfig 'conf/standard.config' } - conda { includeConfig 'conf/conda.config' } - docker { includeConfig 'conf/docker.config' } - singularity { includeConfig 'conf/singularity.config' } - test { includeConfig 'conf/test.config' } - -} - /* Adding manifest */ From 86c48213b57564ea0046231dbfd644d7b0fb4f2e Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Mon, 6 May 2024 08:09:31 +0000 Subject: [PATCH 03/12] add nf-test --- .gitignore | 2 ++ nf-test.config | 8 +++++++ tests/main.nf.test | 39 ++++++++++++++++++++++++++++++++++ tests/main.nf.test.snap | 46 +++++++++++++++++++++++++++++++++++++++++ tests/nextflow.config | 5 +++++ 5 files changed, 100 insertions(+) create mode 100644 nf-test.config create mode 100644 tests/main.nf.test create mode 100644 tests/main.nf.test.snap create mode 100644 tests/nextflow.config diff --git a/.gitignore b/.gitignore index 3d34302..2e51a56 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ teste testing .vscode/settings.json docs/_build +*.log +.nf-test/ \ No newline at end of file diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 0000000..b3eb27a --- /dev/null +++ b/nf-test.config @@ -0,0 +1,8 @@ +config { + + testsDir "tests" + workDir ".nf-test" + configFile "tests/nextflow.config" + profile "docker" + +} diff --git a/tests/main.nf.test b/tests/main.nf.test new file mode 100644 index 0000000..1e445ec --- /dev/null +++ b/tests/main.nf.test @@ -0,0 +1,39 @@ +nextflow_pipeline { + + name "Test Workflow main.nf" + script "main.nf" + + test("Testing workflow - Should run without fails") { + + when { + params { + sra_ids = 'https://github.com/fmalmeida/test_datasets/raw/main/sra_ids.txt' + output = "${outputDir}/test_output" + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + } + } + + then { + + assertAll( + // All good? + {assert workflow.success}, + + // All there? + {assert path("${outputDir}/test_output").list().size() == 5}, + {assert path("${outputDir}/test_output/preprocessing_outputs").list().size() == 3}, + + // Check if files are the same + {assert snapshot( + workflow, + path( "${outputDir}/test_output/final_output" ) + ).match()} + ) + + } + + } + +} diff --git a/tests/main.nf.test.snap b/tests/main.nf.test.snap new file mode 100644 index 0000000..e595911 --- /dev/null +++ b/tests/main.nf.test.snap @@ -0,0 +1,46 @@ +{ + "Testing workflow - Should run without fails": { + "content": [ + { + "stderr": [ + + ], + "errorReport": "", + "exitStatus": 0, + "failed": false, + "stdout": [ + "", + "Pipeline completed at: 2024-05-06T07:56:14.928664372Z", + "Execution status: OK", + "Execution duration: 9m 50s", + "Thank you for using fmalmeida/ngs-preprocess pipeline!" + ], + "errorMessage": "", + "trace": { + "tasksFailed": 0, + "tasksCount": 12, + "tasksSucceeded": 12 + }, + "name": "workflow", + "success": true + }, + [ + [ + "SRR19901041.filtered.fq.gz:md5,1d7a8e18db3b1f7ac0bfc1f5cf1daddf" + ], + [ + "ERR7339227.filtered.fq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + [ + "ERR9850226_R1.preprocessed.fq.gz:md5,69f8bd550fbf99d81f4a110734f4803a", + "ERR9850226_R2.preprocessed.fq.gz:md5,7543436f1614ae814337a9b123d26376" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-05-06T07:56:15.500768292" + } +} \ No newline at end of file diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 0000000..c19b1ad --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,5 @@ +/* +======================================================================================== + Nextflow config file for running tests +======================================================================================== +*/ From bff8abe173584d2cc0335c2c25ea8139994ea1b2 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Mon, 6 May 2024 08:12:41 +0000 Subject: [PATCH 04/12] add nf-test CI --- .github/workflows/nftest.yml | 78 ++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/nftest.yml diff --git a/.github/workflows/nftest.yml b/.github/workflows/nftest.yml new file mode 100644 index 0000000..7cd4723 --- /dev/null +++ b/.github/workflows/nftest.yml @@ -0,0 +1,78 @@ +# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +name: nf-test CI +on: + push: + branches: + - dev + pull_request: + release: + types: [published] + merge_group: + types: + - checks_requested + branches: + - master + - dev + +env: + NXF_ANSI_LOG: false + NFTEST_VER: "0.8.1" + +concurrency: + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" + cancel-in-progress: true + +jobs: + test: + name: "NF: ${{ matrix.NXF_VER }}" + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'fmalmeida/ngs-preprocess') }}" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + NXF_VER: + - "23.04.0" + - "latest-everything" + + steps: + - name: Disk space cleanup + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + + - name: Check out pipeline code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Cache nf-test installation + id: cache-software + uses: actions/cache@v3 + with: + path: | + /usr/local/bin/nf-test + /home/runner/.nf-test/nf-test.jar + key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest + + - name: Install nf-test + if: steps.cache-software.outputs.cache-hit != 'true' + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + + - name: Run nf-test + run: | + nf-test test tests/main.nf.test --junitxml=test.xml + + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails + with: + report_paths: test.xml \ No newline at end of file From afc54a1f926c19fddf94749e86cb0ba7fd334feb Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Wed, 8 May 2024 14:15:41 +0000 Subject: [PATCH 05/12] update nftest --- tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main.nf.test b/tests/main.nf.test index 1e445ec..42b1f9a 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -27,7 +27,7 @@ nextflow_pipeline { // Check if files are the same {assert snapshot( - workflow, + workflow.trace, path( "${outputDir}/test_output/final_output" ) ).match()} ) From 73ee6da434347ba9c22018807513048c4b1574ee Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Wed, 8 May 2024 14:15:51 +0000 Subject: [PATCH 06/12] run only with one NF version --- .github/workflows/nftest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nftest.yml b/.github/workflows/nftest.yml index 7cd4723..881978b 100644 --- a/.github/workflows/nftest.yml +++ b/.github/workflows/nftest.yml @@ -32,7 +32,6 @@ jobs: matrix: NXF_VER: - "23.04.0" - - "latest-everything" steps: - name: Disk space cleanup From 78528b386cc367b96a338e53b7a4b1ac806f5ffe Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 10 May 2024 07:26:06 +0000 Subject: [PATCH 07/12] update nf-test snap --- tests/main.nf.test.snap | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/tests/main.nf.test.snap b/tests/main.nf.test.snap index e595911..55ca990 100644 --- a/tests/main.nf.test.snap +++ b/tests/main.nf.test.snap @@ -2,27 +2,9 @@ "Testing workflow - Should run without fails": { "content": [ { - "stderr": [ - - ], - "errorReport": "", - "exitStatus": 0, - "failed": false, - "stdout": [ - "", - "Pipeline completed at: 2024-05-06T07:56:14.928664372Z", - "Execution status: OK", - "Execution duration: 9m 50s", - "Thank you for using fmalmeida/ngs-preprocess pipeline!" - ], - "errorMessage": "", - "trace": { - "tasksFailed": 0, - "tasksCount": 12, - "tasksSucceeded": 12 - }, - "name": "workflow", - "success": true + "tasksFailed": 0, + "tasksCount": 12, + "tasksSucceeded": 12 }, [ [ @@ -41,6 +23,6 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-05-06T07:56:15.500768292" + "timestamp": "2024-05-08T14:26:25.758021833" } } \ No newline at end of file From 691c6ab19a49c2cf62a39276bda17f785a97f710 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 10 May 2024 14:41:35 +0000 Subject: [PATCH 08/12] fix module for downloading PACBIO data --- modules/get_fastq.nf | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/get_fastq.nf b/modules/get_fastq.nf index 8cbb2d1..82d891f 100644 --- a/modules/get_fastq.nf +++ b/modules/get_fastq.nf @@ -15,7 +15,6 @@ process GET_FASTQ { def sra_ids = "${sra_ids.replaceAll(~/\s/,'')}" """ fasterq-dump \\ - --include-technical \\ --split-files \\ --threads $task.cpus \\ --outdir ./${sra_ids}_data \\ @@ -26,12 +25,25 @@ process GET_FASTQ { if [ \$(grep -ic "is PACBIO, please use fastq-dump instead" fasterq-dump.err) -eq 1 ] then fastq-dump \\ - --split-files \\ + --gzip \\ --outdir ./${sra_ids}_data \\ $sra_ids else echo "fasterq-dump error was:" cat fasterq-dump.err fi + + # make sure they have right extension + for i in \$( find ./${sra_ids}_data -name "*.fq" ) ; do + mv \$i \${i%%.fq}.fastq ; + done + for i in \$( find ./${sra_ids}_data -name "*.fq.gz" ) ; do + mv \$i \${i%%.fq.gz}.fastq.gz ; + done + + # make sure data is compressed + for i in \$( find ./${sra_ids}_data -name "*.fastq" ) ; do + gzip \$i ; + done """ } From 7dcb92ed4da6b81d8ddd7521182108f72a814cd8 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 10 May 2024 14:50:30 +0000 Subject: [PATCH 09/12] update version and changelog --- CHANGELOG.md | 5 +++++ nextflow.config | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19f54a0..09d3ee9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ The tracking for changes started in v2.2 +## v2.7.1 + +* [[#36]](https://github.com/fmalmeida/ngs-preprocess/issues/36) - Include nf-tests to the pipeline. +* [[#40]](https://github.com/fmalmeida/ngs-preprocess/issues/40) - Fix a problem when downloading PACBIO data. + ## v2.7.0 -- [2024-Apr-30] * [[#34]](https://github.com/fmalmeida/ngs-preprocess/issues/34) - Included a new tool in the pipeline (`porechop ABI`). Removed `tracedir` parameter. Build new docker image. diff --git a/nextflow.config b/nextflow.config index 11eacad..4da176c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -92,5 +92,5 @@ manifest { homePage = "https://github.com/fmalmeida/ngs-preprocess" mainScript = "main.nf" nextflowVersion = ">=21.10.0" - version = "2.7.0" + version = "2.7.1" } \ No newline at end of file From 3b75c0082d1ab7842c03df39a1c57b2f89ce57f5 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Fri, 10 May 2024 15:46:39 +0000 Subject: [PATCH 10/12] update tests --- tests/main.nf.test.snap | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/main.nf.test.snap b/tests/main.nf.test.snap index 55ca990..1a0fc7d 100644 --- a/tests/main.nf.test.snap +++ b/tests/main.nf.test.snap @@ -3,15 +3,16 @@ "content": [ { "tasksFailed": 0, - "tasksCount": 12, - "tasksSucceeded": 12 + "tasksCount": 16, + "tasksSucceeded": 16 }, [ [ "SRR19901041.filtered.fq.gz:md5,1d7a8e18db3b1f7ac0bfc1f5cf1daddf" ], [ - "ERR7339227.filtered.fq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + "ERR7339227.filtered.fq.gz:md5,d41d8cd98f00b204e9800998ecf8427e", + "SRR9641620.filtered.fq.gz:md5,f6658663be49766459b123efa690cc42" ], [ "ERR9850226_R1.preprocessed.fq.gz:md5,69f8bd550fbf99d81f4a110734f4803a", @@ -23,6 +24,6 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-05-08T14:26:25.758021833" + "timestamp": "2024-05-10T15:44:14.430345179" } } \ No newline at end of file From f9fe16e807d710a116fbfd7a0c8bc32e781e09bc Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Thu, 20 Jun 2024 16:42:04 +0000 Subject: [PATCH 11/12] allow platform from SRA: 'dnbseq' --- CHANGELOG.md | 4 ++++ main.nf | 2 +- nextflow.config | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d3ee9..a85dd5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ The tracking for changes started in v2.2 +# v2.7.2 + +* Adjust short-reads workflow filter to allow analysis from SRA for platform key-values: illumina, bgiseq and dnbseq + ## v2.7.1 * [[#36]](https://github.com/fmalmeida/ngs-preprocess/issues/36) - Include nf-tests to the pipeline. diff --git a/main.nf b/main.nf index f33e644..c879698 100644 --- a/main.nf +++ b/main.nf @@ -156,7 +156,7 @@ workflow { .filter{ it != '' } .mix( SRA_FETCH.out.fastqs - .filter{ it[1] =~ /illumina|bgiseq/ } + .filter{ it[1] =~ /illumina|bgiseq|dnbseq/ } .map{ def meta = [:] meta.id = it[0] diff --git a/nextflow.config b/nextflow.config index 4da176c..f4e0493 100644 --- a/nextflow.config +++ b/nextflow.config @@ -92,5 +92,5 @@ manifest { homePage = "https://github.com/fmalmeida/ngs-preprocess" mainScript = "main.nf" nextflowVersion = ">=21.10.0" - version = "2.7.1" + version = "2.7.2" } \ No newline at end of file From 7d732a829cd79c8ccd3e5a064efb1cf210cac5e1 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Thu, 20 Jun 2024 16:47:06 +0000 Subject: [PATCH 12/12] fix versions --- CHANGELOG.md | 5 +---- nextflow.config | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a85dd5b..88778b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,11 @@ The tracking for changes started in v2.2 -# v2.7.2 - -* Adjust short-reads workflow filter to allow analysis from SRA for platform key-values: illumina, bgiseq and dnbseq - ## v2.7.1 * [[#36]](https://github.com/fmalmeida/ngs-preprocess/issues/36) - Include nf-tests to the pipeline. * [[#40]](https://github.com/fmalmeida/ngs-preprocess/issues/40) - Fix a problem when downloading PACBIO data. +* Adjust short-reads workflow filter to allow analysis from SRA for platform key-values: illumina, bgiseq and dnbseq ## v2.7.0 -- [2024-Apr-30] diff --git a/nextflow.config b/nextflow.config index f4e0493..4da176c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -92,5 +92,5 @@ manifest { homePage = "https://github.com/fmalmeida/ngs-preprocess" mainScript = "main.nf" nextflowVersion = ">=21.10.0" - version = "2.7.2" + version = "2.7.1" } \ No newline at end of file