+
+
+### Pipeline information
+
+[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage.
+
+Output files
+ +- `multiqc/` + - `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser. + - `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline. + - `multiqc_plots/`: directory containing static images from the report in various formats. + - summary_assembly_metrics_mqc.csv: custom table containing most relevant assembly QC metrics. + +
+
\ No newline at end of file
diff --git a/bu_isciii/assets/reports/md/images/ALM-ASM.png b/bu_isciii/assets/reports/md/images/ALM-ASM.png
new file mode 100644
index 00000000..ba29587c
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/ALM-ASM.png differ
diff --git a/bu_isciii/assets/reports/md/images/IRMA_workflow.png b/bu_isciii/assets/reports/md/images/IRMA_workflow.png
new file mode 100644
index 00000000..b68fd5d8
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/IRMA_workflow.png differ
diff --git a/bu_isciii/assets/reports/md/images/KPN30_000240185_summary.png b/bu_isciii/assets/reports/md/images/KPN30_000240185_summary.png
new file mode 100644
index 00000000..e49bc9fc
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/KPN30_000240185_summary.png differ
diff --git a/bu_isciii/assets/reports/md/images/NIPH-NIPHEM.png b/bu_isciii/assets/reports/md/images/NIPH-NIPHEM.png
new file mode 100644
index 00000000..a22457d4
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/NIPH-NIPHEM.png differ
diff --git a/bu_isciii/assets/reports/md/images/PLOT.png b/bu_isciii/assets/reports/md/images/PLOT.png
new file mode 100644
index 00000000..33178cb9
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/PLOT.png differ
diff --git a/bu_isciii/assets/reports/md/images/SEN30_000195995_NC_013365.1.png b/bu_isciii/assets/reports/md/images/SEN30_000195995_NC_013365.1.png
new file mode 100644
index 00000000..86a24bd2
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/SEN30_000195995_NC_013365.1.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_boxplot.png b/bu_isciii/assets/reports/md/images/deseq2_boxplot.png
new file mode 100644
index 00000000..ca705948
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_boxplot.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_cluster_dendogram.png b/bu_isciii/assets/reports/md/images/deseq2_cluster_dendogram.png
new file mode 100644
index 00000000..2916aee1
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_cluster_dendogram.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_dispersion-estimate.png b/bu_isciii/assets/reports/md/images/deseq2_dispersion-estimate.png
new file mode 100644
index 00000000..8927bcf9
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_dispersion-estimate.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_heatmap-top-20-genes.png b/bu_isciii/assets/reports/md/images/deseq2_heatmap-top-20-genes.png
new file mode 100644
index 00000000..8aa61ec8
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_heatmap-top-20-genes.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_heatmap_all.png b/bu_isciii/assets/reports/md/images/deseq2_heatmap_all.png
new file mode 100644
index 00000000..476d6c0b
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_heatmap_all.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_maplot.png b/bu_isciii/assets/reports/md/images/deseq2_maplot.png
new file mode 100644
index 00000000..c1ebdea0
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_maplot.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_pca.png b/bu_isciii/assets/reports/md/images/deseq2_pca.png
new file mode 100644
index 00000000..3e31268b
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_pca.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_plotSD.png b/bu_isciii/assets/reports/md/images/deseq2_plotSD.png
new file mode 100644
index 00000000..67733bb3
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_plotSD.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_pvalue-hist.png b/bu_isciii/assets/reports/md/images/deseq2_pvalue-hist.png
new file mode 100644
index 00000000..830b5e54
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_pvalue-hist.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_qc_plots.png b/bu_isciii/assets/reports/md/images/deseq2_qc_plots.png
new file mode 100755
index 00000000..bd19f1fd
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_qc_plots.png differ
diff --git a/bu_isciii/assets/reports/md/images/deseq2_sample-to-sample.png b/bu_isciii/assets/reports/md/images/deseq2_sample-to-sample.png
new file mode 100644
index 00000000..98e05c8c
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/deseq2_sample-to-sample.png differ
diff --git a/bu_isciii/assets/reports/md/images/dupradar_example_plot.png b/bu_isciii/assets/reports/md/images/dupradar_example_plot.png
new file mode 100644
index 00000000..72db5d45
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/dupradar_example_plot.png differ
diff --git a/bu_isciii/assets/reports/md/images/exomiser-html-description-1.png b/bu_isciii/assets/reports/md/images/exomiser-html-description-1.png
new file mode 100755
index 00000000..6e60a97b
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/exomiser-html-description-1.png differ
diff --git a/bu_isciii/assets/reports/md/images/exomiser-html-description-2.png b/bu_isciii/assets/reports/md/images/exomiser-html-description-2.png
new file mode 100755
index 00000000..a43a5693
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/exomiser-html-description-2.png differ
diff --git a/bu_isciii/assets/reports/md/images/fastqc.png b/bu_isciii/assets/reports/md/images/fastqc.png
new file mode 100644
index 00000000..8abb95f2
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/fastqc.png differ
diff --git a/bu_isciii/assets/reports/md/images/icarus.png b/bu_isciii/assets/reports/md/images/icarus.png
new file mode 100644
index 00000000..3396e239
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/icarus.png differ
diff --git a/bu_isciii/assets/reports/md/images/kraken2.png b/bu_isciii/assets/reports/md/images/kraken2.png
new file mode 100644
index 00000000..0f0802d4
Binary files /dev/null and b/bu_isciii/assets/reports/md/images/kraken2.png differ
diff --git a/bu_isciii/assets/reports/md/images/mag_workflow.png b/bu_isciii/assets/reports/md/images/mag_workflow.png
index 69c93297..d4cda1a0 100755
Binary files a/bu_isciii/assets/reports/md/images/mag_workflow.png and b/bu_isciii/assets/reports/md/images/mag_workflow.png differ
diff --git a/bu_isciii/assets/reports/md/images/mag_workflow.svg b/bu_isciii/assets/reports/md/images/mag_workflow.svg
index 7f78e0fc..293354db 100755
--- a/bu_isciii/assets/reports/md/images/mag_workflow.svg
+++ b/bu_isciii/assets/reports/md/images/mag_workflow.svg
@@ -1,11 +1,11 @@
Output files
+ +- `pipeline_info/` + - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. + - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. + - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. + - Parameters used by the pipeline run: `params.json`. + +
+
+
+
+
+## FASTP
+
+[Fastp](https://github.com/OpenGene/fastp?tab=readme-ov-file#fastp) is a tool designed to provide fast, all-in-one preprocessing for FastQ files. It has been developed in C++ with multithreading support to achieve higher performance. This tools is used to pre-process the reads using multiple filters. For this specific workflow it is used for quality trimming and to filter any short-reads, adapters, polyG and polyX. This software includes in its output an html with several stats that show the state of the reads before and after processing.
+
+Output files
+ +- `fastqc/raw/` + - `*_fastqc.html`: FastQC report containing quality metrics. + - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. + +**NB:** The FastQC plots in this directory are generated relative to the raw, input reads. They may contain adapter sequence and regions of low quality. To see how your reads look after trimming please refer to the FastQC reports in the `fastqc/trim/` directory. + +
+
+
+
+
+## IRMA workflow
+
+[IRMA (Iterative Refinement Meta-Assembler)](https://wonder.cdc.gov/amd/flu/irma/) was designed for the robust assembly, variant calling, and phasing of highly variable RNA viruses. IRMA is deployed for this service to analyse Influenza virus. IRMA is free to use and parallelizes computations for both cluster computing and single computer multi-core setups. You can read the [IRMA manuscript](https://bmcgenomics.biomedcentral.com/articles/10.1186/s12864-016-3030-6#Sec9) for more background on the methodology.
+
+
+
+**IRMA** uses several auxiliar softwares in its workflow:
+
+- [BLAT](http://www.kentinformatics.com/products.html) for the match step
+- [LABEL](https://wonder.cdc.gov/amd/flu/label), which also packages certain resources used by IRMA:
+ - [Sequence Alignment and Modeling System (SAM)](http://www.ncbi.nlm.nih.gov/pubmed/9927713) for both the rough align and sort steps
+ - [Shogun Toolbox](https://github.com/shogun-toolbox/shogun), which is an essential part of LABEL, is used in the sort step.
+- [SSW](http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0082138) for the final assembly step.
+- [MINIMAP2](https://academic.oup.com/bioinformatics/article/34/18/3094/4994778) for the final assembly step (alternate option that may be useful for long - read assemblies).
+- [samtools](http://samtools.sourceforge.net/) for BAM-SAM conversion as well as BAM sorting and indexing
+- [GNU Parallel](http://www.gnu.org/software/parallel/) for single node parallelization
+
+### IRMA output directory structure
+
+**_gene\_segment_** in the case of influenza can be any of the following: [HA, NA, NS, MP, NP, PA, PB1, PB2] and HE for Influenza-C
+
+### **Final files:**
+
+- **_gene\_segment_.bam** Sorted BAM file for the final gene_segment assembly (merged if applicable)
+- **_gene\_segment_.bam.bai** BAM file index for gene_segment assembly
+- **_gene\_segment_.fasta** Final assembled plurality consensus (no mixed base calls) for gene_segment
+- **_gene\_segment_.a2m** Optional file: Plurality consensus aligned to profile HMM
+- **_gene\_segment_.vcf** Custom variant call file for called IRMA variants for each gene segment
+
+### **Folders:**
+
+**amended_consensus/**: Assembled consensus per gene segment with missed based calls. Numbers correspond to fragments in $SEG_NUMBERS
+- **_sample\_name_\__fragment\_number_.fa** Amended consensus
+- **_sample\_name_\_7.a2m** Optional output: amended global alignment to profile HMM
+- **_sample\_name_\_7.pad.fa** Optional output: N-padded consensus for amplicon dropouts
+
+**figures/**
+
+ - **_gene\_segment_-coverageDiagram.pdf** Shows coverage and variant calls
+ - **_gene\_segment_-heuristics.pdf** Heuristic graphs for gene segment
+ - **_gene\_segment_-EXPENRD.pdf** gene segment variant phasing using experimental enrichment distances
+ - **_gene\_segment_-JACCARD.pdf** gene segment variant phasing using modified Jaccard distances
+ - **_gene\_segment_-MUTUALD.pdf** gene segment variant phasing using mutual association distances
+ - **_gene\_segment_-NJOINTP.pdf** gene segment variant phasing using normalized joint probability distances
+ - **READ_PERCENTAGES.pdf** Break down for reads assembled
+
+**intermediate/** Intermediate data for each step
+
+ - **0-ITERATIVE-REFERENCES/**
+ - **R0-_gene\_segment_.ref** Starting reference library sequence for segment
+ - **R1-_gene\_segment_.ref** Gene segment working reference after round 1,template for round 2
+ - **R2-_gene\_segment_.ref** Working reference for gene segment after round 2
+ - **1-MATCH_BLAT/**
+ - **R1.tar.gz**
+ - **R2.tar.gz** Archive of BLAT results for the MATCH step
+ - **R3.tar.gz**
+ - **2-SORT_BLAT/**
+ - **R1.tar.gz** Classification/sorting intermediate files for round 1
+ - **R1.txt** Summary statistics of sorting results for round 1
+ - **R2.tar.gz** Classification/sorting intermediate files for round 2
+ - **R2.txt** Summary statistics of sorting results for round 2
+ - **3-ALIGN_SAM/BLAT/**
+ - **storedCounts.tar.gz** Statistic files used to create rough consensus sequences
+ - **4-ASSEMBLE_SSW/** Final assembly phase intermediate iterativeresuls
+ - **F1-_gene\_segment_.bam** Unsorted BAM file for gene segment assembly iteration 1
+ - **F1-_gene\_segment_.ref** Reference for final assembly, gene segment, iteration 1
+ - **F2-_gene\_segment_.bam** Unsorted BAM file for gene segment assembly, iteration 2
+ - **F2-_gene\_segment_.ref** Reference for final assembly, gene segment, iteration 2
+ - **reads.tar.gz** Archive of sorted, unmerged reads by gene segment
+
+**logs/**
+
+ - **ASSEMBLY_log.txt** SSw scores per all rounds tried in the iterative refinement
+ - **NR_COUNTS_log.txt** Read pattern counts at various stages
+ - **QC_log.txt** Quality control output
+ - **READ_log.txt** Counts of assembled reads from BAM files
+ - **FLU-_sample\_name_.sh** Configuration file corresponding to this IRMA run
+ - **run_info.txt** Table of parameters used by the IRMA run
+
+**matrices/** Phasing matrices used to generate heat maps
+
+ - **_gene\_segment_-EXPENRD.sqm**
+ - **_gene\_segment_-JACCARD.sqm**
+ - **_gene\_segment_-MUTUALD.sqm**
+ - **_gene\_segment_-NJOINTP.sqm**
+
+**secondary/**
+
+ - **R1-A_NA_N1.fa** Trace A_NA_N1 sorted into secondary status
+ - **R1-UNRECOGNIZABLE.fa** Read patterns that matched flu but had poor signal according to LABEL
+ - **R2-UNRECOGNIZABLE.fa**
+ - **unmatched_read_patterns.tar.gz** Archive of left over read patterns that did not match FLU
+
+**tables/**
+ - **_gene\_segment_-pairingStats.txt** Summary of paired-end merging statistics, if applicable, gene segment
+ - **_gene\_segment_-coverage.txt** Summary coverage statistics for assembly, gene segment
+ - **_gene\_segment_-coverage.a2m.txt** Optional file: Coverage statistics for plurality consensus globally aligned to profile HMM
+ - **_gene\_segment_-coverage.pad.txt** Optional file: Coverage statistics for padded plurality consensus globally aligned to profile HMM
+ - **_gene\_segment_-allAlleles.txt** Statistics for every position & allele in the assembly, gene segment
+ - **_gene\_segment_-insertions.txt** Called insertion variants for gene segment
+ - **_gene\_segment_-deletions.txt** Called deletion variants for gene segment
+ - **_gene\_segment_-variants.txt** Called single nucleotide variants for gene segment
+ - **READ_COUNTS.txt** Read counts for various points in the assembly process
\ No newline at end of file
diff --git a/bu_isciii/assets/reports/md/mag.md b/bu_isciii/assets/reports/md/mag.md
index 644efb1d..67ef902a 100644
--- a/bu_isciii/assets/reports/md/mag.md
+++ b/bu_isciii/assets/reports/md/mag.md
@@ -10,16 +10,19 @@ The directories listed below will be created in the results directory after the
The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps:
-* [Quality control](#quality-control) of input reads - trimming and contaminant removal
-* [Taxonomic classification of trimmed reads](#taxonomic-classification-of-trimmed-reads)
-* [Assembly](#assembly) of trimmed reads
-* [Protein-coding gene prediction](#gene-prediction) of assemblies
-* [Binning](#binning) of assembled contigs
-* [Taxonomic classification of binned genomes](#taxonomic-classification-of-binned-genomes)
-* [Genome annotation of binned genomes](#genome-annotation-of-binned-genomes)
-* [Additional summary for binned genomes](#additional-summary-for-binned-genomes)
-* [MultiQC](#multiqc) - aggregate report, describing results of the whole pipeline
-* [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution
+- [Quality control](#quality-control) of input reads - trimming and contaminant removal
+- [Taxonomic classification of trimmed reads](#taxonomic-classification-of-trimmed-reads)
+- [Digital sequencing normalisation](#digital-normalization-with-BBnorm)
+- [Assembly](#assembly) of trimmed reads
+- [Protein-coding gene prediction](#gene-prediction) of assemblies
+- [Virus identification](#virus-identification-in-assemblies) of assemblies
+- [Binning and binning refinement](#binning-and-binning-refinement) of assembled contigs
+- [Taxonomic classification of binned genomes](#taxonomic-classification-of-binned-genomes)
+- [Genome annotation of binned genomes](#genome-annotation-of-binned-genomes)
+- [Additional summary for binned genomes](#additional-summary-for-binned-genomes)
+- [Ancient DNA](#ancient-dna)
+- [MultiQC](#multiqc) - aggregate report, describing results of the whole pipeline
+- [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution
Note that when specifying the parameter `--coassemble_group`, for the corresponding output filenames/directories of the assembly or downsteam processes the group ID, or more precisely the term `group-[group_id]`, will be used instead of the sample ID.
@@ -36,9 +39,9 @@ FastQC is run for visualising the general quality metrics of the sequencing runs
Output files
+ +- `fastp/` + - `*.fastp.html`: Trimming report in html format. + - `*.fastp.json`: Trimming report in json format. +- `fastp/log/` + - `*.fastp.log`: Trimming log file. +- `fastqc/trim/` + - `*_fastqc.html`: FastQC report of the trimmed reads. + - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. + +Output files
-* `QC_shortreads/fastqc/` - * `[sample]_[1/2]_fastqc.html`: FastQC report, containing quality metrics for your untrimmed raw fastq files - * `[sample].trimmed_[1/2]_fastqc.html`: FastQC report, containing quality metrics for trimmed and, if specified, filtered read files +- `QC_shortreads/fastqc/` + - `[sample]_[1/2]_fastqc.html`: FastQC report, containing quality metrics for your untrimmed raw fastq files + - `[sample].trimmed_[1/2]_fastqc.html`: FastQC report, containing quality metrics for trimmed and, if specified, filtered read filesOutput files
-* `QC_shortreads/fastp/[sample]/` - * `fastp.html`: Interactive report - * `fastp.json`: Report in json format +- `QC_shortreads/fastp/[sample]/` + - `fastp.html`: Interactive report + - `fastp.json`: Report in json format + +
+
@@ -64,8 +79,8 @@ The pipeline uses bowtie2 to map the reads against PhiX and removes mapped reads
Output files
+ +- `QC_shortreads/adapterremoval/[sample]/` + - `[sample]_ar2.settings`: AdapterRemoval log file.Output files
-* `QC_shortreads/remove_phix/` - * `[sample].phix_removed.bowtie2.log`: Contains a brief log file indicating how many reads have been retained. +- `QC_shortreads/remove_phix/` + - `[sample].phix_removed.bowtie2.log`: Contains a brief log file indicating how many reads have been retained.Output files
-* `QC_shortreads/remove_host/` - * `[sample].host_removed.bowtie2.log`: Contains the bowtie2 log file indicating how many reads have been mapped as well as a file listing the read ids of discarded reads. +- `QC_shortreads/remove_host/` + - `[sample].host_removed.bowtie2.log`: Contains the bowtie2 log file indicating how many reads have been mapped. + - `[sample].host_removed.mapped*.read_ids.txt`: Contains a file listing the read ids of discarded reads.Output files
-* `QC_longreads/NanoLyse/` - * `[sample]_nanolyse.log`: Contains a brief log file indicating how many reads have been retained. +- `QC_longreads/NanoLyse/` + - `[sample]_nanolyse.log`: Contains a brief log file indicating how many reads have been retained.Output files
-* `QC_longreads/NanoPlot/[sample]/` - * `raw_*.[png/html/txt]`: Plots and reports for raw data - * `filtered_*.[png/html/txt]`: Plots and reports for filtered data +- `QC_longreads/NanoPlot/[sample]/` + - `raw_*.[png/html/txt]`: Plots and reports for raw data + - `filtered_*.[png/html/txt]`: Plots and reports for filtered data + +
+
@@ -124,9 +154,9 @@ Kraken2 classifies reads using a k-mer based approach as well as assigns taxonom
Output files
+ +- `bbmap/bbnorm/[sample]\*.fastq.gz` +- `bbmap/bbnorm/log/[sample].bbnorm.log`Output files
-* `Taxonomy/kraken2/[sample]/` - * `kraken2.report`: Classification in the Kraken report format. See the [kraken2 manual](https://github.com/DerrickWood/kraken2/wiki/Manual#output-formats) for more details - * `taxonomy.krona.html`: Interactive pie chart produced by [KronaTools](https://github.com/marbl/Krona/wiki) +- `Taxonomy/kraken2/[sample]/` + - `kraken2.report`: Classification in the Kraken report format. See the [kraken2 manual](https://github.com/DerrickWood/kraken2/wiki/Manual#output-formats) for more details + - `taxonomy.krona.html`: Interactive pie chart produced by [KronaTools](https://github.com/marbl/Krona/wiki)Output files
-* `Taxonomy/centrifuge/[sample]/` - * `report.txt`: Tab-delimited result file. See the [centrifuge manual](https://ccb.jhu.edu/software/centrifuge/manual.shtml#centrifuge-classification-output) for information about the fields - * `kreport.txt`: Classification in the Kraken report format. See the [kraken2 manual](https://github.com/DerrickWood/kraken2/wiki/Manual#output-formats) for more details - * `taxonomy.krona.html`: Interactive pie chart produced by [KronaTools](https://github.com/marbl/Krona/wiki) +- `Taxonomy/centrifuge/[sample]/` + - `report.txt`: Tab-delimited result file. See the [centrifuge manual](https://ccb.jhu.edu/software/centrifuge/manual.shtml#centrifuge-classification-output) for information about the fields + - `kreport.txt`: Classification in the Kraken report format. See the [kraken2 manual](https://github.com/DerrickWood/kraken2/wiki/Manual#output-formats) for more details + - `taxonomy.krona.html`: Interactive pie chart produced by [KronaTools](https://github.com/marbl/Krona/wiki)Output files
-* `Assembly/MEGAHIT/` - * `[sample/group].contigs.fa.gz`: Compressed metagenome assembly in fasta format - * `[sample/group].log`: Log file - * `QC/[sample/group]/`: Directory containing QUAST files and Bowtie2 mapping logs - * `MEGAHIT-[sample].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the sample that the metagenome was assembled from, only present if `--coassemble_group` is not set. - * `MEGAHIT-[sample/group]-[sampleToMap].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the respective sample ("sampleToMap"). +- `Assembly/MEGAHIT/` + - `[sample/group].contigs.fa.gz`: Compressed metagenome assembly in fasta format + - `[sample/group].log`: Log file + - `QC/[sample/group]/`: Directory containing QUAST files and Bowtie2 mapping logs + - `MEGAHIT-[sample].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the sample that the metagenome was assembled from, only present if `--coassemble_group` is not set. + - `MEGAHIT-[sample/group]-[sampleToMap].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the respective sample ("sampleToMap"). + - `MEGAHIT-[sample].[bam/bai]`: Optionally saved BAM file of the Bowtie2 mapping of reads against the assembly.Output files
-* `Assembly/SPAdes/` - * `[sample/group]_scaffolds.fasta.gz`: Compressed assembled scaffolds in fasta format - * `[sample/group]_graph.gfa.gz`: Compressed assembly graph in gfa format - * `[sample/group]_contigs.fasta.gz`: Compressed assembled contigs in fasta format - * `[sample/group].log`: Log file - * `QC/[sample/group]/`: Directory containing QUAST files and Bowtie2 mapping logs - * `SPAdes-[sample].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the sample that the metagenome was assembled from, only present if `--coassemble_group` is not set. - * `SPAdes-[sample/group]-[sampleToMap].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the respective sample ("sampleToMap"). +- `Assembly/SPAdes/` + - `[sample/group]_scaffolds.fasta.gz`: Compressed assembled scaffolds in fasta format + - `[sample/group]_graph.gfa.gz`: Compressed assembly graph in gfa format + - `[sample/group]_contigs.fasta.gz`: Compressed assembled contigs in fasta format + - `[sample/group].log`: Log file + - `QC/[sample/group]/`: Directory containing QUAST files and Bowtie2 mapping logs + - `SPAdes-[sample].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the sample that the metagenome was assembled from, only present if `--coassemble_group` is not set. + - `SPAdes-[sample/group]-[sampleToMap].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the respective sample ("sampleToMap"). + - `SPAdes-[sample].[bam/bai]`: Optionally saved BAM file of the Bowtie2 mapping of reads against the assembly.Output files
-* `Assembly/SPAdesHybrid/` - * `[sample/group]_scaffolds.fasta.gz`: Compressed assembled scaffolds in fasta format - * `[sample/group]_graph.gfa.gz`: Compressed assembly graph in gfa format - * `[sample/group]_contigs.fasta.gz`: Compressed assembled contigs in fasta format - * `[sample/group].log`: Log file - * `QC/[sample/group]/`: Directory containing QUAST files and Bowtie2 mapping logs - * `SPAdesHybrid-[sample].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the sample that the metagenome was assembled from, only present if `--coassemble_group` is not set. - * `SPAdesHybrid-[sample/group]-[sampleToMap].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the respective sample ("sampleToMap"). +- `Assembly/SPAdesHybrid/` + - `[sample/group]_scaffolds.fasta.gz`: Compressed assembled scaffolds in fasta format + - `[sample/group]_graph.gfa.gz`: Compressed assembly graph in gfa format + - `[sample/group]_contigs.fasta.gz`: Compressed assembled contigs in fasta format + - `[sample/group].log`: Log file + - `QC/[sample/group]/`: Directory containing QUAST files and Bowtie2 mapping logs + - `SPAdesHybrid-[sample].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the sample that the metagenome was assembled from, only present if `--coassemble_group` is not set. + - `SPAdesHybrid-[sample/group]-[sampleToMap].bowtie2.log`: Bowtie2 log file indicating how many reads have been mapped from the respective sample ("sampleToMap"). + - `SPAdesHybrid-[sample].[bam/bai]`: Optionally saved BAM file of the Bowtie2 mapping of reads against the assembly.Output files
-* `Assembly/[assembler]/QC/[sample/group]/` - * `report.*`: QUAST report in various formats, such as html, txt, tsv or tex - * `quast.log`: QUAST log file - * `predicted_genes/[assembler]-[sample/group].rna.gff`: Contig positions for rRNA genes in gff version 3 format +- `Assembly/[assembler]/QC/[sample/group]/QUAST/` + - `report.*`: QUAST report in various formats, such as html, pdf, tex, tsv, or txt + - `transposed_report.*`: QUAST report that has been transposed into wide format (tex, tsv, or txt) + - `quast.log`: QUAST log file + - `metaquast.log`: MetaQUAST log file + - `icarus.html`: Icarus main menu with links to interactive viewers + - `icarus_viewers/contig_size_viewer.html`: Diagram of contigs that are ordered from longest to shortest + - `basic_stats/cumulative_plot.pdf`: Shows the growth of contig lengths (contigs are ordered from largest to shortest) + - `basic_stats/GC_content_plot.pdf`: Shows the distribution of GC content in the contigs + - `basic_stats/[assembler]-[sample/group]_GC_content_plot.pdf`: Histogram of the GC percentage for the contigs + - `basic_stats/Nx_plot.pdf`: Plot of Nx values as x varies from 0 to 100%. + - `predicted_genes/[assembler]-[sample/group].rna.gff`: Contig positions for rRNA genes in gff version 3 format + - `predicted_genes/barrnap.log`: Barrnap log file (ribosomal RNA predictor)Output files
-* `Prodigal/` - * `[sample/group].gff`: Gene Coordinates in GFF format - * `[sample/group].faa`: The protein translation file consists of all the proteins from all the sequences in multiple FASTA format. - * `[sample/group].fna`: Nucleotide sequences of the predicted proteins using the DNA alphabet, not mRNA (so you will see 'T' in the output and not 'U'). - * `[sample/group]_all.txt`: Information about start positions of genes. +- `Annotation/Prodigal/` + - `[assembler]-[sample/group].gff.gz`: Gene Coordinates in GFF format + - `[assembler]-[sample/group].faa.gz`: The protein translation file consists of all the proteins from all the sequences in multiple FASTA format. + - `[assembler]-[sample/group].fna.gz`: Nucleotide sequences of the predicted proteins using the DNA alphabet, not mRNA (so you will see 'T' in the output and not 'U'). + - `[assembler]-[sample/group]_all.txt.gz`: Information about start positions of genes. + +
+
-## Binning
+## Binning and binning refinement
### Contig sequencing depth
-Sequencing depth per contig and sample is generated by `jgi_summarize_bam_contig_depths --outputDepth`. The values correspond to `(sum of exactely aligned bases) / ((contig length)-2*75)`. For example, for two reads aligned exactly with `10` and `9` bases on a 1000 bp long contig the depth is calculated by `(10+9)/(1000-2*75)` (1000bp length of contig minus 75bp from each end, which is excluded).
+Sequencing depth per contig and sample is generated by MetaBAT2's `jgi_summarize_bam_contig_depths --outputDepth`. The values correspond to `(sum of exactly aligned bases) / ((contig length)-2*75)`. For example, for two reads aligned exactly with `10` and `9` bases on a 1000 bp long contig the depth is calculated by `(10+9)/(1000-2*75)` (1000bp length of contig minus 75bp from each end, which is excluded).
+
+These depth files are used for downstream binning steps.
Output files
+ +- `VirusIdentification/geNomad/[assembler]-[sample/group]*/` + - `[assembler]-[sample/group]*_annotate` + - `[assembler]-[sample/group]*_taxonomy.tsv`: Taxonomic assignment data + - `[assembler]-[sample/group]*_aggregated_classification` + - `[assembler]-[sample/group]*_aggregated_classification.tsv`: Sequence classification in tabular format + - `[assembler]-[sample/group]*_find_proviruses` + - `[assembler]-[sample/group]*_provirus.tsv`: Characteristics of proviruses identified by geNomad + - `[assembler]-[sample/group]*_summary` + - `[assembler]-[sample/group]*_virus_summary.tsv`: Virus classification summary file in tabular format + - `[assembler]-[sample/group]*_plasmid_summary.tsv`: Plasmid classification summary file in tabular format + - `[assembler]-[sample/group]*_viruses_genes.tsv`: Virus gene annotation data in tabular format + - `[assembler]-[sample/group]*_plasmids_genes.tsv`: Plasmid gene annotation data in tabular format + - `[assembler]-[sample/group]*_viruses.fna`: Virus nucleotide sequences in FASTA format + - `[assembler]-[sample/group]*_plasmids.fna`: Plasmid nucleotide sequences in FASTA format + - `[assembler]-[sample/group]*_viruses_proteins.faa`: Virus protein sequences in FASTA format + - `[assembler]-[sample/group]*_plasmids_proteins.faa`: Plasmid protein sequences in FASTA format + - `[assembler]-[sample/group]*.log`: Plain text log file detailing the steps executed by geNomad (annotate, find-proviruses, marker-classification, nn-classification, aggregated-classification and summary)Output files
-* `GenomeBinning/` - * `[assembler]-[sample/group]-depth.txt.gz`: Sequencing depth for each contig and sample or group, only for short reads. +- `GenomeBinning/depths/contigs/` + - `[assembler]-[sample/group]-depth.txt.gz`: Sequencing depth for each contig and sample or group, only for short reads.Output files
-* `GenomeBinning/MetaBAT2/` - * `[assembler]-[sample/group].*.fa`: Genome bins retrieved from input assembly - * `[assembler]-[sample/group].unbinned.*.fa`: Contigs that were not binned with other contigs but considered interesting. By default, these are at least 1 Mbp (`--min_length_unbinned_contigs`) in length and at most the 100 longest contigs (`--max_unbinned_contigs`) are reported +- `GenomeBinning/MetaBAT2/` + - `bins/[assembler]-[binner]-[sample/group].*.fa.gz`: Genome bins retrieved from input assembly + - `unbinned/[assembler]-[binner]-[sample/group].unbinned.[1-9]*.fa.gz`: Contigs that were not binned with other contigs but considered interesting. By default, these are at least 1 Mbp (`--min_length_unbinned_contigs`) in length and at most the 100 longest contigs (`--max_unbinned_contigs`) are reportedOutput files
-* `GenomeBinning/MetaBAT2/discarded/` - * `*.lowDepth.fa.gz`: Low depth contigs that are filtered by MetaBat2 - * `*.tooShort.fa.gz`: Too short contigs that are filtered by MetaBat2 - * `*.unbinned.pooled.fa.gz`: Pooled unbinned contigs equal or above `--min_contig_size`, by default 1500 bp. - * `*.unbinned.remaining.fa.gz`: Remaining unbinned contigs below `--min_contig_size`, by default 1500 bp, but not in any other file. +- `GenomeBinning/MetaBAT2/discarded/` + - `*.lowDepth.fa.gz`: Low depth contigs that are filtered by MetaBAT2 + - `*.tooShort.fa.gz`: Too short contigs that are filtered by MetaBAT2 +- `GenomeBinning/MetaBAT2/unbinned/discarded/` + - `*.unbinned.pooled.fa.gz`: Pooled unbinned contigs equal or above `--min_contig_size`, by default 1500 bp. + - `*.unbinned.remaining.fa.gz`: Remaining unbinned contigs below `--min_contig_size`, by default 1500 bp, but not in any other file.
+
+
+All the files and contigs in these folders will be assessed by QUAST and BUSCO.
+
+Output files
+ +- `GenomeBinning/MaxBin2/` + - `bins/[assembler]-[binner]-[sample/group].*.fa.gz`: Genome bins retrieved from input assembly + - `unbinned/[assembler]-[binner]-[sample/group].noclass.[1-9]*.fa.gz`: Contigs that were not binned with other contigs but considered interesting. By default, these are at least 1 Mbp (`--min_length_unbinned_contigs`) in length and at most the 100 longest contigs (`--max_unbinned_contigs`) are reported. + +
+
+
+All the files in this folder contain small and/or unbinned contigs that are not further processed.
+
+Files in these two folders contain all contigs of an assembly.
+
+### CONCOCT
+
+[CONCOCT](https://github.com/BinPro/CONCOCT) performs unsupervised binning of metagenomic contigs by using nucleotide composition, coverage data in multiple samples and linkage data from paired end reads.
+
+Output files
+ +- `GenomeBinning/MaxBin2/discarded/` + - `*.tooshort.gz`: Too short contigs that are filtered by MaxBin2 +- `GenomeBinning/MaxBin2/unbinned/discarded/` + - `*.noclass.pooled.fa.gz`: Pooled unbinned contigs equal or above `--min_contig_size`, by default 1500 bp. + - `*.noclass.remaining.fa.gz`: Remaining unbinned contigs below `--min_contig_size`, by default 1500 bp, but not in any other file. + +
+
+
+All the files and contigs in these folders will be assessed by QUAST and BUSCO, if the parameter `--postbinning_input` is not set to `refined_bins_only`.
+
+Note that CONCOCT does not output what it considers 'unbinned' contigs, therefore no 'discarded' contigs are produced here. You may still need to do your own manual curation of the resulting bins.
+
+### DAS Tool
+
+[DAS Tool](https://github.com/cmks/DAS_Tool) is an automated binning refinement method that integrates the results of a flexible number of binning algorithms to calculate an optimized, non-redundant set of bins from a single assembly. nf-core/mag uses this tool to attempt to further improve bins based on combining the MetaBAT2 and MaxBin2 binning output, assuming sufficient quality is met for those bins.
+
+DAS Tool will remove contigs from bins that do not pass additional filtering criteria, and will discard redundant lower-quality output from binners that represent the same estimated 'organism', until the single highest quality bin is represented.
+
+> ⚠️ If DAS Tool does not find any bins passing your selected threshold it will exit with an error. Such an error is 'ignored' by nf-core/mag, therefore you will not find files in the `GenomeBinning/DASTool/` results directory for that particular sample.
+
+Output files
+ +- `GenomeBinning/CONCOCT/` + - `bins/[assembler]-[binner]-[sample/group].*.fa.gz`: Genome bins retrieved from input assembly + - `stats/[assembler]-[binner]-[sample/group].csv`: Table indicating which contig goes with which cluster bin. + - `stats/[assembler]-[binner]-[sample/group]*_gt1000.csv`: Various intermediate PCA statistics used for clustering. + - `stats/[assembler]-[binner]-[sample/group]_*.tsv`: Coverage statistics of each sub-contig cut up by CONOCOCT prior in an intermediate step prior to binning. Likely not useful in most cases. + - `stats/[assembler]-[binner]-[sample/group].log.txt`: CONCOCT execution log file. + - `stats/[assembler]-[binner]-[sample/group]_*.args`: List of arguments used in CONCOCT execution. + -
+
+
+By default, only the raw bins (and unbinned contigs) from the actual binning methods, but not from the binning refinement with DAS Tool, will be used for downstream bin quality control, annotation and taxonomic classification. The parameter `--postbinning_input` can be used to change this behaviour.
+
+⚠️ Due to ability to perform downstream QC of both raw and refined bins in parallel (via `--postbinning_input)`, bin names in DAS Tools's `*_allBins.eval` file will include `Refined`. However for this particular file, they _actually_ refer to the 'raw' input bins. The pipeline renames the input files prior to running DASTool to ensure they can be disambiguated from the original bin files in the downstream QC steps.
+
+### Tiara
+
+Tiara is a contig classifier that identifies the domain (prokarya, eukarya) of contigs within an assembly. This is used in this pipeline to rapidly and with few resources identify the most likely domain classification of each bin or unbin based on its contig identities.
+
+Output files
+ +- `GenomeBinning/DASTool/` + - `[assembler]-[sample/group]_allBins.eval`: Tab-delimited description with quality and completeness metrics for the input bin sets. Quality and completeness are estimated by DAS TOOL using a scoring function based on the frequency of bacterial or archaeal reference single-copy genes (SCG). Please see note at the bottom of this section on file names. + - `[assembler]-[sample/group]_DASTool_summary.tsv`: Tab-delimited description with quality and completeness metrics for the refined output bin sets. + - `[assembler]-[sample/group]_DASTool_contig2bin.tsv`: File describing which contig is associated to which bin from the input binners. + - `[assembler]-[sample/group]_DASTool.log`: Log file from the DAS Tool run describing the command executed and additional runtime information. + - `[assembler]-[sample/group].seqlength`: Tab-delimited file describing the length of each contig. + - `bins/[assembler]-[binner]Refined-[sample/group].*.fa`: Refined bins in fasta format. + - `unbinned/[assembler]-DASToolUnbinned-[sample/group].*.fa`: Unbinned contigs from bin refinement in fasta format. + +
+
+
+Typically, you would use `tiara_summary.tsv` as the primary file to see which bins or unbins have been classified to which domains at a glance, whereas the files in `Taxonomy/Tiara` provide classifications for each contig.
+
### Bin sequencing depth
-For each genome bin the median sequencing depth is computed based on the corresponding contig depths given in `GenomeBinning/[assembler]-[sample/group]-depth.txt.gz`.
+For each bin or refined bin the median sequencing depth is computed based on the corresponding contig depths.
Output files
+ +- `Taxonomy/Tiara/` + - `[assembler]-[sample/group].tiara.txt` - Tiara output classifications (with probabilities) for all contigs within the specified sample/group assembly + - `log_[assembler]-[sample/group].txt` - log file detailing the parameters used by the Tiara model for contig classification. +- `GenomeBinning/tiara_summary.tsv` - Summary of Tiara domain classification for all bins. + +Output files
-* `GenomeBinning/` - * `bin_depths_summary.tsv`: Summary of bin sequencing depths for all samples. Depths are available for samples mapped against the corresponding assembly, i.e. according to the mapping strategy specified with `--binning_map_mode`. Only for short reads. - * `[assembler]-[sample/group]-binDepths.heatmap.png`: Clustered heatmap showing bin abundances of the assembly across samples. Bin depths are transformed to centered log-ratios and bins as well as samples are clustered by Euclidean distance. Again, sample depths are available according to the mapping strategy specified with `--binning_map_mode`. +- `GenomeBinning/depths/bins/` + - `bin_depths_summary.tsv`: Summary of bin sequencing depths for all samples. Depths are available for samples mapped against the corresponding assembly, i.e. according to the mapping strategy specified with `--binning_map_mode`. Only for short reads. + - `bin_refined_depths_summary.tsv`: Summary of sequencing depths for refined bins for all samples, if refinement was performed. Depths are available for samples mapped against the corresponding assembly, i.e. according to the mapping strategy specified with `--binning_map_mode`. Only for short reads. + - `[assembler]-[binner]-[sample/group]-binDepths.heatmap.png`: Clustered heatmap showing bin abundances of the assembly across samples. Bin depths are transformed to centered log-ratios and bins as well as samples are clustered by Euclidean distance. Again, sample depths are available according to the mapping strategy specified with `--binning_map_mode`.Output files
-* `GenomeBinning/QC/QUAST/[assembler]-[bin]/` - * `report.*`: QUAST report in various formats, such as html, txt, tsv or tex - * `quast.log`: QUAST log file - * `predicted_genes/[assembler]-[sample/group].rna.gff`: Contig positions for rRNA genes in gff version 3 format -* `GenomeBinning/QC/` - * `quast_summary.tsv`: QUAST output for all bins summarized +- `GenomeBinning/QC/QUAST/[assembler]-[bin]/` + - `report.*`: QUAST report in various formats, such as html, pdf, tex, tsv, or txt + - `transposed_report.*`: QUAST report that has been transposed into wide format (tex, tsv, or txt) + - `quast.log`: QUAST log file + - `metaquast.log`: MetaQUAST log file + - `icarus.html`: Icarus main menu with links to interactive viewers + - `icarus_viewers/contig_size_viewer.html`: Diagram of contigs that are ordered from longest to shortest + - `basic_stats/cumulative_plot.pdf`: Shows the growth of contig lengths (contigs are ordered from largest to shortest) + - `basic_stats/GC_content_plot.pdf`: Shows the distribution of GC content in the contigs + - `basic_stats/[assembler]-[bin]_GC_content_plot.pdf`: Histogram of the GC percentage for the contigs + - `basic_stats/Nx_plot.pdf`: Plot of Nx values as x varies from 0 to 100%. + - `predicted_genes/[assembler]-[bin].rna.gff`: Contig positions for rRNA genes in gff version 3 format + - `predicted_genes/barrnap.log`: Barrnap log file (ribosomal RNA predictor) +- `GenomeBinning/QC/` + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]-quast_summary.tsv`: QUAST output summarized per sample/condition. + - `quast_summary.tsv`: QUAST output for all bins summarizedOutput files
-* `GenomeBinning/QC/BUSCO/` - * `[assembler]-[bin]_busco.log`: Log file containing the standard output of BUSCO. - * `[assembler]-[bin]_busco.err`: File containing potential error messages returned from BUSCO. - * `short_summary.domain.[lineage].[assembler]-[bin].txt`: BUSCO summary of the results for the selected domain when run in automated lineage selection mode. Not available for bins for which a viral lineage was selected. - * `short_summary.specific_lineage.[lineage].[assembler]-[bin].txt`: BUSCO summary of the results in case a more specific lineage than the domain could be selected or for the lineage provided via `--busco_reference`. - * `[assembler]-[bin]_buscos.[lineage].fna.gz`: Nucleotide sequence of all identified BUSCOs for used lineages (domain or specific). - * `[assembler]-[bin]_buscos.[lineage].faa.gz`: Aminoacid sequence of all identified BUSCOs for used lineages (domain or specific). - * `[assembler]-[bin]_prodigal.gff`: Genes predicted with Prodigal. +- `GenomeBinning/QC/BUSCO/` + - `[assembler]-[bin]_busco.log`: Log file containing the standard output of BUSCO. + - `[assembler]-[bin]_busco.err`: File containing potential error messages returned from BUSCO. + - `short_summary.domain.[lineage].[assembler]-[bin].txt`: BUSCO summary of the results for the selected domain when run in automated lineage selection mode. Not available for bins for which a viral lineage was selected. + - `short_summary.specific_lineage.[lineage].[assembler]-[bin].txt`: BUSCO summary of the results in case a more specific lineage than the domain could be selected or for the lineage provided via `--busco_db`. + - `[assembler]-[bin]_buscos.[lineage].fna.gz`: Nucleotide sequence of all identified BUSCOs for used lineages (domain or specific). + - `[assembler]-[bin]_buscos.[lineage].faa.gz`: Aminoacid sequence of all identified BUSCOs for used lineages (domain or specific). + - `[assembler]-[bin]_prodigal.gff`: Genes predicted with Prodigal.Output files
-* `GenomeBinning/QC/BUSCO/` - * `busco_downloads/`: All files and lineage datasets downloaded by BUSCO when run in automated lineage selection mode. (Can currently not be used to reproduce analysis, see the [nf-core/mag website documentation](https://nf-co.re/mag/usage#reproducibility) how to achieve reproducible BUSCO results). - * `reference/*.tar.gz`: BUSCO reference lineage dataset that was provided via `--busco_reference`. +- `GenomeBinning/QC/BUSCO/` + - `busco_downloads/`: All files and lineage datasets downloaded by BUSCO when run in automated lineage selection mode. (Can currently not be used to reproduce analysis, see the [nf-core/mag website documentation](https://nf-co.re/mag/usage#reproducibility) how to achieve reproducible BUSCO results). + - `reference/*.tar.gz`: BUSCO reference lineage dataset that was provided via `--busco_db`.Output files
-* `GenomeBinning/QC/` - * `busco_summary.tsv`: A summary table of the BUSCO results, with % of marker genes found. If run in automated lineage selection mode, both the results for the selected domain and for the selected more specific lineage will be given, if available. +- `GenomeBinning/QC/` + - `busco_summary.tsv`: A summary table of the BUSCO results, with % of marker genes found. If run in automated lineage selection mode, both the results for the selected domain and for the selected more specific lineage will be given, if available. + +
+
+If the parameter `--save_checkm_reference` is set, additionally the used the CheckM reference datasets are stored in the output directory.
+
+Output files
+ +- `GenomeBinning/QC/CheckM/` + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]_qa.txt`: Detailed statistics about bins informing completeness and contamamination scores (output of `checkm qa`). This should normally be your main file to use to evaluate your results. + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]_wf.tsv`: Overall summary file for completeness and contamination (output of `checkm lineage_wf`). + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]/`: intermediate files for CheckM results, including CheckM generated annotations, log, lineage markers etc. + - `checkm_summary.tsv`: A summary table of the CheckM results for all bins (output of `checkm qa`).
+
+
+#### GUNC
+
+[Genome UNClutterer (GUNC)](https://grp-bork.embl-community.io/gunc/index.html) is a tool for detection of chimerism and contamination in prokaryotic genomes resulting from mis-binning of genomic contigs from unrelated lineages. It does so by applying an entropy based score on taxonomic assignment and contig location of all genes in a genome. It is generally considered as a additional complement to CheckM results.
+
+Output files
+ +- `GenomeBinning/QC/CheckM/` + - `checkm_downloads/`: All CheckM reference files downloaded from the CheckM FTP server, when not supplied by the user. + - `checkm_data_2015_01_16/*`: a range of directories and files required for CheckM to run. + +
+
+
+GUNC will be run if specified with `--run_gunc` as a standalone, unless CheckM is also activated via `--qc_tool 'checkm'`, in which case GUNC output will be merged with the CheckM output using `gunc merge_checkm`.
+
+If `--gunc_save_db` is specified, the output directory will also contain the requested database (progenomes, or GTDB) in DIAMOND format.
+
## Taxonomic classification of binned genomes
### CAT
-[CAT](https://github.com/dutilh/CAT) is a toolkit for annotating contigs and bins from metagenome-assembled-genomes. The MAG pipeline uses CAT to assign taxonomy to genome bins based on the taxnomy of the contigs.
+[CAT](https://github.com/dutilh/CAT) is a toolkit for annotating contigs and bins from metagenome-assembled-genomes. The nf-core/mag pipeline uses CAT to assign taxonomy to genome bins based on the taxnomy of the contigs.
Output files
+ +- `GenomeBinning/QC/gunc_summary.tsv` +- `GenomeBinning/QC/gunc_checkm_summary.tsv` +- `[gunc-database].dmnd` +- `GUNC/` + - `raw/` + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]/GUNC_checkM.merged.tsv`: Per sample GUNC [output](https://grp-bork.embl-community.io/gunc/output.html) containing with taxonomic and completeness QC statistics. + - `checkmmerged/` + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]/GUNC.progenomes_2.1.maxCSS_level.tsv`: Per sample GUNC output merged with output from [CheckM](#checkm) + +Output files
-* `Taxonomy/CAT/[assembler]/` - * `[assembler]-[sample/group].ORF2LCA.names.txt.gz`: Tab-delimited files containing the lineage of each contig, with full lineage names - * `[assembler]-[sample/group].bin2classification.names.txt.gz`: Taxonomy classification of the genome bins, with full lineage names -* `Taxonomy/CAT/[assembler]/raw/` - * `[assembler]-[sample/group].concatenated.predicted_proteins.faa.gz`: Predicted protein sequences for each genome bin, in fasta format - * `[assembler]-[sample/group].concatenated.predicted_proteins.gff.gz`: Predicted protein features for each genome bin, in gff format - * `[assembler]-[sample/group].ORF2LCA.txt.gz`: Tab-delimited files containing the lineage of each contig - * `[assembler]-[sample/group].bin2classification.txt.gz`: Taxonomy classification of the genome bins - * `[assembler]-[sample/group].log`: Log files +- `Taxonomy/CAT/[assembler]/[binner]/` + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group].ORF2LCA.names.txt.gz`: Tab-delimited files containing the lineage of each contig, with full lineage names + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group].bin2classification.names.txt.gz`: Taxonomy classification of the genome bins, with full lineage names +- `Taxonomy/CAT/[assembler]/[binner]/raw/` + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group].concatenated.predicted_proteins.faa.gz`: Predicted protein sequences for each genome bin, in fasta format + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group].concatenated.predicted_proteins.gff.gz`: Predicted protein features for each genome bin, in gff format + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group].ORF2LCA.txt.gz`: Tab-delimited files containing the lineage of each contig + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group].bin2classification.txt.gz`: Taxonomy classification of the genome bins + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group].log`: Log filesOutput files
-* `Taxonomy/CAT/CAT_prepare_*.tar.gz`: Generated and used CAT database. +- `Taxonomy/CAT/CAT_prepare_*.tar.gz`: Generated and used CAT database.Output files
-* `Taxonomy/GTDB-Tk/[assembler]/[sample/group]/` - * `gtdbtk.[assembler]-[sample/group].{bac120/ar122}.summary.tsv`: Classifications for bacterial and archaeal genomes (see the [GTDB-Tk documentation for details](https://ecogenomics.github.io/GTDBTk/files/summary.tsv.html). - * `gtdbtk.[assembler]-[sample/group].{bac120/ar122}.classify.tree.gz`: Reference tree in Newick format containing query genomes placed with pplacer. - * `gtdbtk.[assembler]-[sample/group].{bac120/ar122}.markers_summary.tsv`: A summary of unique, duplicated, and missing markers within the 120 bacterial marker set, or the 122 archaeal marker set for each submitted genome. - * `gtdbtk.[assembler]-[sample/group].{bac120/ar122}.msa.fasta.gz`: FASTA file containing MSA of submitted and reference genomes. - * `gtdbtk.[assembler]-[sample/group].{bac120/ar122}.filtered.tsv`: A list of genomes with an insufficient number of amino acids in MSA. - * `gtdbtk.[assembler]-[sample/group].*.log`: Log files. - * `gtdbtk.[assembler]-[sample/group].failed_genomes.tsv`: A list of genomes for which the GTDB-Tk analysis failed, e.g. because Prodigal could not detect any genes. -* `Taxonomy/GTDB-Tk/gtdbtk_summary.tsv`: A summary table of the GTDB-Tk classification results for all bins, also containing bins which were discarded based on the BUSCO QC, which were filtered out by GTDB-Tk ((listed in `*.filtered.tsv`) or for which the analysis failed (listed in `*.failed_genomes.tsv`). +- `Taxonomy/GTDB-Tk/[assembler]/[binner]/[sample/group]/` + - `gtdbtk.[assembler]-[binner]-[sample/group].{bac120/ar122}.summary.tsv`: Classifications for bacterial and archaeal genomes (see the [GTDB-Tk documentation for details](https://ecogenomics.github.io/GTDBTk/files/summary.tsv.html)). + - `gtdbtk.[assembler]-[binner]-[domain]-[refinement]-[sample/group].{bac120/ar122}.classify.tree.gz`: Reference tree in Newick format containing query genomes placed with pplacer. + - `gtdbtk.[assembler]-[binner]-[domain]-[refinement]-[sample/group].{bac120/ar122}.markers_summary.tsv`: A summary of unique, duplicated, and missing markers within the 120 bacterial marker set, or the 122 archaeal marker set for each submitted genome. + - `gtdbtk.[assembler]-[binner]-[domain]-[refinement]-[sample/group].{bac120/ar122}.msa.fasta.gz`: FASTA file containing MSA of submitted and reference genomes. + - `gtdbtk.[assembler]-[binner]-[domain]-[refinement]-[sample/group].{bac120/ar122}.filtered.tsv`: A list of genomes with an insufficient number of amino acids in MSA. + - `gtdbtk.[assembler]-[binner]-[domain]-[refinement]-[sample/group].*.log`: Log files. + - `gtdbtk.[assembler]-[binner]-[domain]-[refinement]-[sample/group].failed_genomes.tsv`: A list of genomes for which the GTDB-Tk analysis failed, e.g. because Prodigal could not detect any genes. +- `Taxonomy/GTDB-Tk/gtdbtk_summary.tsv`: A summary table of the GTDB-Tk classification results for all bins, also containing bins which were discarded based on the BUSCO QC, which were filtered out by GTDB-Tk (listed in `*.filtered.tsv`) or for which the analysis failed (listed in `*.failed_genomes.tsv`).Output files
-* `Prokka/[assembler]/[bin]/` - * `[bin].gff`: annotation in GFF3 format, containing both sequences and annotations - * `[bin].gbk`: annotation in GenBank format, containing both sequences and annotations - * `[bin].fna`: nucleotide FASTA file of the input contig sequences - * `[bin].faa`: protein FASTA file of the translated CDS sequences - * `[bin].ffn`: nucleotide FASTA file of all the prediction transcripts (CDS, rRNA, tRNA, tmRNA, misc_RNA) - * `[bin].sqn`: an ASN1 format "Sequin" file for submission to Genbank - * `[bin].fsa`: nucleotide FASTA file of the input contig sequences, used by "tbl2asn" to create the .sqn file - * `[bin].tbl`: feature Table file, used by "tbl2asn" to create the .sqn file - * `[bin].err`: unacceptable annotations - the NCBI discrepancy report. - * `[bin].log`: contains all the output that Prokka produced during its run - * `[bin].txt`: statistics relating to the annotated features found - * `[bin].tsv`: tab-separated file of all features (locus_tag, ftype, len_bp, gene, EC_number, COG, product) +- `Annotation/Prokka/[assembler]/[bin]/` + - `[assembler]-[binner]-[bin].gff`: annotation in GFF3 format, containing both sequences and annotations + - `[assembler]-[binner]-[bin].gbk`: annotation in GenBank format, containing both sequences and annotations + - `[assembler]-[binner]-[bin].fna`: nucleotide FASTA file of the input contig sequences + - `[assembler]-[binner]-[bin].faa`: protein FASTA file of the translated CDS sequences + - `[assembler]-[binner]-[bin].ffn`: nucleotide FASTA file of all the prediction transcripts (CDS, rRNA, tRNA, tmRNA, misc_RNA) + - `[assembler]-[binner]-[bin].sqn`: an ASN1 format "Sequin" file for submission to Genbank + - `[assembler]-[binner]-[bin].fsa`: nucleotide FASTA file of the input contig sequences, used by "tbl2asn" to create the .sqn file + - `[assembler]-[binner]-[bin].tbl`: feature Table file, used by "tbl2asn" to create the .sqn file + - `[assembler]-[binner]-[bin].err`: unacceptable annotations - the NCBI discrepancy report. + - `[assembler]-[binner]-[bin].log`: contains all the output that Prokka produced during its run + - `[assembler]-[binner]-[bin].txt`: statistics relating to the annotated features found + - `[assembler]-[binner]-[bin].tsv`: tab-separated file of all features (locus_tag, ftype, len_bp, gene, EC_number, COG, product) + +
+
@@ -422,19 +668,725 @@ Whole genome annotation is the process of identifying features of interest in a
Output files
+ +- `Annotation/MetaEuk/[assembler]/[bin]` + - `[assembler]-[binner]-[bin].fas`: fasta file of protein sequences identified by MetaEuk + - `[assembler]-[binner]-[bin].codon.fas`: fasta file of nucleotide sequences corresponding to the protein sequences fasta + - `[assembler]-[binner]-[bin].headersMap.tsv`: tab-separated table containing the information from each header in the fasta files + - `[assembler]-[binner]-[bin].gff`: annotation in GFF3 formatOutput files
-* `GenomeBinning/bin_summary.tsv`: Summary of bin sequencing depths together with BUSCO, QUAST and GTDB-Tk results, if at least one of the later was generated. +- `GenomeBinning/bin_summary.tsv`: Summary of bin sequencing depths together with BUSCO, CheckM, QUAST and GTDB-Tk results, if at least one of the later was generated. This will also include refined bins if `--refine_bins_dastool` binning refinement is performed. Note that in contrast to the other tools, for CheckM the bin name given in the column "Bin Id" does not contain the ".fa" extension. + +
+
+
+### `variant_calling`
+
+Because of aDNA damage, _de novo_ assemblers sometimes struggle to call a correct consensus on the contig sequence. To avoid this situation, the consensus is optionally re-called with a variant calling software using the reads aligned back to the contigs when `--run_ancient_damagecorrection` is supplied.
+
+Output files
+ +- `Ancient_DNA/pydamage/analyze` + - `[assembler]_[sample/group]/pydamage_results/pydamage_results.csv`: PyDamage raw result tabular file in `.csv` format. Format described here: [pydamage.readthedocs.io/en/0.62/output.html](https://pydamage.readthedocs.io/en/0.62/output.html) +- `Ancient_DNA/pydamage/filter` + - `[assembler]_[sample/group]/pydamage_results/pydamage_results.csv`: PyDamage filtered result tabular file in `.csv` format. Format described here: [pydamage.readthedocs.io/en/0.62/output.html](https://pydamage.readthedocs.io/en/0.62/output.html) + +
+
+
+### MultiQC
+
+Output files
+ +- `variant_calling/consensus` + - `[assembler]_[sample/group].fa`: contigs sequence with re-called consensus from read-to-contig alignment +- `variant_calling/unfiltered` + - `[assembler]_[sample/group].vcf.gz`: raw variant calls of the reads aligned back to the contigs. +- `variant_calling/filtered` + - `[assembler]_[sample/group].filtered.vcf.gz`: quality filtered variant calls of the reads aligned back to the contigs. + +
+
+
+[MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory.
+
+Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see Output files
+ +- `multiqc/` + - `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser. + - `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline. + - `multiqc_plots/`: directory containing static images from the report in various formats. + +
+
+
+[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage.
+
+# nf-core/taxprofiler: Output
+
+## Introduction
+
+This document describes the output produced by the pipeline. Most of the plots are taken from the MultiQC report, which summarises results at the end of the pipeline.
+
+The directories listed below will be created in the results directory after the pipeline has finished. All paths are relative to the top-level results directory.
+
+## Pipeline overview
+
+The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps:
+
+- [UNTAR](#untar) - Optionally saved decompressed input databases
+- [FastQC](#fastqc) - Raw read QC
+- [falco](#fastqc) - Alternative to FastQC for raw read QC
+- [fastp](#fastp) - Adapter trimming for Illumina data
+- [AdapterRemoval](#adapterremoval) - Adapter trimming for Illumina data
+- [Porechop](#porechop) - Adapter removal for Oxford Nanopore data
+- [BBDuk](#bbduk) - Quality trimming and filtering for Illumina data
+- [PRINSEQ++](#prinseq) - Quality trimming and filtering for Illunina data
+- [Filtlong](#filtlong) - Quality trimming and filtering for Nanopore data
+- [Bowtie2](#bowtie2) - Host removal for Illumina reads
+- [minimap2](#minimap2) - Host removal for Nanopore reads
+- [SAMtools stats](#samtools-stats) - Statistics from host removal
+- [SAMtools fastq](#samtools-fastq) - Converts unmapped BAM file to fastq format (minimap2 only)
+- [Analysis Ready Reads](#analysis-read-reads) - Optional results directory containing the final processed reads used as input for classification/profiling.
+- [Bracken](#bracken) - Taxonomic classifier using k-mers and abundance estimations
+- [Kraken2](#kraken2) - Taxonomic classifier using exact k-mer matches
+- [KrakenUniq](#krakenuniq) - Taxonomic classifier that combines the k-mer-based classification and the number of unique k-mers found in each species
+- [Centrifuge](#centrifuge) - Taxonomic classifier that uses a novel indexing scheme based on the Burrows-Wheeler transform (BWT) and the Ferragina-Manzini (FM) index.
+- [Kaiju](#kaiju) - Taxonomic classifier that finds maximum (in-)exact matches on the protein-level.
+- [Diamond](#diamond) - Sequence aligner for protein and translated DNA searches.
+- [MALT](#malt) - Sequence alignment and analysis tool designed for processing high-throughput sequencing data, especially in the context of metagenomics
+- [MetaPhlAn](#metaphlan) - Genome-level marker gene based taxonomic classifier
+- [mOTUs](#motus) - Tool for marker gene-based OTU (mOTU) profiling.
+- [KMCP](#kmcp) - Taxonomic classifier that utilizes genome coverage information by splitting the reference genomes into chunks and stores k-mers in a modified and optimized COBS index for fast alignment-free sequence searching.
+- [ganon](#ganon) - Taxonomic classifier and profile that uses Interleaved Bloom Filters as indices based on k-mers/minimizers.
+- [TAXPASTA](#taxpasta) - Tool to standardise taxonomic profiles as well as merge profiles across samples from the same database and classifier/profiler.
+- [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline
+- [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution
+
+
+
+### untar
+
+untar is used in nf-core/taxprofiler to decompress various input files ending in `.tar.gz`. This process is mainly used for decompressing input database archive files.
+
+Output files
+ +- `pipeline_info/` + - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. + - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. + - Parameters used by the pipeline run: `params.json`. + +
+`: directory containing contents of the decompressed archive
+
+
+
+This directory will only be present if `--save_untarred_databases` is supplied. The contained directories can be useful for moving the decompressed directories to a central 'cache' location allowing users to re-use the same databases. This is useful to save unnecessary computational time of decompressing the archives on every run.
+
+### FastQC or Falco
+
+Output files
+ +- `untar/` + - `database/` + - `
+
+
+[FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/).
+
+If preprocessing is turned on, nf-core/taxprofiler runs FastQC/Falco twice -once before and once after adapter removal/read merging, to allow evaluation of the performance of these preprocessing steps. Note in the General Stats table, the columns of these two instances of FastQC/Falco are placed next to each other to make it easier to evaluate. However, the columns of the actual preprocessing steps (i.e, fastp, AdapterRemoval, and Porechop) will be displayed _after_ the two FastQC/Falco columns, even if they were run 'between' the two FastQC/Falco jobs in the pipeline itself.
+
+:::info
+Falco produces identical output to FastQC but in the `falco/` directory.
+:::
+
+
+
+
+
+
+
+:::note
+The FastQC plots displayed in the MultiQC report shows _untrimmed_ reads. They may contain adapter sequence and potentially regions with low quality.
+:::
+
+### fastp
+
+[fastp](https://github.com/OpenGene/fastp) is a FASTQ pre-processing tool for quality control, trimmming of adapters, quality filtering and other features.
+
+It is used in nf-core/taxprofiler for adapter trimming of short-reads.
+
+Output files
+ +- `{fastqc,falco}/` + - {raw,preprocessed} + - `*html`: FastQC or Falco report containing quality metrics in HTML format. + - `*.txt`: FastQC or Falco report containing quality metrics in TXT format. + - `*.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images (FastQC only). + +
+.fastp.fastq.gz`: File with the trimmed unmerged fastq reads.
+ - `.merged.fastq.gz`: File with the reads that were successfully merged.
+ - `.*{log,html,json}`: Log files in different formats.
+
+
+
+By default nf-core/taxprofiler will only provide the `Output files
+ +- `fastp/` + - `
+.settings`: AdapterRemoval log file containing general adapter removal, read trimming and merging statistics
+ - `.collapsed.fastq.gz` - read-pairs that merged and did not undergo trimming (only when `--shortread_qc_mergepairs` supplied)
+ - `.collapsed.truncated.fastq.gz` - read-pairs that merged underwent quality trimming (only when `--shortread_qc_mergepairs` supplied)
+ - `.pair1.truncated.fastq.gz` - read 1 of pairs that underwent quality trimming
+ - `.pair2.truncated.fastq.gz` - read 2 of pairs that underwent quality trimming (and could not merge if `--shortread_qc_mergepairs` supplied)
+ - `.singleton.truncated.fastq.gz` - orphaned read pairs where one of the pair was discarded
+ - `.discard.fastq.gz` - reads that were discarded due to length or quality filtering
+
+
+
+By default nf-core/taxprofiler will only provide the `.settings` file if AdapterRemoval is selected.
+
+You will only find the `.fastq` files in the results directory if you provide ` --save_preprocessed_reads`. If this is selected, you may receive different combinations of `.fastq` files for each sample depending on the input types - e.g. whether you have merged or not, or if you're supplying both single- and paired-end reads. Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+:::warning
+The resulting `.fastq` files may _not_ always be the 'final' reads that go into taxprofiling, if you also run other steps such as complexity filtering, host removal, run merging etc..
+:::
+
+### Porechop
+
+[Porechop](https://github.com/rrwick/Porechop) is a tool for finding and removing adapters from Oxford Nanopore reads. Adapters on the ends of reads are trimmed and if a read has an adapter in its middle, it is considered a chimeric and it chopped into separate reads.
+
+Output files
+ +- `adapterremoval/` + - `
+.log`: Log file containing trimming statistics
+ - `.fastq.gz`: Adapter-trimmed file
+
+
+
+The output logs are saved in the output folder and are part of MultiQC report.You do not normally need to check these manually.
+
+You will only find the `.fastq` files in the results directory if you provide ` --save_preprocessed_reads`. Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+:::warning
+We do **not** recommend using Porechop if you are already trimming the adapters with ONT's basecaller Guppy.
+:::
+
+### BBDuk
+
+[BBDuk](https://jgi.doe.gov/data-and-tools/software-tools/bbtools/bb-tools-user-guide/bbduk-guide/) stands for Decontamination Using Kmers. BBDuk was developed to combine most common data-quality-related trimming, filtering, and masking operations into a single high-performance tool.
+
+It is used in nf-core/taxprofiler for complexity filtering using different algorithms. This means that it will remove reads with low sequence diversity (e.g. mono- or dinucleotide repeats).
+
+Output files
+ +- `porechop/` + - `
+.bbduk.log`: log file containing filtering statistics
+ - `.fastq.gz`: resulting FASTQ file without low-complexity reads
+
+
+
+By default nf-core/taxprofiler will only provide the `.log` file if BBDuk is selected as the complexity filtering tool. You will only find the complexity filtered reads in your results directory if you provide ` --save_complexityfiltered_reads`. Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+:::warning
+The resulting `.fastq` files may _not_ always be the 'final' reads that go into taxprofiling, if you also run other steps such as host removal, run merging etc..
+:::
+
+### PRINSEQ++
+
+[PRINSEQ++](https://github.com/Adrian-Cantu/PRINSEQ-plus-plus) is a C++ implementation of the [prinseq-lite.pl](https://prinseq.sourceforge.net/) program. It can be used to filter, reformat or trim genomic and metagenomic sequence data.
+
+It is used in nf-core/taxprofiler for complexity filtering using different algorithms. This means that it will remove reads with low sequence diversity (e.g. mono- or dinucleotide repeats).
+
+Output files
+ +- `bbduk/` + - `
+.log`: log file containing number of reads. Row IDs correspond to: `min_len, max_len, min_gc, max_gc, min_qual_score, min_qual_mean, ns_max_n, noiupac, derep, lc_entropy, lc_dust, trim_tail_left, trim_tail_right, trim_qual_left, trim_qual_right, trim_left, trim_right`
+ - `_good_out.fastq.gz`: resulting FASTQ file without low-complexity reads
+
+
+
+By default nf-core/taxprofiler will only provide the `.log` file if PRINSEQ++ is selected as the complexity filtering tool. You will only find the complexity filtered `.fastq` files in your results directory if you supply ` --save_complexityfiltered_reads`. Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+:::warning
+The resulting `.fastq` files may _not_ always be the 'final' reads that go into taxprofiling, if you also run other steps such as host removal, run merging etc..
+:::
+
+### Filtlong
+
+[Filtlong](https://github.com/rrwick/Filtlong) is a quality filtering tool for long reads. It can take a set of small reads and produce a smaller, better subset.
+
+Output files
+ +- `prinseqplusplus/` + - `
+_filtered.fastq.gz`: Quality or short read data filtered file
+ - `_filtered.log`: log file containing summary statistics
+
+
+
+You will only find the `.fastq` files in the results directory if you provide ` --save_preprocessed_reads`. Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+:::warning
+We do _not_ recommend using Filtlong if you are performing filtering of low quality reads with ONT's basecaller Guppy.
+:::
+
+### Bowtie2
+
+[Bowtie 2](https://bowtie-bio.sourceforge.net/bowtie2/index.shtml) is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. It is particularly good at aligning reads of about 50 up to 100s or 1,000s of characters, and particularly good at aligning to relatively long (e.g. mammalian) genomes.
+
+It is used with nf-core/taxprofiler to allow removal of 'host' (e.g. human) and/or other possible contaminant reads (e.g. Phi X) from short-read `.fastq` files prior to profiling.
+
+Output files
+ +- `filtlong/` + - `
+.bam`: BAM file containing reads that aligned against the user-supplied reference genome as well as unmapped reads
+ - `.bowtie2.log`: log file about the mapped reads
+ - `.unmapped.fastq.gz`: the off-target reads from the mapping that is used in downstream steps.
+
+
+
+By default nf-core/taxprofiler will only provide the `.log` file if host removal is turned on. You will only have a `.bam` file if you specify `--save_hostremoval_bam`. This will contain _both_ mapped and unmapped reads. You will only get FASTQ files if you specify to save `--save_hostremoval_unmapped` - these contain only unmapped reads. Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+:::info
+Unmapped reads in FASTQ are only found in this directory for short-reads, for long-reads see [`samtools/fastq/`](#samtools-fastq).
+:::
+
+:::info
+The resulting `.fastq` files may _not_ always be the 'final' reads that go into taxprofiling, if you also run other steps such as run merging etc..
+:::
+
+:::info
+While there is a dedicated section in the MultiQC HTML for Bowtie2, these values are not displayed by default in the General Stats table. Rather, alignment statistics to host genome is reported via samtools stats module in MultiQC report for direct comparison with minimap2 (see below).
+:::
+
+### minimap2
+
+[minimap2](https://github.com/lh3/minimap2) is an alignment tool suited to mapping long reads to reference sequences.
+
+It is used with nf-core/taxprofiler to allow removal of 'host' (e.g. human) or other possible contaminant reads from long-read `.fastq` files prior to taxonomic classification/profiling.
+
+Output files
+ +- `bowtie2/` + - `build/` + - `*.bt2`: Bowtie2 indicies of reference genome, only if `--save_hostremoval_index` supplied. + - `align/` + - `
+.bam`: Alignment file in BAM format containing both mapped and unmapped reads.
+
+
+
+By default, nf-core/taxprofiler will only provide the `.bam` file containing mapped and unmapped reads if saving of host removal for long reads is turned on via `--save_hostremoval_bam`.
+
+:::info
+minimap2 is not yet supported as a module in MultiQC and therefore there is no dedicated section in the MultiQC HTML. Rather, alignment statistics to host genome is reported via samtools stats module in MultiQC report.
+:::
+
+:::info
+Unlike Bowtie2, minimap2 does not produce an unmapped FASTQ file by itself. See [`samtools/fastq`](#samtools-fastq).
+:::
+
+### SAMtools fastq
+
+[SAMtools fastq](http://www.htslib.org/doc/1.1/samtools.html) converts a `.sam`, `.bam`, or `.cram` alignment file to FASTQ format
+
+Output files
+ +- `minimap2/` + - `build/` + - `*.mmi2`: minimap2 indices of reference genome, only if `--save_hostremoval_index` supplied. + - `align/` + - `
+_interleaved.fq.gz`: Unmapped reads only in FASTQ gzip format
+
+
+
+This directory will be present and contain the unmapped reads from the `.fastq` format from long-read minimap2 host removal, if `--save_hostremoval_unmapped` is supplied. Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+:::info
+For short-read unmapped reads, see [bowtie2](#bowtie2).
+:::
+
+### Analysis Ready Reads
+
+:::info
+This optional results directory will only be present in the pipeline results when supplying `--save_analysis_ready_reads`.
+:::
+
+Output files
+ +- `samtools/stats/` + - `
+_{fq,fastq}.gz`: Final reads that underwent preprocessing and were sent for classification/profiling.
+
+
+
+The results directory will contain the 'final' processed reads used as input for classification/profiling. It will _only_ include the output of the _last_ step of any combinations of preprocessing steps that may have been specified in the run configuration. For example, if you perform the read QC and host-removal preprocessing steps, the final reads that are sent to classification/profiling are the host-removed FASTQ files - those will be the ones present in this directory.
+
+:::warning
+If you turn off all preprocessing steps, then no results will be present in this directory. This happens independently for short- and long-reads. I.e. you will only have FASTQ files for short reads in this directory if you skip all long-read preprocessing.
+:::
+
+### SAMtools stats
+
+[SAMtools stats](http://www.htslib.org/doc/samtools-stats.html) collects statistics from a `.sam`, `.bam`, or `.cram` alignment file and outputs in a text format.
+
+Output files
+ +- `samtools/stats/` + - `
+.stats`: File containing samtools stats output.
+
+
+
+In most cases you do not need to check this file, as it is rendered in the MultiQC run report.
+
+### Run Merging
+
+nf-core/taxprofiler offers the option to merge FASTQ files of multiple sequencing runs or libraries that derive from the same sample, as specified in the input samplesheet.
+
+This is the last possible preprocessing step, so if you have multiple runs or libraries (and run merging turned on), this will represent the final reads that will go into classification/profiling steps.
+
+Output files
+ +- `samtools/stats/` + - `
+
+
+Note that you will only find samples that went through the run merging step in this directory. For samples that had a single run or library will not go through this step of the pipeline and thus will not be present in this directory.
+
+This directory and its FASTQ files will only be present if you supply `--save_runmerged_reads`.Alternatively, if you wish only to have the 'final' reads that go into classification/profiling (i.e., that may have additional processing), do not specify this flag but rather specify `--save_analysis_ready_reads`, in which case the reads will be in the folder `analysis_ready_reads`.
+
+### Bracken
+
+[Bracken](https://ccb.jhu.edu/software/bracken/) (Bayesian Reestimation of Abundance with Kraken) is a highly accurate statistical method that computes the abundance of species in DNA sequences from a metagenomics sample. Braken uses the taxonomy labels assigned by Kraken, a highly accurate metagenomics classification algorithm, to estimate the number of reads originating from each species present in a sample.
+
+:::info
+The first step of using Bracken requires running Kraken2, therefore the initial results before abundance estimation will be found in `Output files
+ +- `run_merging/` + - `*.fastq.gz`: Concatenated FASTQ files on a per-sample basis + +
+/`
+ - `bracken__combined_reports.txt`: combined bracken results as output from Bracken's `combine_bracken_outputs.py` script
+ - `/`
+ - `_.tsv`: TSV file containing per-sample summary of Bracken results with abundance information
+ - `_.report_bracken_species.txt`: Kraken2 style report with Bracken abundance information
+
+
+
+The main taxonomic profiling file from Bracken is the `*.tsv` file. This provides the basic results from Kraken2 but with the corrected abundance information. Note that the raw Kraken2 version of the upstream step of Bracken can be found in the `kraken2/` directory with the suffix of `Output files
+ +- `bracken/` + - `
+_combined_reports.txt`: A combined profile of all samples aligned to a given database (as generated by `krakentools`)
+ - If you have also run Bracken, the original Kraken report (i.e., _before_ read re-assignment) will also be included in this directory with `-bracken` suffixed to your Bracken database name if you supply `--bracken_save_intermediatekraken2` to the run. For example: `kraken2--bracken.tsv`. However in most cases you want to use the actual Bracken file (i.e., `bracken_.tsv`).
+ - `/`
+ - `_.classified.fastq.gz`: FASTQ file containing all reads that had a hit against a reference in the database for a given sample
+ - `_.unclassified.fastq.gz`: FASTQ file containing all reads that did not have a hit in the database for a given sample
+ - `_.report.txt`: A Kraken2 report that summarises the fraction abundance, taxonomic ID, number of Kmers, taxonomic path of all the hits in the Kraken2 run for a given sample. Will be 6 column rather than 8 if `--save_minimizers` specified. This report will **only** be included if you supply `--bracken_save_intermediatekraken2` to the run.
+ - `_.classifiedreads.txt`: A list of read IDs and the hits each read had against each database for a given sample
+
+
+
+The main taxonomic classification file from Kraken2 is the `_combined_reports.txt` or `*report.txt` file. The former provides you the broadest over view of the taxonomic classification results across all samples against a single database, where you get two columns for each sample e.g. `2_all` and `2_lvl`, as well as a summarised column summing up across all samples `tot_all` and `tot_lvl`. The latter gives you the most information for a single sample. The report file is also used for the taxpasta step.
+
+You will only receive the `.fastq` and `*classifiedreads.txt` file if you supply `--kraken2_save_reads` and/or `--kraken2_save_readclassifications` parameters to the pipeline.
+
+When running Bracken, you will only get the 'intermediate' Kraken2 report files in this directory if you supply `--bracken_save_intermediatekraken2` to the run.
+
+### KrakenUniq
+
+[KrakenUniq](https://github.com/fbreitwieser/krakenuniq) (formerly KrakenHLL) is an extension to the fast k-mer-based classification performed by [Kraken](https://github.com/DerrickWood/kraken) with an efficient algorithm for additionally assessing the coverage of unique k-mers found in each species in a dataset.
+
+Output files
+ +- `kraken2/` + - `
+/`
+ - `_[.merged].classified.fast{a,q}.gz`: Optional FASTA file containing all reads that had a hit against a reference in the database for a given sample. Paired-end input reads are merged in this output.
+ - `_[.merged].unclassified.fast{a,q}.gz`: Optional FASTA file containing all reads that did not have a hit in the database for a given sample. Paired-end input reads are merged in this output.
+ - `_.krakenuniq.report.txt`: A Kraken2-style report that summarises the fraction abundance, taxonomic ID, number of Kmers, taxonomic path of all the hits, with an additional column for k-mer coverage, that allows for more accurate distinguishing between false-positive/true-postitive hits.
+ - `_.krakenuniq.classified.txt`: An optional list of read IDs and the hits each read had against each database for a given sample.
+
+
+
+The main taxonomic classification file from KrakenUniq is the `*.krakenuniq.report.txt` file. This is an extension of the Kraken2 report with the additional k-mer coverage information that provides more information about the accuracy of hits.
+
+You will only receive the `.fasta.gz` and `*.krakenuniq.classified.txt` file if you supply `--krakenuniq_save_reads` and/or `--krakenuniq_save_readclassification` parameters to the pipeline.
+
+:::info
+The output system of KrakenUniq can result in other `stdout` or `stderr` logging information being saved in the report file, therefore you must check your report files before downstream use!
+:::
+
+### Centrifuge
+
+[Centrifuge](https://github.com/DaehwanKimLab/centrifuge) is a taxonomic sequence classifier that uses a Burrows-Wheeler transform and Ferragina-Manzina index for storing and mapping sequences.
+
+Output files
+ +- `krakenuniq/` + - `
+/`
+ - `.centrifuge.mapped.fastq.gz`: `FASTQ` files containing all mapped reads
+ - `.centrifuge.report.txt`: A classification report that summarises the taxonomic ID, the taxonomic rank, length of genome sequence, number of classified and uniquely classified reads
+ - `.centrifuge.results.txt`: A file that summarises the classification assignment for a read, i.e read ID, sequence ID, score for the classification, score for the next best classification, number of classifications for this read
+ - `.centrifuge.txt`: A Kraken2-style report that summarises the fraction abundance, taxonomic ID, number of k-mers, taxonomic path of all the hits in the centrifuge run for a given sample
+ - `.centrifuge.unmapped.fastq.gz`: FASTQ file containing all unmapped reads
+
+
+
+The main taxonomic classification files from Centrifuge are the `_combined_reports.txt`, `*report.txt`, `*results.txt` and the `*centrifuge.txt`. The latter is used by the taxpasta step. You will receive the `.fastq` files if you supply `--centrifuge_save_reads`.
+
+### Kaiju
+
+[Kaiju](https://github.com/bioinformatics-centre/kaiju) is a taxonomic classifier that finds maximum exact matches on the protein-level using the Burrows-Wheeler transform.
+
+Output files
+ +- `centrifuge/` + - `
+_combined_reports.txt`: A combined profile of all samples aligned to a given database (as generated by kaiju2table)
+ - `/`
+ - `_.kaiju.tsv`: Raw output from Kaiju with taxonomic rank, read ID and taxonic ID
+ - `_.kaijutable.txt`: Summarised Kaiju output with fraction abundance, taxonomic ID, number of reads, and taxonomic names (as generated by `kaiju2table`)
+
+
+
+The most useful summary file is the `_combined_reports.txt` file which summarises hits across all reads and samples. Separate per-sample versions summaries can be seen in `Output files
+ +- `kaiju/` + - `kaiju_
+/`
+ - `.log`: A log file containing stdout information
+ - `*.{blast,xml,txt,daa,sam,tsv,paf}`: A file containing alignment information in various formats, or taxonomic information in a text-based format. Exact output depends on user choice.
+
+
+
+By default you will receive a TSV output. Alternatively, you will receive a `*.sam` file if you provide the parameter `--diamond_save_reads` but in this case no taxonomic classification will be available(!), only the aligned reads in sam format.
+
+:::info
+DIAMOND has many output formats, so depending on your [choice](https://github.com/bbuchfink/diamond/wiki/3.-Command-line-options) with ` --diamond_output_format` you will receive the taxonomic information in a different format.
+:::
+
+### MALT
+
+[MALT](https://software-ab.cs.uni-tuebingen.de/download/malt) is a fast replacement for BLASTX, BLASTP and BLASTN, and provides both local and semi-global alignment capabilities.
+
+Output files
+ +- `diamond/` + - `
+/`
+ - `.blastn.sam`: sparse SAM file containing alignments of each hit
+ - `.megan`: summary file that can be loaded into the [MEGAN6](https://uni-tuebingen.de/fakultaeten/mathematisch-naturwissenschaftliche-fakultaet/fachbereiche/informatik/lehrstuehle/algorithms-in-bioinformatics/software/megan6/) interactive viewer. Generated by MEGAN6 companion tool `rma2info`
+ - `.rma6`: binary file containing all alignments and taxonomic information of hits that can be loaded into the [MEGAN6](https://uni-tuebingen.de/fakultaeten/mathematisch-naturwissenschaftliche-fakultaet/fachbereiche/informatik/lehrstuehle/algorithms-in-bioinformatics/software/megan6/) interactive viewer
+ - `.txt.gz`: text file containing taxonomic IDs and read counts against each taxon. Generated by MEGAN6 companion tool `rma2info`
+
+
+
+The main output of MALT is the `.rma6` file format, which can be only loaded into MEGAN and it's related tools. We provide the `rma2info` text files for improved compatibility with spreadsheet programs and other programmtic data manipulation tools, however this has only limited information compared to the 'binary' RMA6 file format (the `.txt` file only contains taxonomic ID and count, whereas RMA6 has taxonomic lineage information).
+
+You will only receive the `.sam` and `.megan` files if you supply `--malt_save_reads` and/or `--malt_generate_megansummary` parameters to the pipeline.
+
+### MetaPhlAn
+
+[MetaPhlAn](https://github.com/biobakery/metaphlan) is a computational tool for profiling the composition of microbial communities (Bacteria, Archaea and Eukaryotes) from metagenomic shotgun sequencing data (i.e. not 16S) with species-level resolution via marker genes.
+
+Output files
+ +- `malt/` + - `
+_combined_reports.txt`: A combined profile of all samples aligned to a given database (as generated by `metaphlan_merge_tables`)
+ - `/`
+ - `.biom`: taxonomic profile in BIOM format
+ - `.bowtie2out.txt`: BowTie2 alignment information (can be re-used for skipping alignment when re-running MetaPhlAn with different parameters)
+ - `_profile.txt`: MetaPhlAn taxonomic profile including abundance estimates
+
+
+
+The output contains a file named `*_combined_reports.txt`, which provides an overview of the classification results for all samples. The main taxonomic profiling file from MetaPhlAn is the `*_profile.txt` file. This provides the abundance estimates from MetaPhlAn however does not include raw counts by default. Additionally, it contains intermediate Bowtie2 output `.bowtie2out.txt`, which presents a condensed representation of the mapping results of your sequencing reads to MetaPhlAn's marker gene sequences. The alignments are listed in tab-separated columns, including Read ID and Marker Gene ID, with each alignment represented on a separate line.
+
+### mOTUs
+
+[mOTUS](https://github.com/motu-tool/mOTUs) is a taxonomic profiler that maps reads to a unique marker specific database and estimates the relative abundance of known and unknown species.
+
+Output files
+ +- `metaphlan/` + - `metaphlan_
+/`
+ - `.log`: A log file that contains summary statistics
+ - `.out`: A classification file that summarises taxonomic identifiers, by default at the rank of mOTUs (i.e., species level), and their relative abundances in the profiled sample.
+ - `motus__combined_reports.txt`: A combined profile of all samples aligned to a given database (as generated by `motus_merge`)
+Normally `*_combined_reports.txt` is the most useful file for downstream analyses, but the per sample `.out` file can provide additional more specific information. By default, nf-core/taxprofiler is providing a column describing NCBI taxonomic ID as this is used in the taxpasta step. You can disable this column by activating the argument `--motus_remove_ncbi_ids`.
+You will receive the relative abundance instead of read counts if you provide the argument `--motus_use_relative_abundance`.
+
+### KMCP
+
+[KMCP](https://github.com/shenwei356/kmcp) utilises genome coverage information by splitting the reference genomes into chunks and stores k-mers in a modified and optimised COBS index for fast alignment-free sequence searching. KMCP combines k-mer similarity and genome coverage information to reduce the false positive rate of k-mer-based taxonomic classification and profiling methods.
+
+Output files
+ +- `motus/` + - `
+/`
+ - `.gz`: output of `kmcp_search` containing search sequences against a database in tab-delimited format with 15 columns.
+ - `_kmcp.profile`: output of `kmcp_profile` containing the taxonomic profile from search results.
+
+
+
+You will receive the `Output files
+ +- `kmcp/` + + - `
+/`
+
+ - `_report.tre`: output of `ganon report` containing taxonomic classifications with possible formatting and/or filtering depending on options specified.
+ - ``.tre: output of `ganon classify` containing raw taxonomic classifications and abundance estimations with no additional formatting or filtering.
+ - ``.rep: 'raw' report of counts against each taxon.
+ - ``.all: per-read summary of all hits of each reads.
+ - ``.lca: per-read summary of the best single hit after LCA for each read.
+ - ``.unc: list of read IDs with no hits.
+ - ``.log: the stdout console messages printed by `ganon classify`, containing some classification summary information
+
+ - `ganon__combined_reports.txt`: A combined profile of all samples aligned to a given database (as generated by `ganon table`)
+
+
+
+Generally you will want to refer to the `combined_reports.txt` or `_report.tre` file. For further descriptions of the contents of each file, see the [ganon documentation](https://pirovc.github.io/ganon/outputfiles/).
+
+You will only receive the `.all`, `.lca`, and `.unc` files if you supply the `--ganon_save_readclassifications` parameter to the pipeline.
+
+### Krona
+
+[Krona](https://github.com/marbl/Krona) allows the exploration of (metagenomic) hierarchical data with interactive zooming, multi-layered pie charts.
+
+Krona charts will be generated by the pipeline for supported tools (Kraken2, Centrifuge, Kaiju, and MALT)
+
+Output files
+ +- `ganon/` + + - `
+_.html`: per-tool/per-database interactive HTML file containing hierarchical piecharts
+
+
+
+The resulting HTML files can be loaded into your web browser for exploration. Each file will have a dropdown to allow you to switch between each sample aligned against the given database of the tool.
+
+### TAXPASTA
+
+[TAXPASTA](https://github.com/taxprofiler/taxpasta) standardises and optionally merges two or more taxonomic profiles across samples into one single table. It supports multiple different classifiers simplifying comparison of taxonomic classification results between tools and databases.
+
+Output files
+ +- `krona/` + - `
+_*.{tsv,csv,arrow,parquet,biom}`: Standardised taxon table containing multiple samples. The standard format is the `tsv`.
+ - The first column describes the taxonomy ID and the rest of the columns describe the read counts for each sample.
+ - Note that the file naming scheme will apply regardless of whether `TAXPASTA_MERGE` (multiple sample run) or `TAXPASTA_STANDARDISE` (single sample run) are executed.
+ - If you have also run Bracken, the initial Kraken report (i.e., _before_ read re-assignment) will also be included in this directory with `-bracken` suffixed to your Bracken database name. For example: `kraken2--bracken.tsv`. However in most cases you want to use the actual Bracken file (i.e., `bracken_.tsv`).
+
+
+
+By providing the path to a directory containing taxdump files to `--taxpasta_taxonomy_dir`, the taxon name, the taxon rank, the taxon's entire lineage including taxon names and/or the taxon's entire lineage including taxon identifiers can also be added in the output in addition to just the taxon ID. Addition of this extra information can be turned by using the parameters `--taxpasta_add_name`, `--taxpasta_add_rank`, `--taxpasta_add_lineage` and `--taxpasta_add_idlineage` respectively.
+
+These files will likely be the most useful files for the comparison of differences in classification between different tools or building consensuses, with the caveat they have slightly less information than the actual output from each tool (which may have non-standard information e.g. taxonomic rank, percentage of hits, abundance estimations).
+
+The following report files are used for the taxpasta step:
+
+- Bracken: `Output files
+ +- `taxpasta/` + + - `Output files
-* `multiqc/` - * `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser. - * `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline. - * `multiqc_plots/`: directory containing static images from the report in various formats. +- `multiqc/` + - `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser. + - `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline. + - `multiqc_plots/`: directory containing static images from the report in various formats.Output files
-* `pipeline_info/` - * Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. - * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.tsv`. +- `pipeline_info/` + - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. + - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. + - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. + - Parameters used by the pipeline run: `params.json`.
+
+
+### Prokka annotation
+
+Provided assemblies are automatic annotated using prokka.
+
+Output files description
+ +`NO_GROUP/kmer` + - database.filtered_XX: Contains identifiers for plasmids identified by Mash that exhibit a distance value greater than a specified threshold (e.g., 0.95). + - database.filtered_XX_term.XX.clusters.tab: A tabulated file listing the clusters of plasmids grouped based on their genetic similarities. + - database.filtered_XX_term.XX.representative.fasta: The FASTA formatted sequence file of the longests plasmids selected as representatives for each cluster. + - database.filtered_XX_term.XX.representative.fasta.*.bt2: Bowtie2 index files for the representative FASTA sequences. + - database.filtered_XX_term.fasta: the FASTA formatted sequences from the plasmids in database.filtered_XX. + - database.filtered_XX_term.mash.distances.tab: Tabulated data of Mash-calculated distances between the filtered plasmid sequences, used for clustering. + - database.msh: The Mash sketch file of the database, which is a compact binary representation of the set of plasmids used for quick distance estimation. + - database.screen.tab: Output file listing the results of the Mash screen operation, which compares the sample against the database to find matching plasmids. +
+
+
+### Mapping againts found plasmids
+
+Once we have selected the representative plasmids that may be present in the sample, Bowtie2 is employed to map the raw sequencing reads against these plasmid sequences in FASTA format. Plasmids that achieve more than 80% coverage are retained for further analysis. Coverage metrics are calculated and recorded in temporary output files.
+
+Output files description
+ +`NO_GROUP/database` +Prokka output files can be found [here](https://github.com/tseemann/prokka?tab=readme-ov-file#output-files) + - SAMPLE_NAME.err + - SAMPLE_NAME.fna + - SAMPLE_NAME.gff + - SAMPLE_NAME.gff.renamed + - SAMPLE_NAME.gff.bed: gff in bed format + - SAMPLE_NAME.gff.reverse.bed: only reverse genes + - SAMPLE_NAME.gff.forward.bed: only forward genes + - SAMPLE_NAME.sqn + - SAMPLE_NAME.txt + - SAMPLE_NAME.faa + - SAMPLE_NAME.fsa + - SAMPLE_NAME.tbl + - SAMPLE_NAME.ffn + - SAMPLE_NAME.gbk + - SAMPLE_NAME.log + - SAMPLE_NAME.tsv +
+
+
+### PlasmidID data for circos
+
+Blast is employed to annotate selected databases by comparing them against the assemblies. Additionally, each contig within the assembly is aligned with the identified plasmids. The necessary files for Circos visualization are created in the data folder.
+
+Output files description
+ +`NO_GROUP/mapping` + - SAMPLE_NAME.coverage: Contains initial coverage bedgraph data for each plasmid + - SAMPLE_NAME.coverage_adapted: Adjusted coverage mean for each plasmid + - SAMPLE_NAME.coverage_adapted_clustered: adjusted coverage mean filtered with more than 80% coverage. + - SAMPLE_NAME.coverage_adapted_clustered_ac: identificator of filtered plasmids + - SAMPLE_NAME.coverage_adapted_clustered_percentage: coverage data for each plasmid in percentaje (1-value) + - SAMPLE_NAME.coverage_adapted_filtered_80: Lists plasmids with coverage exceeding 80%, selected for subsequent analysis. + - SAMPLE_NAME.coverage_adapted_filtered_80_term.fasta: FASTA formatted file containing sequences of plasmids with more than 80% coverage. + - SAMPLE_NAME.coverage_adapted_filtered_80_term.fasta.blast.tmp.*: Temporary BLAST files for sequences that have passed the 80% coverage threshold, used for further comparative analysis. + - SAMPLE_NAME.sorted.bam: BAM file of aligned reads sorted by coordinates. + - SAMPLE_NAME.sorted.bam.bai: Index file for the sorted BAM file, facilitating faster data retrieval. +
+
+
+### Circos images
+
+Circos is used for creating one image for each identified plasmid and a summary image with all the plasmids identified in one figure. A manual for image interpretation can be found [here](https://github.com/BU-ISCIII/plasmidID/wiki/Understanding-the-image:-track-by-track) and a manual about how to select the correct plasmid can be found [here](https://github.com/BU-ISCIII/plasmidID/wiki/How-to-chose-the-right-plasmids).
+
+
+
+
+Output files description
+ +`NO_GROUP/SAMPLE_NAME/data` + - pID_highlights.conf: genes highlights for circos. + - pID_text_annotation.coordinates: text annotation coordinates for circos. + - SAMPLE_NAME.bedgraph: bedgraph coverage for each plasmid + - SAMPLE_NAME.bedgraph_term: filtered bedgraph coverage for each plasmid + - SAMPLE_NAME.DB.bed: blast result in bed format + - SAMPLE_NAME.DB.blast: for each annotation database blast result against the assembly + - SAMPLE_NAME.DB.coordinates: blast result with the coordinates needed for the circos image + - SAMPLE_NAME.fna.blast.tmp.*: blast tmp database files + - SAMPLE_NAME.gff.forward.coordinates: gff coordinates for forward genes for annotation track + - SAMPLE_NAME.gff.reverse.coordinates: gff coordinates for reverse genes for annotation track + - SAMPLE_NAME.karyotype_individual.txt: karyotype template for each plasmid individual image + - SAMPLE_NAME.karyotype_summary.txt: karyotype circos file for summary image + - SAMPLE_NAME.plasmids.bed: blast result plasmids in bed format. + - SAMPLE_NAME.plasmids.blast: blast result contigs against identified plasmids. + - SAMPLE_NAME.plasmids.blast.links: blast result for links for contigs that match different plasmids. + - SAMPLE_NAME.plasmids.complete: complete track information for citcos + - SAMPLE_NAME.plasmids.links: links for contigs that match different plasmids. +
+
+
+## Reconstructed plasmid sequences
+
+A multifasta file is created for each plasmid including all the contig sequences that have matched the identified contig.
+
+Output files description
+ +`NO_GROUP/images` + +- SAMPLE_NAME_PLASMID_individual.circos.conf: circos conf file used for generating the individual image +- SAMPLE_NAME_PLASMID.png: circos image for individual plasmidID +- SAMPLE_NAME_summary.circos.conf: circos conf file used for genering the summary image +- SAMPLE_NAME_summary.png: summary image + +
+
+
+### Summary report
+
+A summary report consolidating all samples in the analysis is created.
+
+Output files description
+ +`NO_GROUP/fasta_files` + +- PLASMID_term.fasta: multifasta file for each plasmid identified in the sample. + +
+
diff --git a/bu_isciii/assets/reports/md/rnaseq_deg.md b/bu_isciii/assets/reports/md/rnaseq_deg.md
new file mode 100755
index 00000000..c3157323
--- /dev/null
+++ b/bu_isciii/assets/reports/md/rnaseq_deg.md
@@ -0,0 +1,781 @@
+
+# mRNAseq (DEG): Output
+
+## Introduction
+
+This document describes the output produced by the pipeline.
+
+
+The directories listed below will be created in the results directory (`01-${DATE}_rnaseq`) after the pipeline has finished. All paths are relative to the top-level results directory.
+
+## Pipeline overview
+
+The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps:
+
+- [Preprocessing](#preprocessing)
+ - [cat](#cat) - Merge re-sequenced FastQ files
+ - [FastQC](#fastqc) - Raw read QC
+ - [UMI-tools extract](#umi-tools-extract) - UMI barcode extraction
+ - [TrimGalore](#trimgalore) - Adapter and quality trimming
+ - [BBSplit](#bbsplit) - Removal of genome contaminants
+ - [SortMeRNA](#sortmerna) - Removal of ribosomal RNA
+- [Alignment and quantification](#alignment-and-quantification)
+ - [STAR and Salmon](#star-and-salmon) - Fast spliced aware genome alignment and transcriptome quantification
+ - [STAR via RSEM](#star-via-rsem) - Alignment and quantification of expression levels
+ - [HISAT2](#hisat2) - Memory efficient splice aware alignment to a reference
+- [Alignment post-processing](#alignment-post-processing)
+ - [SAMtools](#samtools) - Sort and index alignments
+ - [UMI-tools dedup](#umi-tools-dedup) - UMI-based deduplication
+ - [picard MarkDuplicates](#picard-markduplicates) - Duplicate read marking
+- [Other steps](#other-steps)
+ - [StringTie](#stringtie) - Transcript assembly and quantification
+ - [BEDTools and bedGraphToBigWig](#bedtools-and-bedgraphtobigwig) - Create bigWig coverage files
+- [Quality control](#quality-control)
+ - [RSeQC](#rseqc) - Various RNA-seq QC metrics
+ - [Qualimap](#qualimap) - Various RNA-seq QC metrics
+ - [dupRadar](#dupradar) - Assessment of technical / biological read duplication
+ - [Preseq](#preseq) - Estimation of library complexity
+ - [featureCounts](#featurecounts) - Read counting relative to gene biotype
+ - [DESeq2](#deseq2) - PCA plot and sample pairwise distance heatmap and dendrogram
+ - [MultiQC](#multiqc) - Present QC for raw reads, alignment, read counting and sample similiarity
+- [Pseudo-alignment and quantification](#pseudo-alignment-and-quantification)
+ - [Salmon](#salmon) - Wicked fast gene and isoform quantification relative to the transcriptome
+- [Workflow reporting and genomes](#workflow-reporting-and-genomes)
+ - [Reference genome files](#reference-genome-files) - Saving reference genome indices/files
+ - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution
+
+## Preprocessing
+
+### cat
+
+Output files description
+ +`NO_GROUP` + +- `NO_GROUP_final_results.html`: report with same info as table below that can be viewed using chrome. +- `NO_GROUP_final_results.tab`: plasmid info for each sample. + +
+
+
+If multiple libraries/runs have been provided for the same sample in the input samplesheet (e.g. to increase sequencing depth) then these will be merged at the very beginning of the pipeline in order to have consistent sample naming throughout the pipeline. Please refer to the [usage documentation](https://nf-co.re/rnaseq/usage#samplesheet-input) to see how to specify these samples in the input samplesheet.
+
+### FastQC
+
+Output files
+ +- `fastq/` + - `*.merged.fastq.gz`: If `--save_merged_fastq` is specified, concatenated FastQ files will be placed in this directory. + +
+
+
+[FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/).
+
+
+
+
+
+
+
+### UMI-tools extract
+
+Output files
+ +- `fastqc/` + - `*_fastqc.html`: FastQC report containing quality metrics. + - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. + +> **NB:** The FastQC plots in this directory are generated relative to the raw, input reads. They may contain adapter sequence and regions of low quality. To see how your reads look after adapter and quality trimming please refer to the FastQC reports in the `trimgalore/fastqc/` directory. + +
+
+
+[UMI-tools](https://github.com/CGATOxford/UMI-tools) deduplicates reads based on unique molecular identifiers (UMIs) to address PCR-bias. Firstly, the UMI-tools `extract` command removes the UMI barcode information from the read sequence and adds it to the read name. Secondly, reads are deduplicated based on UMI identifier after mapping as highlighted in the [UMI-tools dedup](#umi-tools-dedup) section.
+
+To facilitate processing of input data which has the UMI barcode already embedded in the read name from the start, `--skip_umi_extract` can be specified in conjunction with `--with_umi`.
+
+### TrimGalore
+
+Output files
+ +- `umitools/` + - `*.fastq.gz`: If `--save_umi_intermeds` is specified, FastQ files **after** UMI extraction will be placed in this directory. + - `*.log`: Log file generated by the UMI-tools `extract` command. + +
+
+
+[Trim Galore!](https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/) is a wrapper tool around Cutadapt and FastQC to peform quality and adapter trimming on FastQ files. By default, Trim Galore! will automatically detect and trim the appropriate adapter sequence.
+
+> **NB:** TrimGalore! will only run using multiple cores if you are able to use more than > 5 and > 6 CPUs for single- and paired-end data, respectively. The total cores available to TrimGalore! will also be capped at 4 (7 and 8 CPUs in total for single- and paired-end data, respectively) because there is no longer a run-time benefit. See [release notes](https://github.com/FelixKrueger/TrimGalore/blob/master/Changelog.md#version-060-release-on-1-mar-2019) and [discussion whilst adding this logic to the nf-core/atacseq pipeline](https://github.com/nf-core/atacseq/pull/65).
+
+
+
+### BBSplit
+
+Output files
+ +- `trimgalore/` + - `*.fq.gz`: If `--save_trimmed` is specified, FastQ files **after** adapter trimming will be placed in this directory. + - `*_trimming_report.txt`: Log file generated by Trim Galore!. +- `trimgalore/fastqc/` + - `*_fastqc.html`: FastQC report containing quality metrics for read 1 (_and read2 if paired-end_) **after** adapter trimming. + - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. + +
+_.fastq.gz" where `` is the first column in `--bbsplit_fasta_list` that needs to be provided to initially build the index.
+ - `*.txt`: File containing statistics on how many reads were assigned to each reference.
+
+
+
+[BBSplit](http://seqanswers.com/forums/showthread.php?t=41288) is a tool that bins reads by mapping to multiple references simultaneously, using BBMap. The reads go to the bin of the reference they map to best. There are also disambiguation options, such that reads that map to multiple references can be binned with all of them, none of them, one of them, or put in a special "ambiguous" file for each of them.
+
+This functionality would be especially useful, for example, if you have [mouse PDX](https://en.wikipedia.org/wiki/Patient_derived_xenograft) samples that contain a mixture of human and mouse genomic DNA/RNA and you would like to filter out any mouse derived reads.
+
+The BBSplit index will have to be built at least once with this pipeline by providing [`--bbsplit_fasta_list`](https://nf-co.re/rnaseq/parameters#bbsplit_fasta_list) which has to be a file containing 2 columns: short name and full path to reference genome(s):
+
+```bash
+mm10,/path/to/mm10.fa
+ecoli,/path/to/ecoli.fa
+sarscov2,/path/to/sarscov2.fa
+```
+
+You can save the index by using the [`--save_reference`](https://nf-co.re/rnaseq/parameters#save_reference) parameter and then provide it via [`--bbsplit_index`](https://nf-co.re/rnaseq/parameters#bbsplit_index) for future runs. As described in the `Output files` dropdown box above the FastQ files relative to the main reference genome will always be called `*primary*.fastq.gz`.
+
+### SortMeRNA
+
+Output files
+ +- `bbsplit/` + - `*.fastq.gz`: If `--save_bbsplit_reads` is specified FastQ files split by reference will be saved to the results directory. Reads from the main reference genome will be named "_primary_.fastq.gz". Reads from contaminating genomes will be named "_
+
+
+When `--remove_ribo_rna` is specified, the pipeline uses [SortMeRNA](https://github.com/biocore/sortmerna) for the removal of ribosomal RNA. By default, [rRNA databases](https://github.com/biocore/sortmerna/tree/master/data/rRNA_databases) defined in the SortMeRNA GitHub repo are used. You can see an example in the pipeline Github repository in `assets/rrna-default-dbs.txt` which is used by default via the `--ribo_database_manifest` parameter. Please note that commercial/non-academic entities require [`licensing for SILVA`](https://www.arb-silva.de/silva-license-information) for these default databases.
+
+
+
+## Alignment and quantification
+
+### STAR and Salmon
+
+Output files
+ +- `sortmerna/` + - `*.fastq.gz`: If `--save_non_ribo_reads` is specified, FastQ files containing non-rRNA reads will be placed in this directory. + - `*.log`: Log file generated by SortMeRNA with information regarding reads that matched the reference database(s). + +
+
+
+[STAR](https://github.com/alexdobin/STAR) is a read aligner designed for splice aware mapping typical of RNA sequencing data. STAR stands for *S*pliced *T*ranscripts *A*lignment to a *R*eference, and has been shown to have high accuracy and outperforms other aligners by more than a factor of 50 in mapping speed, but it is memory intensive. Using `--aligner star_salmon` is the default alignment and quantification option.
+
+[Salmon](https://salmon.readthedocs.io/en/latest/salmon.html) from [Ocean Genomics](https://oceangenomics.com/) is a tool for wicked-fast transcript quantification from RNA-seq data. It requires a set of target transcripts (either from a reference or de-novo assembly) in order to perform quantification. All you need to run Salmon is a FASTA file containing your reference transcripts and a set of FASTA/FASTQ/BAM file(s) containing your reads. The transcriptome-level BAM files generated by STAR are provided to Salmon for downstream quantification. You can of course also provide FASTQ files directly as input to Salmon in order to pseudo-align and quantify your data by providing the `--pseudo_aligner salmon` parameter. The results generated by the pipeline are exactly the same whether you provide BAM or FASTQ input so please see the [Salmon](#salmon) results section for more details.
+
+The STAR section of the MultiQC report shows a bar plot with alignment rates: good samples should have most reads as _Uniquely mapped_ and few _Unmapped_ reads.
+
+
+
+### STAR via RSEM
+
+Output files
+ +- `star_salmon/` + - `*.Aligned.out.bam`: If `--save_align_intermeds` is specified the original BAM file containing read alignments to the reference genome will be placed in this directory. + - `*.Aligned.toTranscriptome.out.bam`: If `--save_align_intermeds` is specified the original BAM file containing read alignments to the transcriptome will be placed in this directory. +- `star_salmon/log/` + - `*.SJ.out.tab`: File containing filtered splice junctions detected after mapping the reads. + - `*.Log.final.out`: STAR alignment report containing the mapping results summary. + - `*.Log.out` and `*.Log.progress.out`: STAR log files containing detailed information about the run. Typically only useful for debugging purposes. +- `star_salmon/unmapped/` + - `*.fastq.gz`: If `--save_unaligned` is specified, FastQ files containing unmapped reads will be placed in this directory. + +
+.stat/`
+ - `*.cnt`, `*.model`, `*.theta`: RSEM counts and statistics for each sample.
+ - `star_rsem/log/`
+ - `*.log`: STAR alignment report containing the mapping results summary.
+
+
+
+[RSEM](https://github.com/deweylab/RSEM) is a software package for estimating gene and isoform expression levels from RNA-seq data. It has been widely touted as one of the most accurate quantification tools for RNA-seq analysis. RSEM wraps other popular tools to map the reads to the genome (i.e. STAR, Bowtie2, HISAT2; STAR is used in this pipeline) which are then subsequently filtered relative to a transcriptome before quantifying at the gene- and isoform-level. Other advantages of using RSEM are that it performs both the alignment and quantification in a single package and its ability to effectively use ambiguously-mapping reads.
+
+You can choose to align and quantify your data with RSEM by providing the `--aligner star_rsem` parameter.
+
+
+
+
+
+### HISAT2
+
+Output files
+ +- `star_rsem/` + - `rsem.merged.gene_counts.tsv`: Matrix of gene-level raw counts across all samples. + - `rsem.merged.gene_tpm.tsv`: Matrix of gene-level TPM values across all samples. + - `rsem.merged.transcript_counts.tsv`: Matrix of isoform-level raw counts across all samples. + - `rsem.merged.transcript_tpm.tsv`: Matrix of isoform-level TPM values across all samples. + - `*.genes.results`: RSEM gene-level quantification results for each sample. + - `*.isoforms.results`: RSEM isoform-level quantification results for each sample. + - `*.STAR.genome.bam`: If `--save_align_intermeds` is specified the original BAM file containing read alignments to the reference genome will be placed in this directory. + - `*.transcript.bam`: If `--save_align_intermeds` is specified the original BAM file containing read alignments to the transcriptome will be placed in this directory. +- `star_rsem/
+.bam`: If `--save_align_intermeds` is specified the original BAM file containing read alignments to the reference genome will be placed in this directory.
+- `hisat2/log/`
+ - `*.log`: HISAT2 alignment report containing the mapping results summary.
+- `hisat2/unmapped/`
+ - `*.fastq.gz`: If `--save_unaligned` is specified, FastQ files containing unmapped reads will be placed in this directory.
+
+
+
+[HISAT2](http://daehwankimlab.github.io/hisat2/) is a fast and sensitive alignment program for mapping next-generation sequencing reads (both DNA and RNA) to a population of human genomes as well as to a single reference genome. It introduced a new indexing scheme called a Hierarchical Graph FM index (HGFM) which when combined with several alignment strategies, enable rapid and accurate alignment of sequencing reads. The HISAT2 route through the pipeline is a good option if you have memory limitations on your compute. However, quantification isn't performed if using `--aligner hisat2` due to the lack of an appropriate option to calculate accurate expression estimates from HISAT2 derived genomic alignments. However, you can use this route if you have a preference for the alignment, QC and other types of downstream analysis compatible with the output of HISAT2.
+
+You can choose to align your data with HISAT2 by providing the `--aligner hisat2` parameter.
+
+
+
+## Alignment post-processing
+
+The pipeline has been written in a way where all the files generated downstream of the alignment are placed in the same directory as specified by `--aligner` e.g. if `--aligner star_salmon` is specified then all the downstream results will be placed in the `star_salmon/` directory. This helps with organising the directory structure and more importantly, allows the end-user to get the results from multiple aligners by simply re-running the pipeline with a different `--aligner` option along the `-resume` parameter. It also means that results won't be overwritten when resuming the pipeline and can be used for benchmarking between alignment algorithms if required.
+
+### SAMtools
+
+Output files
+ +- `hisat2/` + - `
+/`
+ - `.sorted.bam`: If `--save_align_intermeds` is specified the original coordinate sorted BAM file containing read alignments will be placed in this directory.
+ - `.sorted.bam.bai`: If `--save_align_intermeds` is specified the BAI index file for the original coordinate sorted BAM file will be placed in this directory.
+ - `.sorted.bam.csi`: If `--save_align_intermeds --bam_csi_index` is specified the CSI index file for the original coordinate sorted BAM file will be placed in this directory.
+- `/samtools_stats/`
+ - SAMtools `.sorted.bam.flagstat`, `.sorted.bam.idxstats` and `.sorted.bam.stats` files generated from the alignment files.
+
+
+
+The original BAM files generated by the selected alignment algorithm are further processed with [SAMtools](http://samtools.sourceforge.net/) to sort them by coordinate, for indexing, as well as to generate read mapping statistics.
+
+
+
+
+
+### UMI-tools dedup
+
+Output files
+ +- `
+/`
+ - `.umi_dedup.sorted.bam`: If `--save_umi_intermeds` is specified the UMI deduplicated, coordinate sorted BAM file containing read alignments will be placed in this directory.
+ - `.umi_dedup.sorted.bam.bai`: If `--save_umi_intermeds` is specified the BAI index file for the UMI deduplicated, coordinate sorted BAM file will be placed in this directory.
+ - `.umi_dedup.sorted.bam.csi`: If `--save_umi_intermeds --bam_csi_index` is specified the CSI index file for the UMI deduplicated, coordinate sorted BAM file will be placed in this directory.
+- `/umitools/`
+ - `*_edit_distance.tsv`: Reports the (binned) average edit distance between the UMIs at each position.
+ - `*_per_umi.tsv`: UMI-level summary statistics.
+ - `*_per_umi_per_position.tsv`: Tabulates the counts for unique combinations of UMI and position.
+
+The content of the files above is explained in more detail in the [UMI-tools documentation](https://umi-tools.readthedocs.io/en/latest/reference/dedup.html#dedup-specific-options).
+
+
+
+After extracting the UMI information from the read sequence (see [UMI-tools extract](#umi-tools-extract)), the second step in the removal of UMI barcodes involves deduplicating the reads based on both mapping and UMI barcode information using the UMI-tools `dedup` command. This will generate a filtered BAM file after the removal of PCR duplicates.
+
+### picard MarkDuplicates
+
+Output files
+ +- `
+/`
+ - `.markdup.sorted.bam`: Coordinate sorted BAM file after duplicate marking. This is the final post-processed BAM file and so will be saved by default in the results directory.
+ - `.markdup.sorted.bam.bai`: BAI index file for coordinate sorted BAM file after duplicate marking. This is the final post-processed BAM index file and so will be saved by default in the results directory.
+ - `.markdup.sorted.bam.csi`: CSI index file for coordinate sorted BAM file after duplicate marking. This is the final post-processed BAM index file and so will be saved by default in the results directory. Only generated if `--bam_csi_index` is specified as a parameter.
+- `/samtools_stats/`
+ - SAMtools `.markdup.sorted.bam.flagstat`, `.markdup.sorted.bam.idxstats` and `.markdup.sorted.bam.stats` files generated from the duplicate marked alignment files.
+- `/picard_metrics/`
+ - `.markdup.sorted.MarkDuplicates.metrics.txt`: Metrics file from MarkDuplicates.
+
+
+
+Unless you are using [UMIs](https://emea.illumina.com/science/sequencing-method-explorer/kits-and-arrays/umi.html) it is not possible to establish whether the fragments you have sequenced from your sample were derived via true biological duplication (i.e. sequencing independent template fragments) or as a result of PCR biases introduced during the library preparation. By default, the pipeline uses [picard MarkDuplicates](https://broadinstitute.github.io/picard/command-line-overview.html#MarkDuplicates) to _mark_ the duplicate reads identified amongst the alignments to allow you to guage the overall level of duplication in your samples. However, for RNA-seq data it is not recommended to physically remove duplicate reads from the alignments (unless you are using UMIs) because you expect a significant level of true biological duplication that arises from the same fragments being sequenced from for example highly expressed genes. This step will be skipped automatically when using the `--with_umi` option or explicitly via the `--skip_markduplicates` parameter.
+
+
+
+## Other steps
+
+### StringTie
+
+Output files
+ +- `
+/stringtie/`
+ - `*.coverage.gtf`: GTF file containing transcripts that are fully covered by reads.
+ - `*.transcripts.gtf`: GTF file containing all of the assembled transcipts from StringTie.
+ - `*.gene_abundance.txt`: Text file containing gene aboundances and FPKM values.
+- `/stringtie/.ballgown/`: Ballgown output directory.
+
+
+
+[StringTie](https://ccb.jhu.edu/software/stringtie/) is a fast and highly efficient assembler of RNA-Seq alignments into potential transcripts. It uses a novel network flow algorithm as well as an optional de novo assembly step to assemble and quantitate full-length transcripts representing multiple splice variants for each gene locus. In order to identify differentially expressed genes between experiments, StringTie's output can be processed by specialized software like [Ballgown](https://github.com/alyssafrazee/ballgown), [Cuffdiff](http://cole-trapnell-lab.github.io/cufflinks/cuffdiff/index.html) or other programs ([DESeq2](https://bioconductor.org/packages/release/bioc/html/DESeq2.html), [edgeR](https://bioconductor.org/packages/release/bioc/html/edgeR.html), etc.).
+
+### BEDTools and bedGraphToBigWig
+
+Output files
+ +- `
+/bigwig/`
+ - `*.forward.bigWig`: bigWig coverage file relative to genes on the forward DNA strand.
+ - `*.reverse.bigWig`: bigWig coverage file relative to genes on the reverse DNA strand.
+
+
+
+The [bigWig](https://genome.ucsc.edu/goldenpath/help/bigWig.html) format is an indexed binary format useful for displaying dense, continuous data in Genome Browsers such as the [UCSC](https://genome.ucsc.edu/cgi-bin/hgTracks) and [IGV](http://software.broadinstitute.org/software/igv/). This mitigates the need to load the much larger BAM files for data visualisation purposes which will be slower and result in memory issues. The bigWig format is also supported by various bioinformatics software for downstream processing such as meta-profile plotting.
+
+## Quality control
+
+### RSeQC
+
+[RSeQC](<(http://rseqc.sourceforge.net/)>) is a package of scripts designed to evaluate the quality of RNA-seq data. This pipeline runs several, but not all RSeQC scripts. You can tweak the supported scripts you would like to run by adjusting the `--rseqc_modules` parameter which by default will run all of the following: `bam_stat.py`, `inner_distance.py`, `infer_experiment.py`, `junction_annotation.py`, `junction_saturation.py`,`read_distribution.py` and `read_duplication.py`.
+
+The majority of RSeQC scripts generate output files which can be plotted and summarised in the MultiQC report.
+
+#### Infer experiment
+
+Output files
+ +- `
+/rseqc/infer_experiment/`
+ - `*.infer_experiment.txt`: File containing fraction of reads mapping to given strandedness configurations.
+
+
+
+This script predicts the "strandedness" of the protocol (i.e. unstranded, sense or antisense) that was used to prepare the sample for sequencing by assessing the orientation in which aligned reads overlay gene features in the reference genome. The strandedness of each sample has to be provided to the pipeline in the input samplesheet (see [usage docs](https://nf-co.re/rnaseq/usage#samplesheet-input)). However, this information is not always available, especially for public datasets. As a result, additional features have been incorporated into this pipeline to auto-detect whether you have provided the correct information in the samplesheet, and if this is not the case then a warning table will be placed at the top of the MultiQC report highlighting the offending samples (see image below). If required, this will allow you to correct the input samplesheet and rerun the pipeline with the accurate strand information. Note, it is important to get this information right because it can affect the final results.
+
+RSeQC documentation: [infer_experiment.py](http://rseqc.sourceforge.net/#infer-experiment-py)
+
+
+
+
+
+#### Read distribution
+
+Output files
+ +- `
+/rseqc/read_distribution/`
+ - `*.read_distribution.txt`: File containing fraction of reads mapping to genome feature e.g. CDS exon, 5’UTR exon, 3’ UTR exon, Intron, Intergenic regions etc.
+
+
+
+This tool calculates how mapped reads are distributed over genomic features. A good result for a standard RNA-seq experiments is generally to have as many exonic reads as possible (`CDS_Exons`). A large amount of intronic reads could be indicative of DNA contamination in your sample but may be expected for a total RNA preparation.
+
+RSeQC documentation: [read_distribution.py](http://rseqc.sourceforge.net/#read-distribution-py)
+
+
+
+#### Junction annotation
+
+Output files
+ +- `
+/rseqc/junction_annotation/bed/`
+ - `*.junction.bed`: BED file containing splice junctions.
+ - `*.junction.Interact.bed`: BED file containing interacting splice junctions.
+- `/rseqc/junction_annotation/log/`
+ - `*.junction_annotation.log`: Log file generated by the program.
+- `/rseqc/junction_annotation/pdf/`
+ - `*.splice_events.pdf`: PDF file containing splicing events plot.
+ - `*.splice_junction.pdf`: PDF file containing splice junctions plot.
+- `/rseqc/junction_annotation/rscript/`
+ - `*.junction_plot.r`: R script used to generate pdf plots above.
+- `/rseqc/junction_annotation/xls/`
+ - `*.junction.xls`: Excel spreadsheet with junction information.
+
+
+
+Junction annotation compares detected splice junctions to a reference gene model. Splicing annotation is performed in two levels: splice event level and splice junction level.
+
+RSeQC documentation: [junction_annotation.py](http://rseqc.sourceforge.net/#junction-annotation-py)
+
+
+
+#### Inner distance
+
+Output files
+ +- `
+/rseqc/inner_distance/pdf/`
+ - `*.inner_distance_plot.pdf`: PDF file containing inner distance plot.
+- `/rseqc/inner_distance/rscript/`
+ - `*.inner_distance_plot.r`: R script used to generate pdf plot above.
+- `/rseqc/inner_distance/txt/`
+ - `*.inner_distance_freq.txt`: File containing frequency of insert sizes.
+ - `*.inner_distance_mean.txt`: File containing mean, median and standard deviation of insert sizes.
+
+
+
+The inner distance script tries to calculate the inner distance between two paired-end reads. It is the distance between the end of read 1 to the start of read 2, and it is sometimes confused with the insert size (see [this blog post](http://thegenomefactory.blogspot.com.au/2013/08/paired-end-read-confusion-library.html) for disambiguation):
+
+This plot will not be generated for single-end data. Very short inner distances are often seen in old or degraded samples (_eg._ FFPE) and values can be negative if the reads overlap consistently.
+
+RSeQC documentation: [inner_distance.py](http://rseqc.sourceforge.net/#inner-distance-py)
+
+
+
+#### Junction saturation
+
+Output files
+ +- `
+/rseqc/junction_saturation/pdf/`
+ - `*.junctionSaturation_plot.pdf`: PDF file containing junction saturation plot.
+- `/rseqc/junction_saturation/rscript/`
+ - `*.junctionSaturation_plot.r`: R script used to generate pdf plot above.
+
+
+
+This script shows the number of splice sites detected within the data at various levels of subsampling. A sample that reaches a plateau before getting to 100% data indicates that all junctions in the library have been detected, and that further sequencing will not yield any more observations. A good sample should approach such a plateau of _Known junctions_, however, very deep sequencing is typically required to saturate all _Novel Junctions_ in a sample.
+
+RSeQC documentation: [junction_saturation.py](http://rseqc.sourceforge.net/#junction-saturation-py)
+
+
+
+#### Read duplication
+
+Output files
+ +- `
+/rseqc/read_duplication/pdf/`
+ - `*.DupRate_plot.pdf`: PDF file containing read duplication plot.
+- `/rseqc/read_duplication/rscript/`
+ - `*.DupRate_plot.r`: R script used to generate pdf plot above.
+- `/rseqc/read_duplication/xls/`
+ - `*.pos.DupRate.xls`: Read duplication rate determined from mapping position of read. First column is “occurrence” or duplication times, second column is number of uniquely mapped reads.
+ - `*.seq.DupRate.xls`: Read duplication rate determined from sequence of read. First column is “occurrence” or duplication times, second column is number of uniquely mapped reads.
+
+
+
+This plot shows the number of reads (y-axis) with a given number of exact duplicates (x-axis). Most reads in an RNA-seq library should have a low number of exact duplicates. Samples which have many reads with many duplicates (a large area under the curve) may be suffering excessive technical duplication.
+
+RSeQC documentation: [read_duplication.py](http://rseqc.sourceforge.net/#read-duplication-py)
+
+
+
+#### BAM stat
+
+Output files
+ +- `
+/rseqc/bam_stat/`
+ - `*.bam_stat.txt`: Mapping statistics for the BAM file.
+
+
+
+This script gives numerous statistics about the aligned BAM files. A typical output looks as follows:
+
+```txt
+#Output (all numbers are read count)
+#==================================================
+Total records: 41465027
+QC failed: 0
+Optical/PCR duplicate: 0
+Non Primary Hits 8720455
+Unmapped reads: 0
+
+mapq < mapq_cut (non-unique): 3127757
+mapq >= mapq_cut (unique): 29616815
+Read-1: 14841738
+Read-2: 14775077
+Reads map to '+': 14805391
+Reads map to '-': 14811424
+Non-splice reads: 25455360
+Splice reads: 4161455
+Reads mapped in proper pairs: 21856264
+Proper-paired reads map to different chrom: 7648
+```
+
+MultiQC plots each of these statistics in a dot plot. Each sample in the project is a dot - hover to see the sample highlighted across all fields.
+
+RSeQC documentation: [bam_stat.py](http://rseqc.sourceforge.net/#bam-stat-py)
+
+#### TIN
+
+Output files
+ +- `
+/rseqc/tin/`
+ - `*.summary.txt`: File containing TIN results summary.
+ - `*.tin.xls`: XLS file containing TIN results.
+
+
+
+This script is designed to evaluate RNA integrity at the transcript level. TIN (transcript integrity number) is named in analogous to RIN (RNA integrity number). RIN (RNA integrity number) is the most widely used metric to evaluate RNA integrity at sample (or transcriptome) level. It is a very useful preventive measure to ensure good RNA quality and robust, reproducible RNA sequencing. This process isn't run by default - please see [this issue](https://github.com/nf-core/rnaseq/issues/769).
+
+RSeQC documentation: [tin.py](http://rseqc.sourceforge.net/#tin-py)
+
+### Qualimap
+
+Output files
+ +- `
+/qualimap//`
+ - `qualimapReport.html`: Qualimap HTML report that can be viewed in a web browser.
+ - `rnaseq_qc_results.txt`: Textual results output.
+- `/qualimap//images_qualimapReport/`: Images required for the HTML report.
+- `/qualimap//raw_data_qualimapReport/`: Raw data required for the HTML report.
+- `/qualimap//css/`: CSS files required for the HTML report.
+
+
+
+[Qualimap](http://qualimap.bioinfo.cipf.es/) is a platform-independent application written in Java and R that provides both a Graphical User Interface (GUI) and a command-line interface to facilitate the quality control of alignment sequencing data. Shortly, Qualimap:
+
+- Examines sequencing alignment data according to the features of the mapped reads and their genomic properties.
+- Provides an overall view of the data that helps to to the detect biases in the sequencing and/or mapping of the data and eases decision-making for further analysis.
+
+The [Qualimap RNA-seq QC module](http://qualimap.bioinfo.cipf.es/doc_html/analysis.html#rna-seq-qc) is used within this pipeline to assess the overall mapping and coverage relative to gene features.
+
+
+
+
+
+### dupRadar
+
+Output files
+ +- `
+/dupradar/box_plot/`
+ - `*_duprateExpBoxplot.pdf`: PDF file containing box plot for duplicate rate relative to mean expression.
+- `/dupradar/gene_data/`
+ - `*_dupMatrix.txt`: Text file containing duplicate metrics per gene.
+- `/dupradar/histogram/`
+ - `*_expressionHist.pdf`: PDF file containing histogram of reads per kilobase values per gene.
+- `/dupradar/intercepts_slope/`
+ - `*_intercept_slope.txt`: Text file containing intercept slope values.
+- `/dupradar/scatter_plot/`
+ - `*_duprateExpDens.pdf`: PDF file containing typical dupRadar 2D density scatter plot.
+
+See [dupRadar docs](https://www.bioconductor.org/packages/devel/bioc/vignettes/dupRadar/inst/doc/dupRadar.html) for further information regarding the content of these files.
+
+
+
+[dupRadar](https://www.bioconductor.org/packages/release/bioc/html/dupRadar.html) is a Bioconductor library written in the R programming language. It generates various QC metrics and plots that relate duplication rate with gene expression levels in order to identify experiments with high technical duplication. A good sample with little technical duplication will only show high numbers of duplicates for highly expressed genes. Samples with technical duplication will have high duplication for all genes, irrespective of transcription level.
+
+
+
+> _Credit: [dupRadar documentation](https://www.bioconductor.org/packages/devel/bioc/vignettes/dupRadar/inst/doc/dupRadar.html)_
+
+### Preseq
+
+Output files
+ +- `
+/preseq/`
+ - `*.lc_extrap.txt`: Preseq expected future yield file.
+- `/preseq/log/`
+ - `*.command.log`: Standard error output from command.
+
+
+
+The [Preseq](http://smithlabresearch.org/software/preseq/) package is aimed at predicting and estimating the complexity of a genomic sequencing library, equivalent to predicting and estimating the number of redundant reads from a given sequencing depth and how many will be expected from additional sequencing using an initial sequencing experiment. The estimates can then be used to examine the utility of further sequencing, optimize the sequencing depth, or to screen multiple libraries to avoid low complexity samples. A shallow curve indicates that the library has reached complexity saturation and further sequencing would likely not add further unique reads. The dashed line shows a perfectly complex library where total reads = unique reads. Note that these are predictive numbers only, not absolute. The MultiQC plot can sometimes give extreme sequencing depth on the X axis - click and drag from the left side of the plot to zoom in on more realistic numbers.
+
+
+
+### featureCounts
+
+Output files
+ +- `
+/featurecounts/`
+ - `*.featureCounts.txt`: featureCounts biotype-level quantification results for each sample.
+ - `*.featureCounts.txt.summary`: featureCounts summary file containing overall statistics about the counts.
+ - `*_mqc.tsv`: MultiQC custom content files used to plot biotypes in report.
+
+
+
+[featureCounts](http://bioinf.wehi.edu.au/featureCounts/) from the [Subread](http://subread.sourceforge.net/) package is a quantification tool used to summarise the mapped read distribution over genomic features such as genes, exons, promotors, gene bodies, genomic bins and chromosomal locations. We can also use featureCounts to count overlaps with different classes of genomic features. This provides an additional QC to check which features are most abundant in the sample, and to highlight potential problems such as rRNA contamination.
+
+
+
+### DESeq2
+
+Output files
+ +- `
+/deseq2_qc/`
+ - `*.plots.pdf`: File containing PCA and hierarchical clustering plots.
+ - `*.dds.RData`: File containing R `DESeqDataSet` object generated
+ by DESeq2, with either an rlog or vst `assay` storing the
+ variance-stabilised data.
+ - `*.rds`: Alternative version of the RData file suitable for
+ `readRDS` to give user control of the eventual object name.
+ - `*pca.vals.txt`: Matrix of values for the first 2 principal components.
+ - `*sample.dists.txt`: Sample distance matrix.
+ - `R_sessionInfo.log`: File containing information about R, the OS and attached or loaded packages.
+- `/deseq2_qc/size_factors/`
+ - `*.txt`, `*.RData`: Files containing DESeq2 sizeFactors per sample.
+
+
+
+[DESeq2](https://bioconductor.org/packages/release/bioc/vignettes/DESeq2/inst/doc/DESeq2.html) is one of the most commonly used software packages to perform differential expression analysis for RNA-seq datasets.
+
+**This pipeline uses a standardised DESeq2 analysis script to get an idea of the reproducibility across samples within the experiment. Please note that this will not suit every experimental design, and if there are other problems with the experiment then it may not work as well as expected.**
+
+The script included in the pipeline uses DESeq2 to normalise read counts across all of the provided samples in order to create a PCA plot and a clustered heatmap showing pairwise Euclidean distances between the samples in the experiment. These help to show the similarity between groups of samples and can reveal batch effects and other potential issues with the experiment.
+
+By default, the pipeline uses the `vst` transformation which is more suited to larger experiments. You can set the parameter `--deseq2_vst false` if you wish to use the DESeq2 native `rlog` option. See [DESeq2 docs](http://bioconductor.org/packages/devel/bioc/vignettes/DESeq2/inst/doc/DESeq2.html#data-transformations-and-visualization) for a more detailed explanation.
+
+The PCA plots are generated based alternately on the top five hundred most variable genes, or all genes. The former is the conventional approach that is more likely to pick up strong effects (ie the biological signal) and the latter, when different, is picking up a weaker but consistent effect that is synchronised across many transcripts. We project both of these onto the first two PCs (shown in the top row of the figure below), which is the best two dimensional representation of the variation between samples.
+
+We also explore higher components in terms of experimental factors inferred from sample names. If your sample naming convention follows a strict policy of using underscores to delimit values of experimental factors (for example `WT_UNTREATED_REP1`) and all names have the same number of underscores (so excluding `WT_TREATED_10ml_REP1` from being compatible with the previous label), then any of these factors that are informative (ie label some but not all samples the same) then we individually plot upto the first five PCs, per experimental level, for each of the experimental factors.
+
+The plot on the left hand side shows the standard PC plot - notice the variable number of underscores, meaning that the central plot would not be produced: here we have changed the underscore that is hyphenating the treatment to a '-' character. This allows the central plot to be generated, and we can see that replicate (the 2nd part of the sample name) seems to be affecting the 3rd principal component, but the treatment factor is affecting the more important first two components. The right-most plot shows all pairwise euclidean distances between the samples.
+
+Output files
+ +- `
+/`
+ - `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser.
+ - `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline.
+
+
+
+[MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory.
+
+Results generated by MultiQC collate pipeline QC from supported tools i.e. FastQC, Cutadapt, SortMeRNA, STAR, RSEM, HISAT2, Salmon, SAMtools, Picard, RSeQC, Qualimap, Preseq and featureCounts. Additionally, various custom content has been added to the report to assess the output of dupRadar, DESeq2 and featureCounts biotypes, and to highlight samples failing a mimimum mapping threshold or those that failed to match the strand-specificity provided in the input samplesheet. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see Output files
+ +- `multiqc/
+/`
+ - `aux_info/`: Auxiliary info e.g. versions and number of mapped reads.
+ - `cmd_info.json`: Information about the Salmon quantification command, version and options.
+ - `lib_format_counts.json`: Number of fragments assigned, unassigned and incompatible.
+ - `libParams/`: Contains the file `flenDist.txt` for the fragment length distribution.
+ - `logs/`: Contains the file `salmon_quant.log` giving a record of Salmon's quantification.
+ - `quant.genes.sf`: Salmon _gene_-level quantification of the sample, including feature length, effective length, TPM, and number of reads.
+ - `quant.sf`: Salmon _transcript_-level quantification of the sample, including feature length, effective length, TPM, and number of reads.
+
+
+
+As described in the [STAR and Salmon](#star-and-salmon) section, you can choose to pseudo-align and quantify your data with [Salmon](https://salmon.readthedocs.io/en/latest/salmon.html) by providing the `--pseudo_aligner salmon` parameter. By default, Salmon is run in addition to the standard alignment workflow defined by `--aligner`, mainly because it allows you to obtain QC metrics with respect to the genomic alignments. However, you can provide the `--skip_alignment` parameter if you would like to run Salmon in isolation. If Salmon is run in isolation, the outputs mentioned above will be found in a folder named `salmon`. If Salmon is run alongside STAR, the folder will be named `star_salmon`.
+
+Transcripts with large inferential uncertainty won't be assigned the exact number of reads reproducibly, every time Salmon is run. Read more about this on the [nf-core/rnaseq](https://github.com/nf-core/rnaseq/issues/585) and [salmon](https://github.com/COMBINE-lab/salmon/issues/613) Github repos.
+
+The [tximport](https://bioconductor.org/packages/release/bioc/html/tximport.html) package is used in this pipeline to summarise the results generated by Salmon into matrices for use with downstream differential analysis packages. We use tximport with different options to summarize count and TPM quantifications at the gene- and transcript-level. Please see [#499](https://github.com/nf-core/rnaseq/issues/499) for discussion and links regarding which counts are suitable for different types of analysis.
+
+According to the `txtimport` documentation you can do one of the following:
+
+- Use bias corrected counts with an offset: import all the salmon files with `tximport` and then use `DESeq2` with `dds <- DESeqDataSetFromTximport(txi, sampleTable, ~condition)` to correct for changes to the average transcript length across samples.
+- Use bias corrected counts without an offset: load and use `salmon.merged.gene_counts_length_scaled.tsv` or `salmon.merged.gene_counts_scaled.tsv` directly as you would with a regular counts matrix.
+- Use bias uncorrected counts: load and use the `txi$counts` matrix (or `salmon.merged.gene_counts.tsv`) with `DESeq2`. This does not correct for potential differential isoform usage. Alternatively, if you have 3’ tagged RNA-seq data this is the most suitable method.
+
+> **NB:** The default Salmon parameters and a k-mer size of 31 are used to create the index. As [documented here](https://salmon.readthedocs.io/en/latest/salmon.html#preparing-transcriptome-indices-mapping-based-mode) and [discussed here](https://github.com/COMBINE-lab/salmon/issues/482#issuecomment-583799668), a k-mer size off 31 works well with reads that are 75bp or longer.
+
+
+
+## Workflow reporting and genomes
+
+### Reference genome files
+
+Output files
+ +- `salmon/` + - `salmon.merged.gene_counts.tsv`: Matrix of gene-level raw counts across all samples. + - `salmon.merged.gene_tpm.tsv`: Matrix of gene-level TPM values across all samples. + - `salmon.merged.gene_counts.rds`: RDS object that can be loaded in R that contains a [SummarizedExperiment](https://bioconductor.org/packages/release/bioc/html/SummarizedExperiment.html) container with the TPM (`abundance`), estimated counts (`counts`) and transcript length (`length`) in the assays slot for genes. + - `salmon.merged.gene_counts_scaled.tsv`: Matrix of gene-level library size-scaled counts across all samples. + - `salmon.merged.gene_counts_scaled.rds`: RDS object that can be loaded in R that contains a [SummarizedExperiment](https://bioconductor.org/packages/release/bioc/html/SummarizedExperiment.html) container with the TPM (`abundance`), estimated library size-scaled counts (`counts`) and transcript length (`length`) in the assays slot for genes. + - `salmon.merged.gene_counts_length_scaled.tsv`: Matrix of gene-level length-scaled counts across all samples. + - `salmon.merged.gene_counts_length_scaled.rds`: RDS object that can be loaded in R that contains a [SummarizedExperiment](https://bioconductor.org/packages/release/bioc/html/SummarizedExperiment.html) container with the TPM (`abundance`), estimated length-scaled counts (`counts`) and transcript length (`length`) in the assays slot for genes. + - `salmon.merged.transcript_counts.tsv`: Matrix of isoform-level raw counts across all samples. + - `salmon.merged.transcript_tpm.tsv`: Matrix of isoform-level TPM values across all samples. + - `salmon.merged.transcript_counts.rds`: RDS object that can be loaded in R that contains a [SummarizedExperiment](https://bioconductor.org/packages/release/bioc/html/SummarizedExperiment.html) container with the TPM (`abundance`), estimated isoform-level raw counts (`counts`) and transcript length (`length`) in the assays slot for transcripts. + - `salmon_tx2gene.tsv`: Tab-delimited file containing gene to transcripts ids mappings. +- `salmon/
+
+
+A number of genome-specific files are generated by the pipeline because they are required for the downstream processing of the results. If the `--save_reference` parameter is provided then these will be saved in the `genome/` directory. It is recommended to use the `--save_reference` parameter if you are using the pipeline to build new indices so that you can save them somewhere locally. The index building step can be quite a time-consuming process and it permits their reuse for future runs of the pipeline to save disk space.
+
+### Pipeline information
+
+Output files
+ +- `genome/` + - `*.fa`, `*.gtf`, `*.gff`, `*.bed`, `.tsv`: If the `--save_reference` parameter is provided then all of the genome reference files will be placed in this directory. +- `genome/index/` + - `star/`: Directory containing STAR indices. + - `hisat2/`: Directory containing HISAT2 indices. + - `rsem/`: Directory containing STAR and RSEM indices. + - `salmon/`: Directory containing Salmon indices. + +
+
+
+[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage.
+
+# Differential expression analysis with DESeq2: Output
+
+After nf-core/rnaseq pipeline is completed, then custom differental expression anlysis with DESeq2 is performed.
+
+The directories listed below will be created in the differential expression analysis results directory (`02-differential_expression`). All paths are relative to the top-level results directory.
+
+## Output files
+
+Output files
+ +- `pipeline_info/` + - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. + - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. + - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. + +
+/Differential_expression/DESeq2/Differential_expression.csv`: This file contains the results of the differential expression analysis performed using DESeq2, including information on differentially expressed genes and associated statistical metrics such as fold change, p-values, and adjusted p-values.
+
+- `/Differential_expression/DESeq2/heatmapCount_top20_differentially_expressed.pdf`: This PDF file presents a heatmap visualization displaying the expression patterns of the top 20 differentially expressed genes, clustered by sample distance, as determined by the DESeq2 analysis.
+ 
+
+- `/Differential_expression/DESeq2/maPlot_all.pdf`: This PDF file illustrates MA plots depicting the log fold changes (M) versus the mean average (A) expression levels of all genes analyzed in the DESeq2 differential expression analysis.
+
+ 
+
+- `/Differential_expression/DESeq2/pvalues.pdf`: This PDF file provides graphical representations, such as histograms or scatter plots, illustrating the distribution and significance of p-values calculated during the DESeq2 analysis.
+ 
+
+- `/Quality_plots/DESeq2/boxplot.pdf`: This PDF file displays boxplots depicting the distribution of normalized count expression values values across samples, allowing for the assessment of data variability and potential batch effects.
+ 
+
+- `/Quality_plots/DESeq2/cluster_dendrogram.pdf`: This PDF file presents a dendrogram visualization illustrating the hierarchical clustering of samples based on gene expression profiles, enabling the identification of sample similarities and differences.
+ 
+
+- `/Quality_plots/DESeq2/heatmapCount_all_genes.pdf`: This PDF file contains a heatmap visualization showing the expression patterns of all genes analyzed in the experiment, facilitating the identification of gene expression trends and patterns.
+ 
+
+- `/Quality_plots/DESeq2/heatmapCount_top20_highest_expression.pdf`: This PDF file presents a heatmap visualization highlighting the expression patterns of the top 20 genes with the highest expression levels across samples, aiding in the identification of highly expressed genes.
+
+ 
+
+- `/Quality_plots/DESeq2/heatmap_sample_to_sample.pdf`: This PDF file contains a heatmap visualization illustrating the pairwise sample-to-sample correlation matrix based on gene expression profiles, enabling the assessment of sample similarities and reproducibility.
+
+ 
+
+- `/Quality_plots/DESeq2/plotDispersions.pdf`: This PDF file displays dispersion plots showing the relationship between the mean expression levels and the dispersion estimates for each gene, allowing for the assessment of data variability and the adequacy of the statistical model.
+
+ 
+
+- `/Quality_plots/DESeq2/plotPCA.pdf`: This PDF file presents a PCA (Principal Component Analysis) plot visualizing the distribution of samples in a multidimensional space based on their gene expression profiles, allowing for the exploration of sample relationships and potential batch effects.
+ 
+
+- `/Quality_plots/DESeq2/plotSD.pdf`:The standard deviation of the transformed data, across samples, against the mean, using the shifted logarithm transformation, the regularized log transformation and the variance stabilizing transformation. This plot enables the assessment of data variability and the identification of potential outliers.
+
+ 
+
+- `99-stats/Quality_plots/`: This folder contains the same quality plots as described above, but they are generated considering all samples in the service without accounting for the expermiental design specified in DESeq2. This allows for a general overview of the data in the service without incorporating the experimental design.
+
+
diff --git a/bu_isciii/assets/reports/md/sarek.md b/bu_isciii/assets/reports/md/sarek.md
new file mode 100755
index 00000000..783ff1eb
--- /dev/null
+++ b/bu_isciii/assets/reports/md/sarek.md
@@ -0,0 +1,1318 @@
+# nf-core/sarek: Output
+
+## Introduction
+
+This document describes the output produced by the pipeline. Most of the plots are taken from the MultiQC report, which summarises results at the end of the pipeline.
+
+The directories listed below will be created in the results directory after the pipeline has finished. All paths are relative to the top-level results directory.
+
+## Pipeline overview
+
+The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps:
+
+- [Directory Structure](#directory-structure)
+- [Preprocessing](#preprocessing)
+ - [Preparation of input files (FastQ or (u)BAM)](#preparation-of-input-files-fastq-or-ubam)
+ - [Trim adapters](#trim-adapters)
+ - [Split FastQ files](#split-fastq-files)
+ - [UMI consensus](#umi-consensus)
+ - [Map to Reference](#map-to-reference)
+ - [BWA](#bwa)
+ - [BWA-mem2](#bwa-mem2)
+ - [DragMap](#dragmap)
+ - [Sentieon BWA mem](#sentieon-bwa-mem)
+ - [Mark Duplicates](#mark-duplicates)
+ - [GATK MarkDuplicates (Spark)](#gatk-markduplicates-spark)
+ - [Sentieon LocusCollector and Dedup](#sentieon-locuscollector-and-dedup)
+ - [Base Quality Score Recalibration](#base-quality-score-recalibration)
+ - [GATK BaseRecalibrator (Spark)](#gatk-baserecalibrator-spark)
+ - [GATK ApplyBQSR (Spark)](#gatk-applybqsr-spark)
+ - [CSV files](#csv-files)
+- [Variant Calling](#variant-calling)
+ - [SNVs and small indels](#snvs-and-small-indels)
+ - [bcftools](#bcftools)
+ - [DeepVariant](#deepvariant)
+ - [FreeBayes](#freebayes)
+ - [GATK HaplotypeCaller](#gatk-haplotypecaller)
+ - [GATK Germline Single Sample Variant Calling](#gatk-germline-single-sample-variant-calling)
+ - [GATK Joint Germline Variant Calling](#gatk-joint-germline-variant-calling)
+ - [GATK Mutect2](#gatk-mutect2)
+ - [Sentieon DNAscope](#sentieon-dnascope)
+ - [Sentieon DNAscope joint germline variant calling](#sentieon-dnascope-joint-germline-variant-calling)
+ - [Sentieon Haplotyper](#sentieon-haplotyper)
+ - [Sentieon Haplotyper joint germline variant calling](#sentieon-haplotyper-joint-germline-variant-calling)
+ - [Strelka2](#strelka2)
+ - [Structural Variants](#structural-variants)
+ - [Manta](#manta)
+ - [TIDDIT](#tiddit)
+ - [Sample heterogeneity, ploidy and CNVs](#sample-heterogeneity-ploidy-and-cnvs)
+ - [ASCAT](#ascat)
+ - [CNVKit](#cnvkit)
+ - [Control-FREEC](#control-freec)
+ - [Microsatellite instability (MSI)](#microsatellite-instability-msi)
+ - [MSIsensorPro](#msisensorpro)
+ - [Concatenation](#concatenation)
+- [Variant annotation](#variant-annotation)
+ - [snpEff](#snpeff)
+ - [VEP](#vep)
+ - [BCFtools annotate](#bcftools-annotate)
+- [Quality control and reporting](#quality-control-and-reporting)
+ - [Quality control](#quality-control)
+ - [FastQC](#fastqc)
+ - [FastP](#fastp)
+ - [Mosdepth](#mosdepth)
+ - [NGSCheckMate](#ngscheckmate)
+ - [GATK MarkDuplicates reports](#gatk-markduplicates-reports)
+ - [Sentieon Dedup reports](#sentieon-dedup-reports)
+ - [samtools stats](#samtools-stats)
+ - [bcftools stats](#bcftools-stats)
+ - [VCFtools](#vcftools)
+ - [snpEff reports](#snpeff-reports)
+ - [VEP reports](#vep-reports)
+ - [Reporting](#reporting)
+ - [MultiQC](#multiqc)
+ - [Pipeline information](#pipeline-information)
+- [Reference files](#reference-files)
+
+## Directory Structure
+
+The default directory structure is as follows
+
+```
+{outdir}
+├── csv
+├── multiqc
+├── pipeline_info
+├── preprocessing
+│ ├── markduplicates
+│ └── Output files
+ +- `
+`**
+
+- `__{1,2}.fastp.fastq.gz>`
+ - Bgzipped FastQ file
+
+
+
+#### Split FastQ files
+
+[FastP](https://github.com/OpenGene/fastp) supports splitting of one FastQ file into multiple files allowing parallel alignment of sharded FastQ file. To enable splitting, the number of reads per output can be specified. For more information, take a look into the parameter `--split_fastq`in the parameter docs.
+
+These files are intermediate and by default not placed in the output-folder kept in the final files delivered to users. Set `--save_split` to enable publishing of these files to:
+
+Output files for all samples
+ +**Output directory: `{outdir}/preprocessing/fastp/
+/`**
+
+- ``
+ - Bgzipped FastQ file
+
+
+
+#### UMI consensus
+
+Sarek can process UMI-reads, using [fgbio](http://fulcrumgenomics.github.io/fgbio/tools/latest/) tools.
+
+These files are intermediate and by default not placed in the output-folder kept in the final files delivered to users. Set `--save_split` to enable publishing of these files to:
+
+Output files for all samples
+ +**Output directory: `{outdir}/preprocessing/fastp/
+/`**
+
+- ``
+
+**Output directory: `{outdir}/reports/umi/`**
+
+- ``
+
+
+
+### Map to Reference
+
+#### BWA
+
+[BWA](https://github.com/lh3/bwa) is a software package for mapping low-divergent sequences against a large reference genome. The aligned reads are then coordinate-sorted (or name-sorted if [`GATK MarkDuplicatesSpark`](https://gatk.broadinstitute.org/hc/en-us/articles/5358833264411-MarkDuplicatesSpark) is used for duplicate marking) with [samtools](https://www.htslib.org/doc/samtools.html).
+
+#### BWA-mem2
+
+[BWA-mem2](https://github.com/bwa-mem2/bwa-mem2) is a software package for mapping low-divergent sequences against a large reference genome.The aligned reads are then coordinate-sorted (or name-sorted if [`GATK MarkDuplicatesSpark`](https://gatk.broadinstitute.org/hc/en-us/articles/5358833264411-MarkDuplicatesSpark) is used for duplicate marking) with [samtools](https://www.htslib.org/doc/samtools.html).
+
+#### DragMap
+
+[DragMap](https://github.com/Illumina/dragmap) is an open-source software implementation of the DRAGEN mapper, which the Illumina team created so that we would have an open-source way to produce the same results as their proprietary DRAGEN hardware. The aligned reads are then coordinate-sorted (or name-sorted if [`GATK MarkDuplicatesSpark`](https://gatk.broadinstitute.org/hc/en-us/articles/5358833264411-MarkDuplicatesSpark) is used for duplicate marking) with [samtools](https://www.htslib.org/doc/samtools.html).
+
+These files are intermediate and by default not placed in the output-folder kept in the final files delivered to users. Set `--save_mapped` to enable publishing, furthermore add the flag `save_output_as_bam` for publishing in BAM format.
+
+#### Sentieon BWA mem
+
+Sentieon [bwa mem](https://support.sentieon.com/manual/usages/general/#bwa-mem-syntax) is a subroutine for mapping low-divergent sequences against a large reference genome. It is part of the proprietary software package [DNAseq](https://www.sentieon.com/detailed-description-of-pipelines/#dnaseq) from [Sentieon](https://www.sentieon.com/).
+
+The aligned reads are coordinate-sorted with Sentieon.
+
+Output files for all samples
+ +**Output directory: `{outdir}/preprocessing/umi/
+/`**
+
+- if `--save_mapped`: `.sorted.cram` and `.sorted.cram.crai`
+
+ - CRAM file and index
+
+- if `--save_mapped --save_output_as_bam`: `.sorted.bam` and `.sorted.bam.bai`
+ - BAM file and index
+
+
+### Mark Duplicates
+
+During duplicate marking, read pairs that are likely to have originated from duplicates of the same original DNA fragments through some artificial processes are identified. These are considered to be non-independent observations, so all but a single read pair within each set of duplicates are marked, causing the marked pairs to be ignored by default during the variant discovery process.
+
+For further reading and documentation see the [data pre-processing for variant discovery from the GATK best practices](https://gatk.broadinstitute.org/hc/en-us/articles/360035535912-Data-pre-processing-for-variant-discovery).
+
+#### GATK MarkDuplicates (Spark)
+
+By default, Sarek will use [GATK MarkDuplicates](https://gatk.broadinstitute.org/hc/en-us/articles/5358880192027-MarkDuplicates-Picard-).
+
+To use the corresponding spark implementation [`GATK MarkDuplicatesSpark`](https://gatk.broadinstitute.org/hc/en-us/articles/5358833264411-MarkDuplicatesSpark), please specify `--use_gatk_spark markduplicates`. The resulting files are converted to CRAM with either [samtools](https://www.htslib.org/doc/samtools.html), when GATK MarkDuplicates is used, or, implicitly, by GATK MarkDuplicatesSpark.
+
+The resulting CRAM files are delivered to the users.
+
+Output files for all mappers and samples
+ +The alignment files (BAM or CRAM) produced by the chosen aligner are not published by default. CRAM output files will not be saved in the output-folder (`outdir`), unless the flag `--save_mapped` is used. BAM output can be selected by setting the flag `--save_output_as_bam`. + +**Output directory: `{outdir}/preprocessing/mapped/
+/`**
+
+- `.md.cram` and `.md.cram.crai`
+ - CRAM file and index
+- if `--save_output_as_bam`:
+ - `.md.bam` and `.md.bam.bai`
+
+
+
+### Sentieon LocusCollector and Dedup
+
+The subroutines LocusCollector and Dedup are part of Sentieon DNAseq packages with speedup versions of the standard GATK tools, and together those two subroutines correspond to GATK's MarkDuplicates.
+
+The subroutine [LocusCollector](https://support.sentieon.com/manual/usages/general/#driver-algorithm-syntax) collects read information that will be used for removing or tagging duplicate reads; its output is the score file indicating which reads are likely duplicates.
+
+The subroutine [Dedup](https://support.sentieon.com/manual/usages/general/#dedup-algorithm) marks or removes duplicate reads based on the score file supplied by LocusCollector, and produces a BAM or CRAM file.
+
+Output files for all samples
+ +**Output directory: `{outdir}/preprocessing/markduplicates/
+/`**
+
+- `.dedup.cram` and `.dedup.cram.crai`
+ - CRAM file and index
+- if `--save_output_as_bam`:
+ - `.dedup.bam` and `.dedup.bam.bai`
+
+
+
+### Base Quality Score Recalibration
+
+During Base Quality Score Recalibration, systematic errors in the base quality scores are corrected by applying machine learning to detect and correct for them. This is important for evaluating the correct call of a variant during the variant discovery process. However, this is not needed for all combinations of tools in Sarek. Notably, this should be turned off when having UMI tagged reads or using DragMap (see [here](https://gatk.broadinstitute.org/hc/en-us/articles/4407897446939--How-to-Run-germline-single-sample-short-variant-discovery-in-DRAGEN-mode)) as mapper.
+
+For further reading and documentation see the [technical documentation by GATK](https://gatk.broadinstitute.org/hc/en-us/articles/360035890531-Base-Quality-Score-Recalibration-BQSR-).
+
+#### GATK BaseRecalibrator (Spark)
+
+[GATK BaseRecalibrator](https://gatk.broadinstitute.org/hc/en-us/articles/360042477672-BaseRecalibrator) generates a recalibration table based on various co-variates.
+
+To use the corresponding spark implementation [GATK BaseRecalibratorSpark](https://gatk.broadinstitute.org/hc/en-us/articles/5358896138011-BaseRecalibrator), please specify `--use_gatk_spark baserecalibrator`.
+
+Output files for all samples
+ +**Output directory: `{outdir}/preprocessing/sentieon_dedup/
+/`**
+
+- `.recal.table`
+ - Recalibration table associated to the duplicates-marked CRAM file.
+
+
+
+#### GATK ApplyBQSR (Spark)
+
+[GATK ApplyBQSR](https://gatk.broadinstitute.org/hc/en-us/articles/5358826654875-ApplyBQSR) recalibrates the base qualities of the input reads based on the recalibration table produced by the [GATK BaseRecalibrator](#gatk-baserecalibrator) tool.
+
+Specify `--use_gatk_spark baserecalibrator` to use [GATK ApplyBQSRSpark](https://gatk.broadinstitute.org/hc/en-us/articles/5358898266011-ApplyBQSRSpark-BETA-) instead, the respective spark implementation.
+
+The resulting recalibrated CRAM files are delivered to the user. Recalibrated CRAM files are usually 2-3 times larger than the duplicate-marked CRAM files.
+
+Output files for all samples
+ +**Output directory: `{outdir}/preprocessing/recal_table/
+/`**
+
+- `.recal.cram` and `.recal.cram.crai`
+ - CRAM file and index
+- if `--save_output_as_bam`:
+ - `.recal.bam` and `.recal.bam.bai` - BAM file and index
+
+
+### CSV files
+
+The CSV files are auto-generated and can be used by Sarek for further processing and/or variant calling.
+
+See the [`input`](usage#input-sample-sheet-configurations) section in the usage documentation for further reading and documentation on how to make the most of them.
+
+Output files for all samples
+ +**Output directory: `{outdir}/preprocessing/recalibrated/
+
+
+## Variant Calling
+
+The results regarding variant calling are collected in `{outdir}/variantcalling/`.
+If some results from a variant caller do not appear here, please check out the `--tools` section in the parameter [documentation](https://nf-co.re/sarek/latest/parameters).
+
+(Recalibrated) CRAM files can used as an input to start the variant calling.
+
+### SNVs and small indels
+
+For single nucleotide variants (SNVs) and small indels, multiple tools are available for normal (germline), tumor-only, and tumor-normal (somatic) paired data. For a list of the appropriate tool(s) for the data and sequencing type at hand, please check [here](usage#which-tool).
+
+#### bcftools
+
+[bcftools mpileup](https://samtools.github.io/bcftools/bcftools.html#mpileup) generates pileup of a CRAM file, followed by [bcftools call](https://samtools.github.io/bcftools/bcftools.html#call) and filtered with `-i 'count(GT==\"RR\")==0`.
+For further reading and documentation see the [bcftools manual](https://samtools.github.io/bcftools/howtos/variant-calling.html).
+
+Output files:
+ +**Output directory: `{outdir}/preprocessing/csv`** + +- `mapped.csv` + - if `--save_mapped` + - CSV containing an entry for each sample with the columns `patient,sample,sex,status,bam,bai` +- `markduplicates_no_table.csv` + - CSV containing an entry for each sample with the columns `patient,sample,sex,status,cram,crai` +- `markduplicates.csv` + - CSV containing an entry for each sample with the columns `patient,sample,sex,status,cram,crai,table` +- `recalibrated.csv` + - CSV containing an entry for each sample with the columns`patient,sample,sex,status,cram,crai` +- `variantcalled.csv` + - CSV containing an entry for each sample with the columns `patient,sample,vcf` +
+/`**
+
+- `.bcftools.vcf.gz` and `.bcftools.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+#### DeepVariant
+
+[DeepVariant](https://github.com/google/deepvariant) is a deep learning-based variant caller that takes aligned reads, produces pileup image tensors from them, classifies each tensor using a convolutional neural network and finally reports the results in a standard VCF or gVCF file. For further documentation take a look [here](https://github.com/google/deepvariant/tree/r1.4/docs).
+
+Output files for all samples
+ +**Output directory: `{outdir}/variantcalling/bcftools/
+/`**
+
+- `.deepvariant.vcf.gz` and `.deepvariant.vcf.gz.tbi`
+ - VCF with tabix index
+- `.deepvariant.g.vcf.gz` and `.deepvariant.g.vcf.gz.tbi`
+ - gVCF with tabix index
+
+
+#### FreeBayes
+
+[FreeBayes](https://github.com/ekg/freebayes) is a Bayesian genetic variant detector designed to find small polymorphisms, specifically SNPs, indels, MNPs, and complex events smaller than the length of a short-read sequencing alignment. For further reading and documentation see the [FreeBayes manual](https://github.com/ekg/freebayes/blob/master/README.md#user-manual-and-guide).
+
+Output files for normal samples
+ +**Output directory: `{outdir}/variantcalling/deepvariant/
+.freebayes.vcf.gz` and `.freebayes.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+#### GATK HaplotypeCaller
+
+[GATK HaplotypeCaller](https://gatk.broadinstitute.org/hc/en-us/articles/5358864757787-HaplotypeCaller) calls germline SNPs and indels via local re-assembly of haplotypes.
+
+Output files for all samples
+ +**Output directory: `{outdir}/variantcalling/freebayes/{sample,normalsample_vs_tumorsample}/`** + +- `
+/`**
+
+- `.haplotypecaller.vcf.gz` and `.haplotypecaller.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+##### GATK Germline Single Sample Variant Calling
+
+[GATK Single Sample Variant Calling](https://gatk.broadinstitute.org/hc/en-us/articles/360035535932-Germline-short-variant-discovery-SNPs-Indels-)
+uses HaplotypeCaller in its default single-sample mode to call variants. The VCF that HaplotypeCaller emits errors on the side of sensitivity, therefore they are filtered by first running the [CNNScoreVariants](https://gatk.broadinstitute.org/hc/en-us/articles/5358904862107-CNNScoreVariants) tool. This tool annotates each variant with a score indicating the model's prediction of the quality of each variant. To apply filters based on those scores run the [FilterVariantTranches](https://gatk.broadinstitute.org/hc/en-us/articles/5358928898971-FilterVariantTranches) tool with SNP and INDEL sensitivity tranches appropriate for your task.
+
+If the haplotype-called VCF files are not filtered, then Sarek should be run with at least one of the options `--dbsnp` or `--known_indels`.
+
+Output files for normal samples
+ +**Output directory: `{outdir}/variantcalling/haplotypecaller/
+/`**
+
+- `.haplotypecaller.filtered.vcf.gz` and `.haplotypecaller.filtered.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+##### GATK Joint Germline Variant Calling
+
+[GATK Joint germline Variant Calling](https://gatk.broadinstitute.org/hc/en-us/articles/360035535932-Germline-short-variant-discovery-SNPs-Indels-) uses Haplotypecaller per sample in `gvcf` mode. Next, the gVCFs are consolidated from multiple samples into a [GenomicsDB](https://gatk.broadinstitute.org/hc/en-us/articles/5358869876891-GenomicsDBImport) datastore. After joint [genotyping](https://gatk.broadinstitute.org/hc/en-us/articles/5358906861083-GenotypeGVCFs), [VQSR](https://gatk.broadinstitute.org/hc/en-us/articles/5358906115227-VariantRecalibrator) is applied for filtering to produce the final multisample callset with the desired balance of precision and sensitivity.
+
+Output files for normal samples
+ +**Output directory: `{outdir}/variantcalling/haplotypecaller/
+/`**
+
+- `.haplotypecaller.g.vcf.gz` and `.haplotypecaller.g.vcf.gz.tbi`
+ - gVCF with tabix index
+
+**Output directory: `{outdir}/variantcalling/haplotypecaller/joint_variant_calling/`**
+
+- `joint_germline.vcf.gz` and `joint_germline.vcf.gz.tbi`
+ - VCF with tabix index
+- `joint_germline_recalibrated.vcf.gz` and `joint_germline_recalibrated.vcf.gz.tbi`
+ - variant recalibrated VCF with tabix index (if VQSR is applied)
+
+
+
+#### GATK Mutect2
+
+[GATK Mutect2](https://gatk.broadinstitute.org/hc/en-us/articles/5358911630107-Mutect2) calls somatic SNVs and indels via local assembly of haplotypes.
+When `--joint_mutect2` is used, Mutect2 subworkflow outputs will be saved in a subfolder named with the patient ID and `{patient}.mutect2.vcf.gz` file will contain variant calls from all of the normal and tumor samples of the patient.
+For further reading and documentation see the [Mutect2 manual](https://gatk.broadinstitute.org/hc/en-us/articles/360035531132).
+It is not required, but recommended to have a [panel of normals (PON)](https://gatk.broadinstitute.org/hc/en-us/articles/360035890631-Panel-of-Normals-PON) using at least 40 normal samples to get filtered somatic calls. When using `--genome GATK.GRCh38`, a panel-of-normals file is available. However, it is _highly_ recommended to create one matching your tumor samples. Creating your own panel-of-normals is currently not natively supported by the pipeline. See [here](https://gatk.broadinstitute.org/hc/en-us/articles/360035531132) for how to create one manually.
+
+Output files from joint germline variant calling
+ +**Output directory: `{outdir}/variantcalling/haplotypecaller/
+
+
+#### Sentieon DNAscope
+
+[Sentieon DNAscope](https://support.sentieon.com/appnotes/dnascope_ml/#dnascope-germline-variant-calling-with-a-machine-learning-model) is a variant-caller which aims at outperforming GATK's Haplotypecaller in terms of both speed and accuracy. DNAscope allows you to use a machine learning model to perform variant calling with higher accuracy by improving the candidate detection and filtering.
+
+Output files for tumor-only and tumor/normal paired samples
+ +**Output directory: `{outdir}/variantcalling/mutect2/{sample,tumorsample_vs_normalsample,patient}/`** + +Files created: + +- `{sample,tumorsample_vs_normalsample,patient}.mutect2.vcf.gz` and `{sample,tumorsample_vs_normalsample,patient}.mutect2.vcf.gz.tbi` + - unfiltered (raw) Mutect2 calls VCF with tabix index +- `{sample,tumorsample_vs_normalsample,patient}.mutect2.vcf.gz.stats` + - a stats file generated during calling of raw variants (needed for filtering) +- `{sample,tumorsample_vs_normalsample}.mutect2.contamination.table` + - table calculating the fraction of reads coming from cross-sample contamination +- `{sample,tumorsample_vs_normalsample}.mutect2.segmentation.table` + - table containing segmentation of the tumor by minor allele fraction +- `{sample,tumorsample_vs_normalsample,patient}.mutect2.artifactprior.tar.gz` + - prior probabilities for read orientation artifacts +- `{sample,tumorsample,normalsample}.mutect2.pileups.table` + - tabulates pileup metrics for inferring contamination +- `{sample,tumorsample_vs_normalsample,patient}.mutect2.filtered.vcf.gz` and `{sample,tumorsample_vs_normalsample,patient}.mutect2.filtered.vcf.gz.tbi` + - filtered Mutect2 calls VCF with tabix index based on the probability that a variant is somatic +- `{sample,tumorsample_vs_normalsample,patient}.mutect2.filtered.vcf.gz.filteringStats.tsv` + - a stats file generated during the filtering of Mutect2 called variants + +
+/`**
+
+- `.dnascope.unfiltered.vcf.gz` and `.dnascope.unfiltered.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+The output from Sentieon's DNAscope can be controlled through the option `--sentieon_dnascope_emit_mode` for Sarek, see [Basic usage of Sentieon functions](#basic-usage-of-sentieon-functions).
+
+Unless `dnascope_filter` is listed under `--skip_tools` in the nextflow command, Sentieon's [DNAModelApply](https://support.sentieon.com/manual/usages/general/#dnamodelapply-algorithm) is applied to the unfiltered VCF-files in order to obtain filtered VCF-files.
+
+Unfiltered VCF-files for normal samples
+ +**Output directory: `{outdir}/variantcalling/sentieon_dnascope/
+/`**
+
+- `.dnascope.filtered.vcf.gz` and `.dnascope.filtered.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+##### Sentieon DNAscope joint germline variant calling
+
+In Sentieon's package DNAscope, joint germline variant calling is done by first running Sentieon's Dnacope in emit-mode `gvcf` for each sample and then running Sentieon's [GVCFtyper](https://support.sentieon.com/manual/usages/general/#gvcftyper-algorithm) on the set of gVCF-files. See [Basic usage of Sentieon functions](#basic-usage-of-sentieon-functions) for information on how joint germline variant calling can be done in Sarek using Sentieon's DNAscope.
+
+Filtered VCF-files for normal samples
+ +**Output directory: `{outdir}/variantcalling/sentieon_dnascope/
+/`**
+
+- `.dnascope.g.vcf.gz` and `.dnascope.g.vcf.gz.tbi`
+ - VCF with tabix index
+
+**Output directory: `{outdir}/variantcalling/sentieon_dnascope/joint_variant_calling/`**
+
+- `joint_germline.vcf.gz` and `joint_germline.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+#### Sentieon Haplotyper
+
+[Sentieon Haplotyper](https://support.sentieon.com/manual/usages/general/#haplotyper-algorithm) is Sention's speedup version of GATK's Haplotypecaller (see above).
+
+Output files from joint germline variant calling
+ +**Output directory: `{outdir}/variantcalling/sentieon_dnascope/
+/`**
+
+- `.haplotyper.unfiltered.vcf.gz` and `.haplotyper.unfiltered.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+The output from Sentieon's Haplotyper can be controlled through the option `--sentieon_haplotyper_emit_mode` for Sarek, see [Basic usage of Sentieon functions](#basic-usage-of-sentieon-functions).
+
+Unless `haplotyper_filter` is listed under `--skip_tools` in the nextflow command, GATK's CNNScoreVariants and FilterVariantTranches (see above) is applied to the unfiltered VCF-files in order to obtain filtered VCF-files.
+
+Unfiltered VCF-files for normal samples
+ +**Output directory: `{outdir}/variantcalling/sentieon_haplotyper/
+/`**
+
+- `.haplotyper.filtered.vcf.gz` and `.haplotyper.filtered.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+##### Sentieon Haplotyper joint germline variant calling
+
+In Sentieon's package DNAseq, joint germline variant calling is done by first running Sentieon's Haplotyper in emit-mode `gvcf` for each sample and then running Sentieon's [GVCFtyper](https://support.sentieon.com/manual/usages/general/#gvcftyper-algorithm) on the set of gVCF-files. See [Basic usage of Sentieon functions](#basic-usage-of-sentieon-functions) for information on how joint germline variant calling can be done in Sarek using Sentieon's DNAseq. After joint genotyping, Sentieon's version of VQSR ([VarCal](https://support.sentieon.com/manual/usages/general/#varcal-algorithm) and [ApplyVarCal](https://support.sentieon.com/manual/usages/general/#applyvarcal-algorithm)) is applied for filtering to produce the final multisample callset with the desired balance of precision and sensitivity.
+
+Filtered VCF-files for normal samples
+ +**Output directory: `{outdir}/variantcalling/sentieon_haplotyper/
+/`**
+
+- `.haplotyper.g.vcf.gz` and `.haplotyper.g.vcf.gz.tbi`
+ - VCF with tabix index
+
+**Output directory: `{outdir}/variantcalling/sentieon_haplotyper/joint_variant_calling/`**
+
+- `joint_germline.vcf.gz` and `joint_germline.vcf.gz.tbi`
+ - VCF with tabix index
+- `joint_germline_recalibrated.vcf.gz` and `joint_germline_recalibrated.vcf.gz.tbi`
+ - variant recalibrated VCF with tabix index (if VarCal is applied)
+
+
+
+#### Strelka2
+
+[Strelka2](https://github.com/Illumina/strelka) is a fast and accurate small variant caller optimized for analysis of germline variation in small cohorts and somatic variation in tumor/normal sample pairs. For further reading and documentation see the [Strelka2 user guide](https://github.com/Illumina/strelka/blob/master/docs/userGuide/README.md). If [Strelka2](https://github.com/Illumina/strelka) is used for somatic variant calling and [Manta](https://github.com/Illumina/manta) is also specified in tools, the output candidate indels from [Manta](https://github.com/Illumina/manta) are used according to [Strelka Best Practices](https://github.com/Illumina/strelka/blob/master/docs/userGuide/README.md#somatic-configuration-example).
+For further downstream analysis, take a look [here](https://github.com/Illumina/strelka/blob/v2.9.x/docs/userGuide/README.md#interpreting-the-germline-multi-sample-variants-vcf).
+
+Output files from joint germline variant calling
+ +**Output directory: `{outdir}/variantcalling/sentieon_haplotyper/
+/`**
+
+- `.strelka.genome.vcf.gz` and `.strelka.genome.vcf.gz.tbi`
+ - genome VCF with tabix index
+- `.strelka.variants.vcf.gz` and `.strelka.variants.vcf.gz.tbi`
+ - VCF with tabix index with all potential variant loci across the sample. Note this file includes non-variant loci if they have a non-trivial level of variant evidence or contain one or more alleles for which genotyping has been forced.
+
+
+Output files for all single samples (normal or tumor-only)
+ +**Output directory: `{outdir}/variantcalling/strelka/
+/`**
+
+- `.strelka.somatic_indels.vcf.gz` and `.strelka.somatic_indels.vcf.gz.tbi`
+ - VCF with tabix index with all somatic indels inferred in the tumor sample.
+- `.strelka.somatic_snvs.vcf.gz` and `.strelka.somatic_snvs.vcf.gz.tbi`
+ - VCF with tabix index with all somatic SNVs inferred in the tumor sample.
+
+
+
+### Structural Variants
+
+#### Manta
+
+[Manta](https://github.com/Illumina/manta) calls structural variants (SVs) and indels from mapped paired-end sequencing reads.
+It is optimized for analysis of germline variation in small sets of individuals and somatic variation in tumor/normal sample pairs.
+[Manta](https://github.com/Illumina/manta) provides a candidate list for small indels that can be fed to [Strelka2](https://github.com/Illumina/strelka) following [Strelka Best Practices](https://github.com/Illumina/strelka/blob/master/docs/userGuide/README.md#somatic-configuration-example). For further reading and documentation see the [Manta user guide](https://github.com/Illumina/manta/blob/master/docs/userGuide/README.md).
+
+Output files for tumor/normal paired samples
+ +**Output directory: `{outdir}/variantcalling/strelka/
+/`**
+
+- `.manta.diploid_sv.vcf.gz` and `.manta.diploid_sv.vcf.gz.tbi`
+ - VCF with tabix index containing SVs and indels scored and genotyped under a diploid model for the sample.
+
+
+Output files for normal samples
+ +**Output directory: `{outdir}/variantcalling/manta/
+/`**
+
+- `.manta.tumor_sv.vcf.gz` and `.manta.tumor_sv.vcf.gz.tbi`
+ - VCF with tabix index containing a subset of the candidateSV.vcf.gz file after removing redundant candidates and small indels less than the minimum scored variant size (50 by default). The SVs are not scored, but include additional details: (1) paired and split read supporting evidence counts for each allele (2) a subset of the filters from the scored tumor-normal model are applied to the single tumor case to improve precision.
+
+
+Output files for tumor-only samples
+ +**Output directory: `{outdir}/variantcalling/manta/
+/`**
+
+- `.manta.diploid_sv.vcf.gz` and `.manta.diploid_sv.vcf.gz.tbi`
+ - VCF with tabix index containing SVs and indels scored and genotyped under a diploid model for the sample. In the case of a tumor/normal subtraction, the scores in this file do not reflect any information from the tumor sample.
+- `.manta.somatic_sv.vcf.gz` and `.manta.somatic_sv.vcf.gz.tbi`
+ - VCF with tabix index containing SVs and indels scored under a somatic variant model.
+
+
+#### TIDDIT
+
+[TIDDIT](https://github.com/SciLifeLab/TIDDIT) identifies intra and inter-chromosomal translocations, deletions, tandem-duplications and inversions. For further reading and documentation see the [TIDDIT manual](https://github.com/SciLifeLab/TIDDIT/blob/master/README.md).
+
+Output files for tumor/normal paired samples
+ +**Output directory: `{outdir}/variantcalling/manta/
+/`**
+
+- `.tiddit.vcf.gz` and `.tiddit.vcf.gz.tbi`
+ - VCF with tabix index containing SV calls
+- `.tiddit.ploidies.tab`
+ - tab file describing the estimated ploidy and coverage across each contig
+
+
+
+Output files for normal and tumor-only samples
+ +**Output directory: `{outdir}/variantcalling/tiddit/
+/`**
+
+- `.tiddit.normal.vcf.gz` and `.tiddit.normal.vcf.gz.tbi`
+ - VCF with tabix index containing SV calls
+- `.tiddit.tumor.vcf.gz` and `.tiddit.tumor.vcf.gz.tbi`
+ - VCF with tabix index containing SV calls
+- `_sv_merge.tiddit.vcf.gz` and `_sv_merge.tiddit.vcf.gz.tbi`
+ - merged tumor/normal VCF with tabix index
+- `.tiddit.ploidies.tab`
+ - tab file describing the estimated ploidy and coverage across each contig
+
+
+
+### Sample heterogeneity, ploidy and CNVs
+
+#### ASCAT
+
+[ASCAT](https://github.com/VanLoo-lab/ascat) is a software for performing allele-specific copy number analysis of tumor samples and for estimating tumor ploidy and purity (normal contamination).
+It infers tumor purity and ploidy and calculates whole-genome allele-specific copy number profiles.
+The [ASCAT](https://github.com/VanLoo-lab/ascat) process gives several images as output, described in detail in this [book chapter](http://www.ncbi.nlm.nih.gov/pubmed/22130873).
+Running ASCAT on NGS data requires that the BAM files are converted into BAF and LogR values.
+This is done internally using the software [AlleleCount](https://github.com/cancerit/alleleCount). For further reading and documentation see the [ASCAT manual](https://www.crick.ac.uk/research/labs/peter-van-loo/software).
+
+Output files for tumor/normal paired samples
+ +**Output directory: `{outdir}/variantcalling/tiddit/
+/`**
+
+- `.tumour.ASCATprofile.png`
+ - image with information about allele-specific copy number profile
+- `.tumour.ASPCF.png`
+ - image with information about allele-specific copy number segmentation
+- `.before_correction_Tumour..tumour.png`
+ - image with information about raw profile of tumor sample of logR and BAF values before GC correction
+- `.before_correction_Tumour..germline.png`
+ - image with information about raw profile of normal sample of logR and BAF values before GC correction
+- `.after_correction_GC_Tumour..tumour.png`
+ - image with information about GC and RT corrected logR and BAF values of tumor sample after GC correction
+- `.after_correction_GC_Tumour..germline.png`
+ - image with information about GC and RT corrected logR and BAF values of normal sample after GC correction
+- `.tumour.sunrise.png`
+ - image visualising the range of ploidy and tumor percentage values
+- `.metrics.txt`
+ - file with information about different metrics from ASCAT profiles
+- `.cnvs.txt`
+ - file with information about CNVS
+- `.purityploidy.txt`
+ - file with information about purity and ploidy
+- `.segments.txt`
+ - file with information about copy number segments
+- `.tumour_tumourBAF.txt` and `.tumour_normalBAF.txt`
+ - file with beta allele frequencies
+- `.tumour_tumourLogR.txt` and `.tumour_normalLogR.txt`
+ - file with total copy number on a logarithmic scale
+
+The text file `.cnvs.txt` contains predictions about copy number state for all the segments.
+The output is a tab delimited text file with the following columns:
+
+- _chr_: chromosome number
+- _startpos_: start position of the segment
+- _endpos_: end position of the segment
+- _nMajor_: number of copies of one of the allels (for example the chromosome inherited of one parent)
+- _nMinor_: number of copies of the other allele (for example the chromosome inherited of the other parent)
+
+The file `.cnvs.txt` contains all segments predicted by ASCAT, both those with normal copy number (nMinor = 1 and nMajor =1) and those corresponding to copy number aberrations.
+
+
+
+#### CNVKit
+
+[CNVKit](https://cnvkit.readthedocs.io/en/stable/) is a toolkit to infer and visualize copy number from high-throughput DNA sequencing data. It is designed for use with hybrid capture, including both whole-exome and custom target panels, and short-read sequencing platforms such as Illumina. For further reading and documentation, see the [CNVKit Documentation](https://cnvkit.readthedocs.io/en/stable/plots.html)
+
+Output files for tumor/normal paired samples
+ +**Output directory: `{outdir}/variantcalling/ascat/
+/`**
+
+- `.antitargetcoverage.cnn`
+ - file containing coverage information
+- `.targetcoverage.cnn`
+ - file containing coverage information
+- `-diagram.pdf`
+ - file with plot of copy numbers or segments on chromosomes
+- `-scatter.png`
+ - file with plot of bin-level log2 coverages and segmentation calls
+- `.bintest.cns`
+ - file containing copy number segment information
+- `.cnr`
+ - file containing copy number ratio information
+- `.cns`
+ - file containing copy number segment information
+- `.call.cns`
+ - file containing copy number segment information
+- `.genemetrics.tsv`
+ - file containing per gene copy number information (if input files are annotated)
+
+
+Output files for normal and tumor-only samples
+ +**Output directory: `{outdir}/variantcalling/cnvkit/
+/`**
+
+- `.antitargetcoverage.cnn`
+ - file containing coverage information
+- `.targetcoverage.cnn`
+ - file containing coverage information
+- `.antitargetcoverage.cnn`
+ - file containing coverage information
+- `.targetcoverage.cnn`
+ - file containing coverage information
+- `.bintest.cns`
+ - file containing copy number segment information
+- `-scatter.png`
+ - file with plot of bin-level log2 coverages and segmentation calls
+- `-diagram.pdf`
+ - file with plot of copy numbers or segments on chromosomes
+- `.cnr`
+ - file containing copy number ratio information
+- `.cns`
+ - file containing copy number segment information
+- `.call.cns`
+ - file containing copy number segment information
+- `.genemetrics.tsv`
+ - file containing per gene copy number information (if input files are annotated)
+
+
+#### Control-FREEC
+
+[Control-FREEC](https://github.com/BoevaLab/FREEC) is a tool for detection of copy-number changes and allelic imbalances (including loss of heterozygoity (LOH)) using deep-sequencing data.
+[Control-FREEC](https://github.com/BoevaLab/FREEC) automatically computes, normalizes, segments copy number and beta allele frequency profiles, then calls copy number alterations and LOH.
+It also detects subclonal gains and losses and evaluates the most likely average ploidy of the sample. For further reading and documentation see the [Control-FREEC Documentation](http://boevalab.inf.ethz.ch/FREEC/tutorial.html).
+
+Output files for tumor/normal samples
+ +**Output directory: `{outdir}/variantcalling/cnvkit/
+_BAF.png` and `_BAF.png`
+ - image of BAF plot
+- `_ratio.log2.png` and `_ratio.log2.png`
+ - image of ratio log2 plot
+- `_ratio.png` and `_ratio.png`
+ - image of ratio plot
+- `.bed` and `.bed`
+ - translated output to a .BED file (so to view it in the UCSC Genome Browser)
+- `.circos.txt` and `.circos.txt`
+ - translated output to the Circos format
+- `.p.value.txt` and `.p.value.txt`
+ - CNV file containing p_values for each call
+- `_BAF.txt` and `.mpileup.gz_BAF.txt`
+ - file with beta allele frequencies for each possibly heterozygous SNP position
+- `.tumor.mpileup.gz_CNVs`
+ - file with coordinates of predicted copy number alterations
+- `_info.txt` and `.tumor.mpileup.gz_info.txt`
+ - parsable file with information about FREEC run
+- ` _ratio.BedGraph` and `.tumor.mpileup.gz_ratio.BedGraph `
+ - file with ratios in BedGraph format for visualization in the UCSC genome browser. The file contains tracks for normal copy number, gains and losses, and copy neutral LOH (\*).
+- `_ratio.txt` and `.tumor.mpileup.gz_ratio.txt`
+ - file with ratios and predicted copy number alterations for each window
+- `_sample.cpn` and `.tumor.mpileup.gz_sample.cpn`
+ - files with raw copy number profiles for the tumor sample
+- `.normal.mpileup.gz_control.cpn`
+ - files with raw copy number profiles for the control sample
+- `.cpn>`
+ - file with GC-content profile
+
+
+
+### Microsatellite instability (MSI)
+
+[Microsatellite instability](https://en.wikipedia.org/wiki/Microsatellite_instability) is a genetic condition associated with deficiencies in the mismatch repair (MMR) system which causes a tendency to accumulate a high number of mutations (SNVs and indels).
+An altered distribution of microsatellite length is associated with a missed replication slippage which would be corrected under normal MMR conditions.
+
+#### MSIsensorPro
+
+[MSIsensorPro](https://github.com/xjtu-omics/msisensor-pro) is a tool to detect the MSI status of a tumor scanning the length of the microsatellite regions.
+It requires a normal sample for each tumour to differentiate the somatic and germline cases. For further reading see the [MSIsensor paper](https://www.ncbi.nlm.nih.gov/pubmed/24371154).
+
+Output files for tumor-only and tumor/normal paired samples
+ +**Output directory: `{outdir}/variantcalling/controlfreec/{tumorsample,tumorsample_vs_normalsample}/`** + +- `config.txt` + - Configuration file used to run Control-FREEC +- `
+/`**
+
+- ``
+ - MSI score output, contains information about the number of somatic sites.
+- `_dis`
+ - The normal and tumor length distribution for each microsatellite position.
+- `_germline`
+ - Somatic sites detected.
+- `_somatic`
+ - Germline sites detected.
+
+
+### Concatenation
+
+Germline VCFs from `DeepVariant`, `FreeBayes`, `HaplotypeCaller`, `Haplotyper`, `Manta`, `bcftools mpileup`, `Strelka2`, or `Tiddit` are concatenated with `bcftools concat`. The field `SOURCE` is added to the VCF header to report the variant caller.
+
+Output files for tumor/normal paired samples
+ +**Output directory: `{outdir}/variantcalling/msisensor/
+/`**
+
+- `.germline.vcf.gz` and `.germline.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+## Variant annotation
+
+This directory contains results from the final annotation steps: two tools are used for annotation, [snpEff](http://snpeff.sourceforge.net/) and [VEP](https://www.ensembl.org/info/docs/tools/vep/index.html). Both results can also be combined by setting `--tools merge`.
+All variants present in the called VCF files are annotated. For some variant callers this can mean that the variants are already filtered by `PASS`, for some this needs to be done during post-processing.
+
+### snpEff
+
+[snpeff](http://snpeff.sourceforge.net/) is a genetic variant annotation and effect prediction toolbox.
+It annotates and predicts the effects of variants on genes (such as amino acid changes) using multiple databases for annotations.
+The generated VCF header contains the software version and the used command line. For further reading and documentation see the [snpEff manual](http://snpeff.sourceforge.net/SnpEff_manual.html#outputSummary).
+
+Concatenated VCF-files for normal samples
+ +**Output directory: `{outdir}/variantcalling/concat/
+_snpEff.ann.vcf.gz` and `{sample,tumorsample_vs_normalsample}._snpEff.ann.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+### VEP
+
+[VEP (Variant Effect Predictor)](https://www.ensembl.org/info/docs/tools/vep/index.html), based on `Ensembl`, is a tool to determine the effects of all sorts of variants, including SNPs, indels, structural variants, CNVs.
+The generated VCF header contains the software version, also the version numbers for additional databases like [Clinvar](https://www.ncbi.nlm.nih.gov/clinvar/) or [dbSNP](https://www.ncbi.nlm.nih.gov/snp/) used in the [VEP](https://www.ensembl.org/info/docs/tools/vep/index.html) line.
+The format of the [consequence annotations](https://www.ensembl.org/info/genome/variation/prediction/predicted_data.html) is also in the VCF header describing the `INFO` field.
+For further reading and documentation see the [VEP manual](https://www.ensembl.org/info/docs/tools/vep/index.html).
+
+Currently, it contains:
+
+- _Consequence_: impact of the variation, if there is any
+- _Codons_: the codon change, i.e. cGt/cAt
+- _Amino_acids_: change in amino acids, i.e. R/H if there is any
+- _Gene_: ENSEMBL gene name
+- _SYMBOL_: gene symbol
+- _Feature_: actual transcript name
+- _EXON_: affected exon
+- _PolyPhen_: prediction based on [PolyPhen](http://genetics.bwh.harvard.edu/pph2/)
+- _SIFT_: prediction by [SIFT](http://sift.bii.a-star.edu.sg/)
+- _Protein_position_: Relative position of amino acid in protein
+- _BIOTYPE_: Biotype of transcript or regulatory feature
+
+plus any additional filed selected via the plugins: [dbNSFP](https://sites.google.com/site/jpopgen/dbNSFP), [LOFTEE](https://github.com/konradjk/loftee), [SpliceAI](https://spliceailookup.broadinstitute.org/), [SpliceRegion](https://www.ensembl.info/2018/10/26/cool-stuff-the-vep-can-do-splice-site-variant-annotation/).
+
+Output files for all samples
+ +**Output directory: `{outdir}/annotation/{sample,tumorsample_vs_normalsample}`** + +- `{sample,tumorsample_vs_normalsample}.
+_VEP.ann.vcf.gz` and `{sample,tumorsample_vs_normalsample}._VEP.ann.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+### BCFtools annotate
+
+[BCFtools annotate](https://samtools.github.io/bcftools/bcftools.html#annotate) is used to add annotations to VCF files. The annotations are added to the INFO column of the VCF file. The annotations are added to the VCF header and the VCF header is updated with the new annotations. For further reading and documentation see the [BCFtools annotate manual](https://samtools.github.io/bcftools/bcftools.html#annotate).
+
+Output files for all samples
+ +**Output directory: `{outdir}/annotation/{sample,tumorsample_vs_normalsample}`** + +- `{sample,tumorsample_vs_normalsample}.
+_bcf.ann.vcf.gz` and `{sample,tumorsample_vs_normalsample}._bcf.ann.vcf.gz.tbi`
+ - VCF with tabix index
+
+
+
+## Quality control and reporting
+
+### Quality control
+
+#### FastQC
+
+[FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/).
+
+The plots display:
+
+- Sequence counts for each sample.
+- Sequence Quality Histograms: The mean quality value across each base position in the read.
+- Per Sequence Quality Scores: The number of reads with average quality scores. Shows if a subset of reads has poor quality.
+- Per Base Sequence Content: The proportion of each base position for which each of the four normal DNA bases has been called.
+- Per Sequence GC Content: The average GC content of reads. Normal random library typically have a roughly normal distribution of GC content.
+- Per Base N Content: The percentage of base calls at each position for which an N was called.
+- Sequence Length Distribution.
+- Sequence Duplication Levels: The relative level of duplication found for each sequence.
+- Overrepresented sequences: The total amount of overrepresented sequences found in each library.
+- Adapter Content: The cumulative percentage count of the proportion of your library which has seen each of the adapter sequences at each position.
+
+Output files for all samples
+ +- `{sample,tumorsample_vs_normalsample}.
+`**
+
+- `_fastqc.html` and `_fastqc.html`
+ - [FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) report containing quality metrics for your untrimmed raw FastQ files
+- `_fastqc.zip` and `_fastqc.zip`
+ - Zip archive containing the [FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) report, tab-delimited data file and plot images
+
+> **NB:** The FastQC plots displayed in the [MultiQC](https://multiqc.info/) report shows _untrimmed_ reads.
+> They may contain adapter sequence and potentially regions with low quality.
+
+
+
+#### FastP
+
+[FastP](https://github.com/OpenGene/fastp) is a tool designed to provide all-in-one preprocessing for FastQ files and is used for trimming and splitting. The tool then determines QC metrics for the processed reads.
+
+Output files for all samples
+ +:::note +The FastQC plots displayed in the MultiQC report shows _untrimmed_ reads. They may contain adapter sequence and potentially regions with low quality. +::: +**Output directory: `{outdir}/reports/fastqc/
+`**
+
+- `_fastp.html`
+ - report in HTML format
+- `_fastp.json`
+ - report in JSON format
+- `_fastp.log`
+ - FastQ log file
+
+
+
+#### Mosdepth
+
+[Mosdepth](https://github.com/brentp/mosdepth) reports information for the evaluation of the quality of the provided alignment data.
+In short, the basic statistics of the alignment (number of reads, coverage, GC-content, etc.) are summarized and a number of useful graphs are produced.
+For further reading and documentation see the [Mosdepth documentation](https://github.com/brentp/mosdepth).
+
+Plots will show:
+
+- cumulative coverage distribution
+- absolute coverage distribution
+- average coverage per contig/chromosome
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/fastp/
+`**
+
+- `.{sorted,md,recal}.mosdepth.global.dist.txt`
+ - file used by [MultiQC](https://multiqc.info/), if `.region` file does not exist
+- `.{sorted,md,recal}.mosdepth.region.dist.txt`
+ - file used by [MultiQC](https://multiqc.info/)
+- `.{sorted,md,recal}.mosdepth.summary.txt`
+ -A summary of mean depths per chromosome and within specified regions per chromosome.
+- `.{sorted,md,recal}.{per-base,regions}.bed.gz`
+ - per-base depth for targeted data, per-window (500bp) depth of WGS
+- `.{sorted,md,recal}.regions.bed.gz.csi`
+ - CSI index for per-base depth for targeted data, per-window (500bp) depth of WGS
+
+
+#### NGSCheckMate
+
+[NGSCheckMate](https://github.com/parklab/NGSCheckMate) is a tool for determining whether samples come from the same genetic individual, using a set of commonly heterozygous SNPs. This enables for the detecting of sample mislabelling events. The output includes a text file indicating whether samples have matched or not according to the algorithm, as well as a dendrogram visualising these results.
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/mosdepth/
+.vcf.gz`
+ - Set of vcf files for each sample. Contains calls for the set of SNP positions used to calculate sample relatedness.
+
+
+#### GATK MarkDuplicates reports
+
+More information in the [GATK MarkDuplicates section](#gatk-markduplicates)
+
+Duplicates can arise during sample preparation _e.g._ library construction using PCR.
+Duplicate reads can also result from a single amplification cluster, incorrectly detected as multiple clusters by the optical sensor of the sequencing instrument.
+These duplication artifacts are referred to as optical duplicates. If [GATK MarkDuplicates](https://gatk.broadinstitute.org/hc/en-us/articles/5358880192027-MarkDuplicates-Picard-) is used, the metrics file generated by the tool is used, if [`GATK MarkDuplicatesSpark`](https://gatk.broadinstitute.org/hc/en-us/articles/5358833264411-MarkDuplicatesSpark) is used the report is generated by [GATK4 EstimateLibraryComplexity](https://gatk.broadinstitute.org/hc/en-us/articles/5358838684187-EstimateLibraryComplexity-Picard-) on the mapped BAM files.
+For further reading and documentation see the [MarkDuplicates manual](https://software.broadinstitute.org/gatk/documentation/tooldocs/4.1.2.0/picard_sam_markduplicates_MarkDuplicates.php).
+
+The plot will show:
+
+- duplication statistics
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/ngscheckmate/`** + +- `ngscheckmate_all.txt` + - Tab delimited text file listing all the comparisons made, whether they were considered as a match, with the correlation and a normalised depth. +- `ngscheckmate_matched.txt` + - Tab delimited text file listing only the comparison that were considered to match, with the correlation and a normalised depth. +- `ngscheckmate_output_corr_matrix.txt` + - Tab delimited text file containing a matrix of all correlations for all comparisons made. +- `vcfs/
+`**
+
+- `.md.cram.metrics`
+ - file used by [MultiQC](https://multiqc.info/)
+
+
+#### Sentieon Dedup reports
+
+Sentieon's DNAseq subroutine Dedup produces a metrics report much like the one produced by GATK's MarkDuplicates. The Dedup metrics are imported into MultiQC as custom content and displayed in a table.
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/markduplicates/
+`**
+
+- `.dedup.cram.metrics`
+ - file used by [MultiQC](https://multiqc.info/).
+
+
+#### samtools stats
+
+[samtools stats](https://www.htslib.org/doc/samtools.html) collects statistics from CRAM files and outputs in a text format.
+For further reading and documentation see the [`samtools` manual](https://www.htslib.org/doc/samtools.html#COMMANDS_AND_OPTIONS).
+
+The plots will show:
+
+- Alignment metrics.
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/sentieon_dedup/
+`**
+
+- `.{sorted,md,recal}.samtools.stats.out`
+ - Raw statistics used by `MultiQC`
+
+
+
+#### bcftools stats
+
+[bcftools stats](https://samtools.github.io/bcftools/bcftools.html#stats) produces a statistics text file which is suitable for machine processing and can be plotted using plot-vcfstats.
+For further reading and documentation see the [bcftools stats manual](https://samtools.github.io/bcftools/bcftools.html#stats).
+
+Plots will show:
+
+- Stats by non-reference allele frequency, depth distribution, stats by quality and per-sample counts, singleton stats, etc.
+- Note: When using [Strelka2](https://github.com/Illumina/strelka), there will be no depth distribution plot, as Strelka2 does not report the INFO/DP field
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/samtools/
+..bcftools_stats.txt`
+ - Raw statistics used by `MultiQC`
+
+
+#### VCFtools
+
+[VCFtools](https://vcftools.github.io/) is a program package designed for working with VCF files. For further reading and documentation see the [VCFtools manual](https://vcftools.github.io/man_latest.html#OUTPUT%20OPTIONS).
+
+Plots will show:
+
+- the summary counts of each type of transition to transversion ratio for each `FILTER` category.
+- the transition to transversion ratio as a function of alternative allele count (using only bi-allelic SNPs).
+- the transition to transversion ratio as a function of SNP quality threshold (using only bi-allelic SNPs).
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/bcftools/`** + +- `
+..FILTER.summary`
+ - Raw statistics used by `MultiQC` with a summary of the number of SNPs and Ts/Tv ratio for each FILTER category
+- `..TsTv.count`
+ - Raw statistics used by `MultiQC` with the Transition / Transversion ratio as a function of alternative allele count. Only uses bi-allelic SNPs.
+- `..TsTv.qual`
+ - Raw statistics used by `MultiQC` with Transition / Transversion ratio as a function of SNP quality threshold. Only uses bi-allelic SNPs.
+
+
+#### snpEff reports
+
+[snpeff](http://snpeff.sourceforge.net/) is a genetic variant annotation and effect prediction toolbox.
+It annotates and predicts the effects of variants on genes (such as amino acid changes) using multiple databases for annotations. For further reading and documentation see the [snpEff manual](http://snpeff.sourceforge.net/SnpEff_manual.html#outputSummary).
+
+The plots will show:
+
+- locations of detected variants in the genome and the number of variants for each location.
+- the putative impact of detected variants and the number of variants for each impact.
+- the effect of variants at protein level and the number of variants for each effect type.
+- the quantity as function of the variant quality score.
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/vcftools/`** + +- `
+/`**
+
+- `._snpEff.csv`
+ - Raw statistics used by [MultiQC](http://multiqc.info)
+- `._snpEff.html`
+ - Statistics to be visualised with a web browser
+- `._snpEff.genes.txt`
+ - TXT (tab separated) summary counts for variants affecting each transcript and gene
+
+
+#### VEP reports
+
+[VEP (Variant Effect Predictor)](https://www.ensembl.org/info/docs/tools/vep/index.html), based on `Ensembl`, is a tool to determine the effects of all sorts of variants, including SNPs, indels, structural variants, CNVs. For further reading and documentation see the [VEP manual](https://www.ensembl.org/info/docs/tools/vep/index.html)
+
+Output files for all samples
+ +**Output directory: `{outdir}/reports/SnpEff/{sample,tumorsample_vs_normalsample}/
+/`**
+
+- `._VEP.summary.html`
+ - Summary of the VEP run to be visualised with a web browser
+
+
+### Reporting
+
+#### MultiQC
+
+[MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarizing all samples in your project.
+Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory.
+Results generated by MultiQC collect pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see Output files for all samples
+ +**Output directory: `{outdir}/reports/EnsemblVEP/{sample,tumorsamplt_vs_normalsample}/
+
+
+### Pipeline information
+
+Output files
+ +- `multiqc/` + - `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser. + - `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline. + - `multiqc_plots/`: directory containing static images from the report in various formats. +
+.html`, `execution_timeline_.html`, `execution_trace_.txt`, `pipeline_dag_.dot`/`pipeline_dag_.svg` and `manifest_.bco.json`.
+ - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline.
+ - Parameters used by the pipeline run: `params_.json`.
+
+
+
+[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage.
+
+## Reference files
+
+Contains reference folders generated by the pipeline. These files are only published, if `--save_reference` is set.
+
+Output files
+ +- `pipeline_info/` + - Reports generated by Nextflow: `execution_report_
+
+
+# Mapping stats with Picard.
+
+[Picard](https://broadinstitute.github.io/picard/) is used to gather mapping quality metrics from sarek's output into a single table that can be easily inspected. The module used for this task is CollectHsMetrics. You may find more information regarding this module and its output [here](http://broadinstitute.github.io/picard/picard-metric-definitions.html#HsMetrics).
+
+# Post-processing and Annotation
+
+[GATK toolkit](https://gatk.broadinstitute.org/hc/en-us) is used to separate SNPs, indels and apply several filters to process the vcf files generated by sarek.
+
+[AWK](https://www.gnu.org/software/gawk/manual/gawk.html) and [BCFtools query](https://samtools.github.io/bcftools/bcftools.html#query) are used to modify the VCF previously processed with GATK toolkit, creating a new variants table which is easier to merge with the annotation data from VEP and Exomiser.
+
+The annotation step is performed apart from sarek in order to provide a more thorough annotation of the variants, which will include prediction of effect and correlation with the disease thanks to [Ensembl's Variant Effect Predictor (VEP)](https://www.ensembl.org/info/docs/tools/vep/index.html) and [exomiser](https://exomiser.readthedocs.io/en/latest/advanced_analysis.html). This latter will also include inheritance typing.
+
+# VEP documentation
+
+You can find information on how to interpret results from VEP toolkit [here](https://www.ensembl.org/info/docs/tools/vep/vep_formats.html#output).
+
+# The Exomiser - A Tool to Annotate and Prioritize Exome Variants
+
+[](https://github.com/exomiser/Exomiser/releases)
+[](https://circleci.com/gh/exomiser/Exomiser/tree/development)
+[](https://www.codacy.com/app/monarch-initiative/Exomiser?utm_source=github.com&utm_medium=referral&utm_content=exomiser/Exomiser&utm_campaign=Badge_Grade)
+[](http://exomiser.readthedocs.io/en/latest)
+#### Overview:
+
+The Exomiser is a Java program that finds potential disease-causing variants from whole-exome or whole-genome sequencing data.
+
+Starting from a [VCF](https://samtools.github.io/hts-specs/VCFv4.3.pdf) file and a set of phenotypes encoded using the [Human Phenotype Ontology](http://www.human-phenotype-ontology.org) (HPO) it will annotate, filter and prioritise likely causative variants. The program does this based on user-defined criteria such as a variant's predicted pathogenicity, frequency of occurrence in a population and also how closely the given phenotype matches the known phenotype of diseased genes from human and model organism data.
+
+The functional annotation of variants is handled by [Jannovar](https://github.com/charite/jannovar) and uses any of [UCSC](http://genome.ucsc.edu), [RefSeq](https://www.ncbi.nlm.nih.gov/refseq/) or [Ensembl](https://www.ensembl.org/Homo_sapiens/Info/Index) KnownGene transcript definitions and hg19 or hg38 genomic coordinates.
+
+Variants are prioritised according to user-defined criteria on variant frequency, pathogenicity, quality, inheritance pattern, and model organism phenotype data. Predicted pathogenicity data is extracted from the [dbNSFP](http://www.ncbi.nlm.nih.gov/pubmed/21520341) resource. Variant frequency data is taken from the [1000 Genomes](http://www.1000genomes.org/), [ESP](http://evs.gs.washington.edu/EVS), [TOPMed](http://www.uk10k.org/studies/cohorts.html), [UK10K](http://www.uk10k.org/studies/cohorts.html), [ExAC](http://exac.broadinstitute.org) and [gnomAD](http://gnomad.broadinstitute.org/) datasets. Subsets of these frequency and pathogenicity data can be defined to further tune the analysis. Cross-species phenotype comparisons come from our PhenoDigm tool powered by the OWLTools [OWLSim](https://github.com/owlcollab/owltools) algorithm.
+
+The Exomiser was developed by the Computational Biology and Bioinformatics group at the Institute for Medical Genetics and Human Genetics of the Charité - Universitätsmedizin Berlin, the Mouse Informatics Group at the Sanger Institute and other members of the [Monarch initiative](https://monarchinitiative.org).
+
+# Interpreting the Results
+
+Depending on the output options provided, Exomiser will write out at least an HTML results file in the `exomiser` sub-directory of the Exomiser installation.
+
+As a general rule all output files contain a ranked list of genes and/or variants with the top-ranked gene/variant displayed first. The exception being the VCF output which, since version 13.1.0, is sorted according to VCF convention and tabix indexed.
+
+Exomiser attempts to predict the variant or variants likely to be causative of a patient's phenotype and does so by associating them with the gene (or genes in the case of large structural variations) they intersect with on the genomic sequence. Variants occurring in intergenic regions are associated to the closest gene and those overlapping two genes are associated with the gene in which they are predicted to have the largest consequence.
+
+Once associated with a gene, Exomiser uses the compatible modes of inheritance for a variant to assess it in the context of any diseases associated with the gene or any mouse knockout models of that gene. These are all bundled together into a `GeneScore` which features filtered variants located in that gene compatible with a given mode of inheritance. After the filtering steps Exomiser ranks these GeneScores according to descending combined score. The results are then written out to the files and formats specified in the output settings.
+
+As of release 13.2.0 the output files only feature a single, combined output file of ranked genes/variants e.g. when supplying the output options (via an output-options.yaml file) `outputFileName: Pfeiffer-hiphive-exome-PASS_ONLY` and `outputFormats: [TSV_VARIANT, TSV_GENE, VCF]`, the following files will be written out: `Pfeiffer-hiphive-exome-PASS_ONLY.variants.tsv` `Pfeiffer-hiphive-exome-PASS_ONLY.genes.tsv`, `Pfeiffer-hiphive-exome-PASS_ONLY.vcf`
+
+These formats are detailed below.
+
+## HTML
+
+
+
+
+
+## JSON
+
+The JSON file represents the most accurate representation of the data, as it is referenced internally by Exomiser. As such, we don't provide a schema for this, but it has been pretty stable and breaking changes will only occur with major version changes to the software. Minor additions are to be expected for minor releases, as per the [SemVer](https://semver.org) specification.
+
+We recommend using [Python](https://docs.python.org/3/library/json.html?highlight=json#module-json) or [JQ](https://stedolan.github.io/jq/) to extract data from this file.
+
+## TSV GENES
+
+In the genes.tsv file it is possible for a gene to appear multiple times, depending on the MOI it is compatible with, given the filtered variants. For example in the example below MUC6 is ranked 7th under the AD model and 8th under an AR model.
+
+```tsv
+#RANK ID GENE_SYMBOL ENTREZ_GENE_ID MOI P-VALUE EXOMISER_GENE_COMBINED_SCORE EXOMISER_GENE_PHENO_SCORE EXOMISER_GENE_VARIANT_SCORE HUMAN_PHENO_SCORE MOUSE_PHENO_SCORE FISH_PHENO_SCORE WALKER_SCORE PHIVE_ALL_SPECIES_SCORE OMIM_SCORE MATCHES_CANDIDATE_GENE HUMAN_PHENO_EVIDENCE MOUSE_PHENO_EVIDENCE FISH_PHENO_EVIDENCE HUMAN_PPI_EVIDENCE MOUSE_PPI_EVIDENCE FISH_PPI_EVIDENCE
+1 FGFR2_AD FGFR2 2263 AD 0.0000 0.9981 1.0000 1.0000 0.8808 1.0000 0.0000 0.5095 1.0000 1.0000 0 Jackson-Weiss syndrome (OMIM:123150): Brachydactyly (HP:0001156)-Broad hallux (HP:0010055), Craniosynostosis (HP:0001363)-Craniosynostosis (HP:0001363), Broad thumb (HP:0011304)-Broad metatarsal (HP:0001783), Broad hallux (HP:0010055)-Broad hallux (HP:0010055), Brachydactyly (HP:0001156)-abnormal sternum morphology (MP:0000157), Craniosynostosis (HP:0001363)-premature cranial suture closure (MP:0000081), Broad thumb (HP:0011304)-abnormal sternum morphology (MP:0000157), Broad hallux (HP:0010055)-abnormal sternum morphology (MP:0000157), Proximity to FGF14 associated with Spinocerebellar ataxia 27 (OMIM:609307): Broad hallux (HP:0010055)-Pes cavus (HP:0001761), Proximity to FGF14 Brachydactyly (HP:0001156)-abnormal digit morphology (MP:0002110), Broad thumb (HP:0011304)-abnormal digit morphology (MP:0002110), Broad hallux (HP:0010055)-abnormal digit morphology (MP:0002110),
+2 ENPP1_AD ENPP1 5167 AD 0.0049 0.8690 0.5773 0.9996 0.6972 0.5773 0.5237 0.5066 0.6972 1.0000 0 Autosomal recessive hypophosphatemic rickets (ORPHA:289176): Brachydactyly (HP:0001156)-Genu varum (HP:0002970), Craniosynostosis (HP:0001363)-Craniosynostosis (HP:0001363), Broad thumb (HP:0011304)-Tibial bowing (HP:0002982), Broad hallux (HP:0010055)-Genu varum (HP:0002970), Brachydactyly (HP:0001156)-fused carpal bones (MP:0008915), Craniosynostosis (HP:0001363)-abnormal nucleus pulposus morphology (MP:0006392), Broad thumb (HP:0011304)-fused carpal bones (MP:0008915), Broad hallux (HP:0010055)-fused carpal bones (MP:0008915), Craniosynostosis (HP:0001363)-ceratohyal cartilage premature perichondral ossification, abnormal (ZP:0012007), Broad thumb (HP:0011304)-cleithrum nodular, abnormal (ZP:0006782), Proximity to PAPSS2 associated with Brachyolmia 4 with mild epiphyseal and metaphyseal changes (OMIM:612847): Brachydactyly (HP:0001156)-Brachydactyly (HP:0001156), Broad thumb (HP:0011304)-Brachydactyly (HP:0001156), Broad hallux (HP:0010055)-Brachydactyly (HP:0001156), Proximity to PAPSS2 Brachydactyly (HP:0001156)-abnormal long bone epiphyseal plate morphology (MP:0003055), Craniosynostosis (HP:0001363)-domed cranium (MP:0000440), Broad thumb (HP:0011304)-abnormal long bone epiphyseal plate morphology (MP:0003055), Broad hallux (HP:0010055)-abnormal long bone epiphyseal plate morphology (MP:0003055),
+//
+7 MUC6_AD MUC6 4588 AD 0.0096 0.7532 0.5030 0.9990 0.0000 0.0000 0.0000 0.5030 0.5030 1.0000 0 Proximity to GKN2 Brachydactyly (HP:0001156)-brachydactyly (MP:0002544), Broad thumb (HP:0011304)-brachydactyly (MP:0002544), Broad hallux (HP:0010055)-brachydactyly (MP:0002544),
+8 MUC6_AR MUC6 4588 AR 0.0096 0.7531 0.5030 0.9990 0.0000 0.0000 0.0000 0.5030 0.5030 1.0000 0 Proximity to GKN2 Brachydactyly (HP:0001156)-brachydactyly (MP:0002544), Broad thumb (HP:0011304)-brachydactyly (MP:0002544), Broad hallux (HP:0010055)-brachydactyly (MP:0002544),
+```
+
+
+## TSV VARIANTS
+
+In the variants.tsv file it is possible for a variant, like a gene, to appear multiple times, depending on the MOI it is
+compatible with. For example in the example below MUC6 has two variants ranked 7th under the AD model and two ranked 8th
+under an AR (compound heterozygous) model. In the AD case the CONTRIBUTING_VARIANT column indicates whether the variant
+was (1) or wasn't (0) used for calculating the EXOMISER_GENE_COMBINED_SCORE and EXOMISER_GENE_VARIANT_SCORE.
+
+``` tsv
+
+ #RANK ID GENE_SYMBOL ENTREZ_GENE_ID MOI P-VALUE EXOMISER_GENE_COMBINED_SCORE EXOMISER_GENE_PHENO_SCORE EXOMISER_GENE_VARIANT_SCORE EXOMISER_VARIANT_SCORE CONTRIBUTING_VARIANT WHITELIST_VARIANT VCF_ID RS_ID CONTIG START END REF ALT CHANGE_LENGTH QUAL FILTER GENOTYPE FUNCTIONAL_CLASS HGVS EXOMISER_ACMG_CLASSIFICATION EXOMISER_ACMG_EVIDENCE EXOMISER_ACMG_DISEASE_ID EXOMISER_ACMG_DISEASE_NAME CLINVAR_VARIANT_ID CLINVAR_PRIMARY_INTERPRETATION CLINVAR_STAR_RATING GENE_CONSTRAINT_LOEUF GENE_CONSTRAINT_LOEUF_LOWER GENE_CONSTRAINT_LOEUF_UPPER MAX_FREQ_SOURCE MAX_FREQ ALL_FREQ MAX_PATH_SOURCE MAX_PATH ALL_PATH
+ 1 10-123256215-T-G_AD FGFR2 2263 AD 0.0000 0.9981 1.0000 1.0000 1.0000 1 1 rs121918506 10 123256215 123256215 T G 0 100.0000 PASS 1|0 missense_variant FGFR2:ENST00000346997.2:c.1688A>C:p.(Glu563Ala) LIKELY_PATHOGENIC PM2,PP3_Strong,PP4,PP5 OMIM:123150 Jackson-Weiss syndrome 28333 LIKELY_PATHOGENIC 1 0.13692 0.074 0.27 REVEL 0.965 REVEL=0.965,MVP=0.9517972
+ 2 6-132203615-G-A_AD ENPP1 5167 AD 0.0049 0.8690 0.5773 0.9996 0.9996 1 0 rs770775549 6 132203615 132203615 G A 0 922.9800 PASS 0/1 splice_donor_variant ENPP1:ENST00000360971.2:c.2230+1G>A:p.? UNCERTAIN_SIGNIFICANCE PVS1_Strong OMIM:615522 Cole disease NOT_PROVIDED 0 0.41042 0.292 0.586 GNOMAD_E_SAS 0.0032486517 TOPMED=7.556E-4,EXAC_NON_FINNISH_EUROPEAN=0.0014985314,GNOMAD_E_NFE=0.0017907989,GNOMAD_E_SAS=0.0032486517
+ //
+ 7 11-1018088-TG-T_AD MUC6 4588 AD 0.0096 0.7532 0.5030 0.9990 0.9990 1 0 rs765231061 11 1018088 1018089 TG T -1 441.8100 PASS 0/1 frameshift_variant MUC6:ENST00000421673.2:c.4712del:p.(Pro1571Hisfs*21) UNCERTAIN_SIGNIFICANCE NOT_PROVIDED 0 0.79622 0.656 0.971 GNOMAD_G_NFE 0.0070363074 GNOMAD_E_AMR=0.0030803352,GNOMAD_G_NFE=0.0070363074
+ 7 11-1018093-G-GT_AD MUC6 4588 AD 0.0096 0.7532 0.5030 0.9990 0.9989 0 0 rs376177791 11 1018093 1018093 G GT 1 592.4500 PASS 0/1 frameshift_elongation MUC6:ENST00000421673.2:c.4707dup:p.(Pro1570Thrfs*136) NOT_AVAILABLE NOT_PROVIDED 0 0.79622 0.656 0.971 GNOMAD_G_NFE 0.007835763 GNOMAD_G_NFE=0.007835763
+ 8 11-1018088-TG-T_AR MUC6 4588 AR 0.0096 0.7531 0.5030 0.9990 0.9990 1 0 rs765231061 11 1018088 1018089 TG T -1 441.8100 PASS 0/1 frameshift_variant MUC6:ENST00000421673.2:c.4712del:p.(Pro1571Hisfs*21) UNCERTAIN_SIGNIFICANCE NOT_PROVIDED 0 0.79622 0.656 0.971 GNOMAD_G_NFE 0.0070363074 GNOMAD_E_AMR=0.0030803352,GNOMAD_G_NFE=0.0070363074
+ 8 11-1018093-G-GT_AR MUC6 4588 AR 0.0096 0.7531 0.5030 0.9990 0.9989 1 0 rs376177791 11 1018093 1018093 G GT 1 592.4500 PASS 0/1 frameshift_elongation MUC6:ENST00000421673.2:c.4707dup:p.(Pro1570Thrfs*136) UNCERTAIN_SIGNIFICANCE NOT_PROVIDED 0 0.79622 0.656 0.971 GNOMAD_G_NFE 0.007835763 GNOMAD_G_NFE=0.007835763
+```
+
+## VCF
+
+In the VCF file it is possible for a variant, like a gene, to appear multiple times, depending on the MOI it is compatible with. For example in the example below MUC6 has two variants ranked 7th under the AD model and two ranked 8th under an AR (compound heterozygous) model. In the AD case the CONTRIBUTING_VARIANT column indicates whether the variant was (1) or wasn't (0) used for calculating the EXOMISER_GENE_COMBINED_SCORE and EXOMISER_GENE_VARIANT_SCORE.
+The ``INFO`` field with the ``ID=Exomiser`` describes the internal format of this sub-field. Be aware that for multi-allelic sites, Exomiser will decompose and trim them for the proband sample and this is what will be displayed in the Exomiser ``ID`` sub-field e.g. ``11-1018088-TG-T_AD``.
+
+``` vcf
+
+ ##INFO=Output files
+ +- `bwa/` + - Index corresponding to the [BWA](https://github.com/lh3/bwa) aligner +- `bwamem2/` + - Index corresponding to the [BWA-mem2](https://github.com/bwa-mem2/bwa-mem2) aligner +- `cnvkit/` + - Reference files generated by [CNVKit](https://cnvkit.readthedocs.io/en/stable/) +- `dragmap/` + - Index corresponding to the [DragMap](https://github.com/Illumina/dragmap) aligner +- `dbsnp/` + - Tabix index generated by [Tabix](http://www.htslib.org/doc/tabix.html) from the given dbsnp file +- `dict/` + - Sequence dictionary generated by [GATK4 CreateSequenceDictionary](https://gatk.broadinstitute.org/hc/en-us/articles/5358872471963-CreateSequenceDictionary-Picard-) from the given fasta +- `fai/` + - Fasta index generated with [samtools faidx](http://www.htslib.org/doc/samtools-faidx.html) from the given fasta +- `germline_resource/` + - Tabix index generated by [Tabix](http://www.htslib.org/doc/tabix.html) from the given gernline resource file +- `intervals/` + - Bed files in various stages: .bed, .bed.gz, .bed per chromosome, .bed.gz per chromsome +- `known_indels/` + - Tabix index generated by [Tabix](http://www.htslib.org/doc/tabix.html) from the given known indels file +- `msi/` + - [MSIsensorPro](https://github.com/xjtu-omics/msisensor-pro) scan of the reference genome to get microsatellites information +- `pon/` + - Tabix index generated by [Tabix](http://www.htslib.org/doc/tabix.html) from the given panel-of-normals file +
+
+
+If multiple libraries/runs have been provided for the same sample in the input samplesheet (e.g. to increase sequencing depth) then these will be merged at the very beginning of the pipeline in order to have consistent sample naming throughout the pipeline. Please refer to the [usage documentation](https://nf-co.re/viralrecon/usage#illumina-samplesheet-format) to see how to specify these samples in the input samplesheet.
+
+### FastQC
Output files
+ +- `fastq/` + - `*.merged.fastq.gz`: These files are not saved by default but can be via a custom config file such as the one below. + +```nextflow +params { + modules { + 'illumina_cat_fastq' { + publish_files = null + } + } +} +``` + +Output files
-- `pycoqc/` - - `*.html` and `.json` file that includes a run summary and graphical representation of various QC metrics including distribution of read length, distribution of read quality scores, mean read quality per sequence length, output per channel over experiment time and percentage of reads per barcode. +- `fastqc/raw/` + - `*_fastqc.html`: FastQC report containing quality metrics. + - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. + +**NB:** The FastQC plots in this directory are generated relative to the raw, input reads. They may contain adapter sequence and regions of low quality. To see how your reads look after trimming please refer to the FastQC reports in the `fastqc/trim/` directory.
Output files
-- `guppyplex/` - - `*.fastq.gz` files generated by aggregate pre-demultiplexed reads from MinKNOW/Guppy. These files are not saved by default but can be via a custom config file such as the one below. - -```nextflow -params { - modules { - 'nanopore_artic_guppyplex' { - publish_files = ['fastq.gz':''] - } - } -} -``` +- `fastp/` + - `*.fastp.html`: Trimming report in html format. + - `*.fastp.json`: Trimming report in json format. +- `fastp/log/` + - `*.fastp.log`: Trimming log file. +- `fastqc/trim/` + - `*_fastqc.html`: FastQC report of the trimmed reads. + - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images.Output files
-- `nanoplot/
Output files
-- `
+.sorted.bam`: Coordinate sorted BAM file containing read alignment information.
+ - `.sorted.bam.bai`: Index file for coordinate sorted BAM file.
+- `variants/bowtie2/samtools_stats/`
+ - SAMtools `.sorted.bam.flagstat`, `.sorted.bam.idxstats` and `.sorted.bam.stats` files generated from the alignment files.
-The [artic minion](https://artic.readthedocs.io/en/latest/commands/) tool from the [ARTIC field bioinformatics pipeline](https://github.com/artic-network/fieldbioinformatics) is used to align reads, call variants and to generate the consensus sequence. By default, artic minion uses [Minimap2](https://github.com/lh3/minimap2) to align the reads to the viral genome, however you can use [BWA](https://github.com/lh3/bwa) instead using the `--artic_minion_aligner bwa` parameter. Similarly, the default variant caller used by artic minion is [Nanopolish](https://github.com/jts/nanopolish), however, you can use [Medaka](https://github.com/nanoporetech/medaka) instead via the `--artic_minion_caller medaka` parameter. Medaka is faster than Nanopolish, performs mostly the same and can be run directly from `fastq` input files as opposed to requiring the `fastq`, `fast5` and `sequencing_summary.txt` files required to run Nanopolish. You must provide the appropriate [Medaka model](https://github.com/nanoporetech/medaka#models) via the `--artic_minion_medaka_model` parameter if using `--artic_minion_caller medaka`.
+Bowtie 2 BAM files are further processed with [SAMtools](http://samtools.sourceforge.net/) to sort them by coordinate, for indexing, as well as to generate read mapping statistics.
-## Nanopore: Downstream analysis
+
-### Nanopore: SAMtools
+### iVar trim
Output files
+ +- `variants/bowtie2/` + - `Output files
-- `
+
-BAM files containing the original alignments from either Minimap2 or BWA are further processed with [SAMtools](http://samtools.sourceforge.net/) to remove unmapped reads as well as to generate read mapping statistics.
+Unless you are using [UMIs](https://emea.illumina.com/science/sequencing-method-explorer/kits-and-arrays/umi.html) it is not possible to establish whether the fragments you have sequenced from your sample were derived via true biological duplication (i.e. sequencing independent template fragments) or as a result of PCR biases introduced during the library preparation. [picard MarkDuplicates](https://gatk.broadinstitute.org/hc/en-us/articles/360037052812-MarkDuplicates-Picard-) isn't run by default because you anticipate high levels of duplication with viral data due to the size of the genome, however, you can activate it by adding `--skip_markduplicates false` to the command you use to run the pipeline. This will only _mark_ the duplicate reads identified amongst the alignments to allow you to guage the overall level of duplication in your samples. You can also choose to remove any reads identified as duplicates via the `--filter_duplicates` parameter.
-
+
-### Nanopore: mosdepth
+### picard CollectMultipleMetrics
Output files
+ +- `variants/bowtie2/` + - `*.markduplicates.sorted.bam`: Coordinate sorted BAM file after duplicate marking. + - `*.markduplicates.sorted.bam.bai`: Index file for coordinate sorted BAM file after duplicate marking. +- `variants/bowtie2/samtools_stats/` + - SAMtools `*.markduplicates.sorted.bam.flagstat`, `*.markduplicates.sorted.bam.idxstats` and `*.markduplicates.sorted.bam.stats` files generated from the duplicate marked alignment files. +- `variants/bowtie2/picard_metrics/` + - `*.markduplicates.sorted.MarkDuplicates.metrics.txt`: Metrics file from MarkDuplicates.Output files
-- `
+/mosdepth/amplicon/`
+- `variants/bowtie2/mosdepth/amplicon/`
- `all_samples.mosdepth.coverage.tsv`: File aggregating per-amplicon coverage values across all samples used for plotting.
- `all_samples.mosdepth.heatmap.pdf`: Heatmap showing per-amplicon coverage across all samples.
- `*.mosdepth.coverage.pdf`: Bar plot showing per-amplicon coverage for an individual sample.
- `*.mosdepth.coverage.tsv`: File containing per-amplicon coverage values for the above plot.
- `*.mosdepth.summary.txt`: Summary metrics including mean, min and max coverage values.
-**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-
-[mosdepth](mosdepth) is a fast BAM/CRAM depth calculation for WGS, exome, or targeted sequencing. mosdepth is used in this pipeline to obtain genome-wide coverage values in 200bp windows and to obtain amplicon/region-specific coverage metrics. The results are then either rendered in MultiQC (genome-wide coverage) or are plotted using custom `R` scripts.
+[mosdepth](mosdepth) is a fast BAM/CRAM depth calculation for WGS, exome, or targeted sequencing. mosdepth is used in this pipeline to obtain genome-wide coverage values in 200bp windows and for `--protocol amplicon` to obtain amplicon/region-specific coverage metrics. The results are then either rendered in MultiQC (genome-wide coverage) or are plotted using custom `R` scripts.
@@ -158,38 +254,61 @@ BAM files containing the original alignments from either Minimap2 or BWA are fur
Output files
+ +- `variants/bowtie2/mosdepth/genome/` - `all_samples.mosdepth.coverage.tsv`: File aggregating genome-wide coverage values across all samples used for plotting. - `*.mosdepth.coverage.pdf`: Whole-genome coverage plot. - `*.mosdepth.coverage.tsv`: File containing coverage values for the above plot. - `*.mosdepth.summary.txt`: Summary metrics including mean, min and max coverage values. -- `
Output files
-- `
+
-[BCFtools](http://samtools.github.io/bcftools/bcftools.html) is a set of utilities that manipulate variant calls in [VCF](https://vcftools.github.io/specs.html) and its binary counterpart BCF format. It can also used be used to generate statistics and counts obtained from VCF files as used here.
+[BCFtools](http://samtools.github.io/bcftools/bcftools.html) can be used to call variants directly from BAM alignment files. It is a set of utilities that manipulate variant calls in [VCF](https://vcftools.github.io/specs.html) and its binary counterpart BCF format. BCFTools is used in the variant calling and _de novo_ assembly steps of this pipeline to obtain basic statistics from the VCF output.
-### Nanopore: SnpEff and SnpSift
+### SnpEff and SnpSift
Output files
+ +- `variants/bcftools/` + - `*.vcf.gz`: Variants VCF file. + - `*.vcf.gz.tbi`: Variants VCF index file. +- `variants/bcftools/bcftools_stats/` + - `*.bcftools_stats.txt`: Statistics and counts obtained from VCF file.Output files
-- `Output files
-- `
Output files
-- `Output files
-- `Output files
-- `
Output files
-- `Output files
-- `multiqc/Output files
-- `fastq/` - - `*.merged.fastq.gz`: These files are not saved by default but can be via a custom config file such as the one below. +- `variants/
-
+```bash
+SAMPLE,CHROM,POS,REF,ALT,FILTER,DP,REF_DP,ALT_DP,AF,GENE,EFFECT,HGVS_C,HGVS_P,HGVS_P_1LETTER,CALLER,LINEAGE
+SAMPLE1_PE,MN908947.3,241,C,T,PASS,489,4,483,0.99,orf1ab,upstream_gene_variant,c.-25C>T,.,.,ivar,B.1
+SAMPLE1_PE,MN908947.3,1875,C,T,PASS,92,62,29,0.32,orf1ab,missense_variant,c.1610C>T,p.Ala537Val,p.A537V,ivar,B.1
+SAMPLE1_PE,MN908947.3,3037,C,T,PASS,213,0,213,1.0,orf1ab,synonymous_variant,c.2772C>T,p.Phe924Phe,p.F924F,ivar,B.1
+SAMPLE1_PE,MN908947.3,11719,G,A,PASS,195,9,186,0.95,orf1ab,synonymous_variant,c.11454G>A,p.Gln3818Gln,p.Q3818Q,ivar,B.1
+```
-[FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/).
+## Illumina: De novo assembly
-
+A file called `summary_assembly_metrics_mqc.csv` containing a selection of read alignment and _de novo_ assembly related metrics will be saved in the `multiqc/` results directory. The same metrics will also be added to the top of the MultiQC report.
-### fastp
+### Cutadapt
Output files
- -- `fastqc/raw/` - - `*_fastqc.html`: FastQC report containing quality metrics. - - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. +Create variants long format table collating per-sample information for individual variants ([`BCFTools`](http://samtools.github.io/bcftools/bcftools.html)), functional effect prediction ([`SnpSift`](http://snpeff.sourceforge.net/SnpSift.html)) and lineage analysis ([`Pangolin`](https://github.com/cov-lineages/pangolin)). -**NB:** The FastQC plots in this directory are generated relative to the raw, input reads. They may contain adapter sequence and regions of low quality. To see how your reads look after trimming please refer to the FastQC reports in the `fastqc/trim/` directory. +The more pertinent variant information is summarised in this table to make it easier for researchers to assess the impact of variants found amongst the sequenced sample(s). An example of the fields included in the table are shown below: -Output files
-- `fastp/` - - `*.fastp.html`: Trimming report in html format. - - `*.fastp.json`: Trimming report in json format. -- `fastp/log/` - - `*.fastp.log`: Trimming log file. -- `fastqc/trim/` +- `assembly/cutadapt/log/` + - `*.cutadapt.log`: Cutadapt log file generated from stdout. +- `assembly/cutadapt/fastqc/` - `*_fastqc.html`: FastQC report of the trimmed reads. - - `*_fastqc.zip`: Zip archive containing the FastQC report, tab-delimited data file and plot images. + - `*_fastqc.zip`: Zip archive containing the FastQC report.Output files
-- `kraken2/` - - `*.kraken2.report.txt`: Kraken 2 taxonomic report. See [here](https://ccb.jhu.edu/software/kraken2/index.shtml?t=manual#sample-report-output-format) for a detailed description of the format. - -
-/`
+ - `*.scaffolds.fa.gz`: SPAdes scaffold assembly.
+ - `*.contigs.fa.gz`: SPAdes assembly contigs.
+ - `*.assembly.gfa.gz`: SPAdes assembly graph in [GFA](https://github.com/GFA-spec/GFA-spec/blob/master/GFA1.md) format.
+- `assembly/spades//bandage/`
+ - `*.png`: Bandage visualisation for SPAdes assembly graph in PNG format.
+ - `*.svg`: Bandage visualisation for SPAdes assembly graph in SVG format.
-- `variants/bowtie2/log/`
- - `*.bowtie2.log`: Bowtie 2 mapping log file.
+**NB:** The value of `` in the output directory name above is determined by the `--spades_mode` parameter (Default: 'rnaviral').
-[Bowtie 2](http://bio-bwa.sourceforge.net/) is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. Bowtie 2 supports gapped, local, and paired-end alignment modes.
+[SPAdes](http://cab.spbu.ru/software/spades/) is an assembly toolkit containing various assembly pipelines. Generically speaking, SPAdes is one of the most popular de Bruijn graph-based assembly algorithms used for bacterial/viral genome reconstruction.
-
+[Bandage](https://rrwick.github.io/Bandage/) is a program for visualising _de novo_ assembly graphs. By displaying connections which are not present in the contigs file, Bandage opens up new possibilities for analysing _de novo_ assemblies.
-### SAMtools
+### Unicycler
Output files
+- `assembly/spades/Output files
-- `variants/bowtie2/` - - `Output files
-- `variants/bowtie2/` - - `*.ivar_trim.sorted.bam`: Coordinate sorted BAM file after primer trimming. - - `*.ivar_trim.sorted.bam.bai`: Index file for coordinate sorted BAM file after primer trimming. -- `variants/bowtie2/samtools_stats/` - - SAMtools `*.ivar_trim.sorted.bam.flagstat`, `*.ivar_trim.sorted.bam.idxstats` and `*.ivar_trim.sorted.bam.stats` files generated from the primer trimmed alignment files. -- `variants/bowtie2/log/` - - `*.ivar_trim.ivar.log`: iVar trim log file obtained from stdout. +- `assembly/minia/` + - `*.contigs.fa`: Minia scaffold assembly. + - `*.unitigs.fa`: Minia unitigs fasta file. + - `*.h5`: Minia h5 output file.Output files
-- `variants/bowtie2/` - - `*.markduplicates.sorted.bam`: Coordinate sorted BAM file after duplicate marking. - - `*.markduplicates.sorted.bam.bai`: Index file for coordinate sorted BAM file after duplicate marking. -- `variants/bowtie2/samtools_stats/` - - SAMtools `*.markduplicates.sorted.bam.flagstat`, `*.markduplicates.sorted.bam.idxstats` and `*.markduplicates.sorted.bam.stats` files generated from the duplicate marked alignment files. -- `variants/bowtie2/picard_metrics/` - - `*.markduplicates.sorted.MarkDuplicates.metrics.txt`: Metrics file from MarkDuplicates. - -
-/blastn/`
+ - `*.blastn.txt`: BLAST results against the target virus.
+ - `*.filter.blastn.txt`: Filtered BLAST results.
-- `variants/bowtie2/picard_metrics/`
- - `*.CollectMultipleMetrics.*`: Alignment QC files from picard CollectMultipleMetrics in `*_metrics` textual format.
-- `variants/bowtie2/picard_metrics/pdf/`
- - `*.pdf` plots for metrics obtained from CollectMultipleMetrics.
+**NB:** The value of `` in the output directory name above is determined by the `--assemblers` parameter (Default: 'spades').
-[picard-tools](https://broadinstitute.github.io/picard/command-line-overview.html) is a set of command-line tools for manipulating high-throughput sequencing data. We use picard-tools in this pipeline to obtain mapping and coverage metrics.
-
-
+[blastn](https://blast.ncbi.nlm.nih.gov/Blast.cgi?PAGE_TYPE=BlastSearch) is used to align the assembled contigs against the virus reference genome.
-### mosdepth
+### ABACAS
Output files
+- `assembly/Output files
-- `variants/bowtie2/mosdepth/genome/` - - `all_samples.mosdepth.coverage.tsv`: File aggregating genome-wide coverage values across all samples used for plotting. - - `*.mosdepth.coverage.pdf`: Whole-genome coverage plot. - - `*.mosdepth.coverage.tsv`: File containing coverage values for the above plot. - - `*.mosdepth.summary.txt`: Summary metrics including mean, min and max coverage values. -- `variants/bowtie2/mosdepth/amplicon/` - - `all_samples.mosdepth.coverage.tsv`: File aggregating per-amplicon coverage values across all samples used for plotting. - - `all_samples.mosdepth.heatmap.pdf`: Heatmap showing per-amplicon coverage across all samples. - - `*.mosdepth.coverage.pdf`: Bar plot showing per-amplicon coverage for an individual sample. - - `*.mosdepth.coverage.tsv`: File containing per-amplicon coverage values for the above plot. - - `*.mosdepth.summary.txt`: Summary metrics including mean, min and max coverage values. - -
-/abacas/`
+ - `*.abacas.bin`: Bin file that contains contigs that are not used in ordering.
+ - `*.abacas.crunch`: Comparison file.
+ - `*.abacas.fasta`: Ordered and orientated sequence file.
+ - `*.abacas.gaps`: Gap information.
+ - `*.abacas.gaps.tab`: Gap information in tab-delimited format.
+ - `*.abacas.MULTIFASTA.fa`: A list of ordered and orientated contigs in a multi-fasta format.
+ - `*.abacas.tab`: Feature file
+ - `*.unused_contigs.out`: Information on contigs that have a mapping information but could not be used in the ordering.
+- `assembly//abacas/nucmer/`: Folder containing the files generated by the NUCmer algorithm used by ABACAS.
-- `variants/ivar/`
- - `*.tsv`: Original iVar variants in TSV format.
- - `*.vcf.gz`: iVar variants in VCF format. Converted using custom `ivar_variants_to_vcf.py` python script.
- - `*.vcf.gz.tbi`: iVar variants VCF index file.
-- `variants/ivar/log/`
- - `*.variant_counts.log`: Counts for type of variants called by iVar.
-- `variants/ivar/bcftools_stats/`
- - `*.bcftools_stats.txt`: Statistics and counts obtained from iVar variants VCF file.
+**NB:** The value of `` in the output directory name above is determined by the `--assemblers` parameter (Default: 'spades').
-[iVar](https://github.com/andersen-lab/ivar/blob/master/docs/MANUAL.md) is a computational package that contains functions broadly useful for viral amplicon-based sequencing. We use iVar in this pipeline to [trim primer sequences](#ivar-trim) for amplicon input data as well as to call variants.
-
-iVar outputs a tsv format which is not compatible with downstream analysis such as annotation using SnpEff. Moreover some issues need to be addressed such as [strand-bias filtering](https://github.com/andersen-lab/ivar/issues/5) and [the consecutive reporting of variants belonging to the same codon](https://github.com/andersen-lab/ivar/issues/92). This pipeline uses a custom Python script [ivar_variants_to_vcf.py](https://github.com/nf-core/viralrecon/blob/master/bin/ivar_variants_to_vcf.py) to convert the default iVar output to VCF whilst also addressing both of these issues.
-
-
+[ABACAS](https://www.sanger.ac.uk/science/tools/pagit) was developed to rapidly contiguate (align, order, orientate), visualize and design primers to close gaps on shotgun assembled contigs based on a reference sequence.
-### BCFTools call
+### PlasmidID
Output files
+- `assembly/Output files
-- `variants/bcftools/` - - `*.vcf.gz`: Variants VCF file. - - `*.vcf.gz.tbi`: Variants VCF index file. -- `variants/bcftools/bcftools_stats/` - - `*.bcftools_stats.txt`: Statistics and counts obtained from VCF file. +- `assembly/Output files
-- `variants/Output files
-- `variants/
-.
-- `variants//consensus/ivar/`
- - `*.consensus.fa`: Consensus Fasta file generated by iVar.
- - `*.consensus.qual.txt`: File with the average quality of each base in the consensus sequence.
-- `variants//consensus/ivar/base_qc/`
- - `*.ACTG_density.pdf`: Plot showing density of ACGT bases within the consensus sequence.
- - `*.base_counts.pdf`: Plot showing frequency and percentages of all bases in consensus sequence.
- - `*.base_counts.tsv`: File containing frequency and percentages of all bases in consensus sequence.
- - `*.N_density.pdf`: Plot showing density of N bases within the consensus sequence.
- - `*.N_run.tsv`: File containing start positions and width of N bases in consensus sequence.
+An example MultiQC report generated from a full-sized dataset can be viewed on the [nf-core website](https://nf-co.re/viralrecon/results).
-**NB:** The value of `` in the output directory name above is determined by the `--variant_caller` parameter (Default: 'ivar' for '--protocol amplicon' and 'bcftools' for '--protocol metagenomic').
+# Nanopore: Pipeline overview
-
+- [nf-core/viralrecon Description](#nf-coreviralrecon-description)
+- [Illumina: Pipeline overview](#illumina-pipeline-overview)
+ - [Illumina: Preprocessing](#illumina-preprocessing)
+ - [cat](#cat)
+ - [FastQC](#fastqc)
+ - [fastp](#fastp)
+ - [Kraken 2](#kraken-2)
+ - [Illumina: Variant calling](#illumina-variant-calling)
+ - [Bowtie 2](#bowtie-2)
+ - [SAMtools](#samtools)
+ - [iVar trim](#ivar-trim)
+ - [picard MarkDuplicates](#picard-markduplicates)
+ - [picard CollectMultipleMetrics](#picard-collectmultiplemetrics)
+ - [mosdepth](#mosdepth)
+ - [iVar variants](#ivar-variants)
+ - [BCFTools call](#bcftools-call)
+ - [SnpEff and SnpSift](#snpeff-and-snpsift)
+ - [ASCIIGenome](#asciigenome)
+ - [iVar consensus](#ivar-consensus)
+ - [BCFTools and BEDTools](#bcftools-and-bedtools)
+ - [QUAST](#quast)
+ - [Pangolin](#pangolin)
+ - [Nextclade](#nextclade)
+ - [Variants long table](#variants-long-table)
+ - [Illumina: De novo assembly](#illumina-de-novo-assembly)
+ - [Cutadapt](#cutadapt)
+ - [SPAdes](#spades)
+ - [Unicycler](#unicycler)
+ - [minia](#minia)
+ - [BLAST](#blast)
+ - [ABACAS](#abacas)
+ - [PlasmidID](#plasmidid)
+ - [Assembly QUAST](#assembly-quast)
+ - [Illumina: Workflow reporting and genomes](#illumina-workflow-reporting-and-genomes)
+ - [MultiQC](#multiqc)
+- [Nanopore: Pipeline overview](#nanopore-pipeline-overview)
+ - [Nanopore: Preprocessing](#nanopore-preprocessing)
+ - [Nanopore: pycoQC](#nanopore-pycoqc)
+ - [Nanopore: artic guppyplex](#nanopore-artic-guppyplex)
+ - [Nanopore: NanoPlot](#nanopore-nanoplot)
+ - [Nanopore: Variant calling](#nanopore-variant-calling)
+ - [Nanopore: artic minion](#nanopore-artic-minion)
+ - [Nanopore: Downstream analysis](#nanopore-downstream-analysis)
+ - [Nanopore: SAMtools](#nanopore-samtools)
+ - [Nanopore: mosdepth](#nanopore-mosdepth)
+ - [Nanopore: BCFTools](#nanopore-bcftools)
+ - [Nanopore: SnpEff and SnpSift](#nanopore-snpeff-and-snpsift)
+ - [Nanopore: QUAST](#nanopore-quast)
+ - [Nanopore: Pangolin](#nanopore-pangolin)
+ - [Nanopore: Nextclade](#nanopore-nextclade)
+ - [Nanopore: ASCIIGenome](#nanopore-asciigenome)
+ - [Nanopore: Variants long table](#nanopore-variants-long-table)
+ - [Nanopore: Workflow reporting](#nanopore-workflow-reporting)
+ - [Nanopore: MultiQC](#nanopore-multiqc)
+ - [Reference genome files](#reference-genome-files)
+- [Pipeline information](#pipeline-information)
-As described in the [iVar variants](#ivar-variants) section, iVar can be used in this pipeline to call variants and for the consensus sequence generation.
+## Nanopore: Preprocessing
-### BCFTools and BEDTools
+A file called `summary_variants_metrics_mqc.csv` containing a selection of read alignment and variant calling metrics will be saved in the `multiqc/Output files
+The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see
-/consensus/bcftools/`
- - `*.consensus.fa`: Consensus fasta file generated by integrating the high allele-frequency variants called by iVar/BCFTools into the reference genome.
- - `*.filtered.vcf.gz`: VCF file containing high allele-frequency variants (default: `>= 0.75`) that were integrated into the consensus sequence.
- - `*.filtered.vcf.gz.tbi`: Variants VCF index file for high allele frequency variants.
-- `variants//consensus/bcftools/base_qc/`
- - `*.ACTG_density.pdf`: Plot showing density of ACGT bases within the consensus sequence.
- - `*.base_counts.pdf`: Plot showing frequency and percentages of all bases in consensus sequence.
- - `*.base_counts.tsv`: File containing frequency and percentages of all bases in consensus sequence.
- - `*.N_density.pdf`: Plot showing density of N bases within the consensus sequence.
- - `*.N_run.tsv`: File containing start positions and width of N bases in consensus sequence.
+
/consensus//quast/`
- - `report.html`: Results report in HTML format. Also available in various other file formats i.e. `report.pdf`, `report.tex`, `report.tsv` and `report.txt`.
+- `guppyplex/`
+ - `*.fastq.gz` files generated by aggregate pre-demultiplexed reads from MinKNOW/Guppy. These files are not saved by default but can be via a custom config file such as the one below.
-**NB:** The value of `` in the output directory name above is determined by the `--variant_caller` parameter (Default: 'ivar' for '--protocol amplicon' and 'bcftools' for '--protocol metagenomic').
-**NB:** The value of `` in the output directory name above is determined by the `--consensus_caller` parameter (Default: 'bcftools' for both '--protocol amplicon' and '--protocol metagenomic').
+```nextflow
+params {
+ modules {
+ 'nanopore_artic_guppyplex' {
+ publish_files = ['fastq.gz':'']
+ }
+ }
+}
+```
-[QUAST](http://bioinf.spbau.ru/quast) is used to generate a single report with which to evaluate the quality of the consensus sequence across all of the samples provided to the pipeline. The HTML results can be opened within any browser (we recommend using Google Chrome). Please see the [QUAST output docs](http://quast.sourceforge.net/docs/manual.html#sec3) for more detailed information regarding the output files.
+The [artic guppyplex](https://artic.readthedocs.io/en/latest/commands/) tool from the [ARTIC field bioinformatics pipeline](https://github.com/artic-network/fieldbioinformatics) is used to perform length filtering of the demultiplexed Nanopore reads obtained per barcode. This essentially filters out chimeric reads that may be generated by the ARTIC protocol. The pipeline uses a default minimum and maximum read length of 400 and 700, respectively as tailored for the [nCoV-2019 primer set](https://artic.network/ncov-2019/ncov2019-bioinformatics-sop.html). However, you may need to adjust these for different primer schemes e.g. by using the minimum length of the amplicons (`--min-length`) as well as the maximum length plus 200 (`--max-length`).
-### Pangolin
+### Nanopore: NanoPlot
/consensus//pangolin/`
- - `*.pangolin.csv`: Lineage analysis results from Pangolin.
-
-**NB:** The value of `` in the output directory name above is determined by the `--variant_caller` parameter (Default: 'ivar' for '--protocol amplicon' and 'bcftools' for '--protocol metagenomic').
-**NB:** The value of `` in the output directory name above is determined by the `--consensus_caller` parameter (Default: 'bcftools' for both '--protocol amplicon' and '--protocol metagenomic').
+- `nanoplot//`
+ - Per-sample `*.html` files for QC metrics and individual `*.png` image files for plots.
-Phylogenetic Assignment of Named Global Outbreak LINeages ([Pangolin](https://github.com/cov-lineages/pangolin)) has been used extensively during the COVID-19 pandemic in order to to assign lineages to SARS-CoV-2 genome sequenced samples. A [web application](https://pangolin.cog-uk.io/) also exists that allows users to upload genome sequences via a web browser to assign lineages to genome sequences of SARS-CoV-2, view descriptive characteristics of the assigned lineage(s), view the placement of the lineage in a phylogeny of global samples, and view the temporal and geographic distribution of the assigned lineage(s).
+[NanoPlot](https://github.com/wdecoster/NanoPlot) it a tool that can be used to produce general quality metrics from various Nanopore-based input files including fastq files e.g. quality score distribution, read lengths and other general stats.
-### Nextclade
+
/consensus//nextclade/`
- - `*.csv`: Analysis results from Nextlade containing genome clade assignment, mutation calling and sequence quality checks.
+- `/`
+ - `*.consensus.fasta`: Consensus fasta file generated by artic minion.
+ - `*.pass.unique.vcf.gz`: VCF file containing unique variants passing quality filters.
+ - `*.pass.unique.vcf.gz.tbi`: VCF index file containing unique variants passing quality filters.
+ - `*.pass.vcf.gz`: VCF file containing variants passing quality filters.
+ - `*.pass.vcf.gz.tbi`: VCF index file containing variants passing quality filters.
+ - `*.primers.vcf`: VCF file containing variants found in primer-binding regions.
+ - `*.merged.vcf`: VCF file containing all detected variants.
+ - `*.fail.vcf`: VCF file containing variants failing quality filters.
+ - `*.sorted.bam`: BAM file generated by initial alignment.
+ - `*.sorted.bam.bai`: BAM index file generated by initial alignment.
+ - `*.trimmed.rg.sorted.bam`: BAM file without primer-binding site trimming.
+ - `*.trimmed.rg.sorted.bam.bai`: BAM index file without primer-binding site trimming.
+ - `*.primertrimmed.rg.sorted.bam`: BAM file generated after primer-binding site trimming.
+ - `*.primertrimmed.rg.sorted.bam.bai`: BAM index file generated after primer-binding site trimming.
-**NB:** The value of `` in the output directory name above is determined by the `--variant_caller` parameter (Default: 'ivar' for '--protocol amplicon' and 'bcftools' for '--protocol metagenomic').
-**NB:** The value of `` in the output directory name above is determined by the `--consensus_caller` parameter (Default: 'bcftools' for both '--protocol amplicon' and '--protocol metagenomic').
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[Nextclade](https://github.com/nextstrain/nextclade) performs viral genome clade assignment, mutation calling and sequence quality checks for the consensus sequences generated in this pipeline. Similar to Pangolin, it has been used extensively during the COVID-19 pandemic. A [web application](https://clades.nextstrain.org/) also exists that allows users to upload genome sequences via a web browser.
+The [artic minion](https://artic.readthedocs.io/en/latest/commands/) tool from the [ARTIC field bioinformatics pipeline](https://github.com/artic-network/fieldbioinformatics) is used to align reads, call variants and to generate the consensus sequence. By default, artic minion uses [Minimap2](https://github.com/lh3/minimap2) to align the reads to the viral genome, however you can use [BWA](https://github.com/lh3/bwa) instead using the `--artic_minion_aligner bwa` parameter. Similarly, the default variant caller used by artic minion is [Nanopolish](https://github.com/jts/nanopolish), however, you can use [Medaka](https://github.com/nanoporetech/medaka) instead via the `--artic_minion_caller medaka` parameter. Medaka is faster than Nanopolish, performs mostly the same and can be run directly from `fastq` input files as opposed to requiring the `fastq`, `fast5` and `sequencing_summary.txt` files required to run Nanopolish. You must provide the appropriate [Medaka model](https://github.com/nanoporetech/medaka#models) via the `--artic_minion_medaka_model` parameter if using `--artic_minion_caller medaka`.
-### Variants long table
+## Nanopore: Downstream analysis
+
+### Nanopore: SAMtools
/`
- - `variants_long_table.csv`: Long format table collating per-sample information for individual variants, functional effect prediction and lineage analysis.
+- `/`
+ - `*.mapped.sorted.bam`: Coordinate sorted BAM file containing read alignment information.
+ - `*.mapped.sorted.bam.bai`: Index file for coordinate sorted BAM file.
+- `/samtools_stats/`
+ - SAMtools `*.mapped.sorted.bam.flagstat`, `*.mapped.sorted.bam.idxstats` and `*.mapped.sorted.bam.stats` files generated from the alignment files.
-**NB:** The value of `` in the output directory name above is determined by the `--variant_caller` parameter (Default: 'ivar' for '--protocol amplicon' and 'bcftools' for '--protocol metagenomic').
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-Create variants long format table collating per-sample information for individual variants ([`BCFTools`](http://samtools.github.io/bcftools/bcftools.html)), functional effect prediction ([`SnpSift`](http://snpeff.sourceforge.net/SnpSift.html)) and lineage analysis ([`Pangolin`](https://github.com/cov-lineages/pangolin)).
-
-The more pertinent variant information is summarised in this table to make it easier for researchers to assess the impact of variants found amongst the sequenced sample(s). An example of the fields included in the table are shown below:
-
-```bash
-SAMPLE,CHROM,POS,REF,ALT,FILTER,DP,REF_DP,ALT_DP,AF,GENE,EFFECT,HGVS_C,HGVS_P,HGVS_P_1LETTER,CALLER,LINEAGE
-SAMPLE1_PE,MN908947.3,241,C,T,PASS,489,4,483,0.99,orf1ab,upstream_gene_variant,c.-25C>T,.,.,ivar,B.1
-SAMPLE1_PE,MN908947.3,1875,C,T,PASS,92,62,29,0.32,orf1ab,missense_variant,c.1610C>T,p.Ala537Val,p.A537V,ivar,B.1
-SAMPLE1_PE,MN908947.3,3037,C,T,PASS,213,0,213,1.0,orf1ab,synonymous_variant,c.2772C>T,p.Phe924Phe,p.F924F,ivar,B.1
-SAMPLE1_PE,MN908947.3,11719,G,A,PASS,195,9,186,0.95,orf1ab,synonymous_variant,c.11454G>A,p.Gln3818Gln,p.Q3818Q,ivar,B.1
-```
-
-## Illumina: De novo assembly
+BAM files containing the original alignments from either Minimap2 or BWA are further processed with [SAMtools](http://samtools.sourceforge.net/) to remove unmapped reads as well as to generate read mapping statistics.
-A file called `summary_assembly_metrics_mqc.csv` containing a selection of read alignment and _de novo_ assembly related metrics will be saved in the `multiqc/` results directory. The same metrics will also be added to the top of the MultiQC report.
+
-### Cutadapt
+### Nanopore: mosdepth
/mosdepth/genome/`
+ - `all_samples.mosdepth.coverage.tsv`: File aggregating genome-wide coverage values across all samples used for plotting.
+ - `*.mosdepth.coverage.pdf`: Whole-genome coverage plot.
+ - `*.mosdepth.coverage.tsv`: File containing coverage values for the above plot.
+ - `*.mosdepth.summary.txt`: Summary metrics including mean, min and max coverage values.
+- `/mosdepth/amplicon/`
+ - `all_samples.mosdepth.coverage.tsv`: File aggregating per-amplicon coverage values across all samples used for plotting.
+ - `all_samples.mosdepth.heatmap.pdf`: Heatmap showing per-amplicon coverage across all samples.
+ - `*.mosdepth.coverage.pdf`: Bar plot showing per-amplicon coverage for an individual sample.
+ - `*.mosdepth.coverage.tsv`: File containing per-amplicon coverage values for the above plot.
+ - `*.mosdepth.summary.txt`: Summary metrics including mean, min and max coverage values.
+
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-In the variant calling branch of the pipeline we are using [iVar trim](#ivar-trim) to remove primer sequences from the aligned BAM files for amplicon data. Since in the _de novo_ assembly branch we don't align the reads, we use [Cutadapt](https://cutadapt.readthedocs.io/en/stable/guide.html) as an alternative option to remove and clean the primer sequences directly from FastQ files.
+[mosdepth](mosdepth) is a fast BAM/CRAM depth calculation for WGS, exome, or targeted sequencing. mosdepth is used in this pipeline to obtain genome-wide coverage values in 200bp windows and to obtain amplicon/region-specific coverage metrics. The results are then either rendered in MultiQC (genome-wide coverage) or are plotted using custom `R` scripts.
-
+
-### SPAdes
+
+
+
/`
- - `*.scaffolds.fa.gz`: SPAdes scaffold assembly.
- - `*.contigs.fa.gz`: SPAdes assembly contigs.
- - `*.assembly.gfa.gz`: SPAdes assembly graph in [GFA](https://github.com/GFA-spec/GFA-spec/blob/master/GFA1.md) format.
-- `assembly/spades//bandage/`
- - `*.png`: Bandage visualisation for SPAdes assembly graph in PNG format.
- - `*.svg`: Bandage visualisation for SPAdes assembly graph in SVG format.
+- `/bcftools_stats/`
+ - `*.bcftools_stats.txt`: Statistics and counts obtained from VCF file.
-**NB:** The value of `` in the output directory name above is determined by the `--spades_mode` parameter (Default: 'rnaviral').
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[SPAdes](http://cab.spbu.ru/software/spades/) is an assembly toolkit containing various assembly pipelines. Generically speaking, SPAdes is one of the most popular de Bruijn graph-based assembly algorithms used for bacterial/viral genome reconstruction.
+[BCFtools](http://samtools.github.io/bcftools/bcftools.html) is a set of utilities that manipulate variant calls in [VCF](https://vcftools.github.io/specs.html) and its binary counterpart BCF format. It can also used be used to generate statistics and counts obtained from VCF files as used here.
-[Bandage](https://rrwick.github.io/Bandage/) is a program for visualising _de novo_ assembly graphs. By displaying connections which are not present in the contigs file, Bandage opens up new possibilities for analysing _de novo_ assemblies.
+
-### Unicycler
+### Nanopore: SnpEff and SnpSift
/snpeff/`
+ - `*.snpeff.csv`: Variant annotation csv file.
+ - `*.snpeff.genes.txt`: Gene table for annotated variants.
+ - `*.snpeff.summary.html`: Summary html file for variants.
+ - `*.snpeff.vcf.gz`: VCF file with variant annotations.
+ - `*.snpeff.vcf.gz.tbi`: Index for VCF file with variant annotations.
+ - `*.snpsift.txt`: SnpSift summary table.
+- `/snpeff/bcftools_stats/`
+ - `*.snpeff.bcftools_stats.txt`: Statistics and counts obtained from SnpEff VCF file.
+
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[Unicycler](https://github.com/rrwick/Unicycler) is an assembly pipeline for bacterial genomes. It can assemble Illumina-only read sets where it functions as a SPAdes-optimiser.
+[SnpEff](http://snpeff.sourceforge.net/SnpEff.html) is a genetic variant annotation and functional effect prediction toolbox. It annotates and predicts the effects of genetic variants on genes and proteins (such as amino acid changes).
-### minia
+[SnpSift](http://snpeff.sourceforge.net/SnpSift.html) annotates genomic variants using databases, filters, and manipulates genomic annotated variants. After annotation with SnpEff, you can use SnpSift to help filter large genomic datasets in order to find the most significant variants.
+
+
+
+### Nanopore: QUAST
/quast/`
+ - `report.html`: Results report in HTML format. Also available in various other file formats i.e. `report.pdf`, `report.tex`, `report.tsv` and `report.txt`.
+
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[Minia](https://github.com/GATB/minia) is a short-read assembler based on a de Bruijn graph, capable of assembling a human genome on a desktop computer in a day. The output of Minia is a set of contigs. Minia produces results of similar contiguity and accuracy to other de Bruijn assemblers.
+[QUAST](http://bioinf.spbau.ru/quast) is used to generate a single report with which to evaluate the quality of the consensus sequence across all of the samples provided to the pipeline. The HTML results can be opened within any browser (we recommend using Google Chrome). Please see the [QUAST output docs](http://quast.sourceforge.net/docs/manual.html#sec3) for more detailed information regarding the output files.
-### BLAST
+### Nanopore: Pangolin
/blastn/`
- - `*.blastn.txt`: BLAST results against the target virus.
- - `*.filter.blastn.txt`: Filtered BLAST results.
+- `/pangolin/`
+ - `*.pangolin.csv`: Lineage analysis results from Pangolin.
-**NB:** The value of `` in the output directory name above is determined by the `--assemblers` parameter (Default: 'spades').
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[blastn](https://blast.ncbi.nlm.nih.gov/Blast.cgi?PAGE_TYPE=BlastSearch) is used to align the assembled contigs against the virus reference genome.
+Phylogenetic Assignment of Named Global Outbreak LINeages ([Pangolin](https://github.com/cov-lineages/pangolin)) has been used extensively during the COVID-19 pandemic to assign lineages to SARS-CoV-2 genome sequenced samples. A [web application](https://pangolin.cog-uk.io/) also exists that allows users to upload genome sequences via a web browser to assign lineages to genome sequences of SARS-CoV-2, view descriptive characteristics of the assigned lineage(s), view the placement of the lineage in a phylogeny of global samples, and view the temporal and geographic distribution of the assigned lineage(s).
-### ABACAS
+### Nanopore: Nextclade
/abacas/`
- - `*.abacas.bin`: Bin file that contains contigs that are not used in ordering.
- - `*.abacas.crunch`: Comparison file.
- - `*.abacas.fasta`: Ordered and orientated sequence file.
- - `*.abacas.gaps`: Gap information.
- - `*.abacas.gaps.tab`: Gap information in tab-delimited format.
- - `*.abacas.MULTIFASTA.fa`: A list of ordered and orientated contigs in a multi-fasta format.
- - `*.abacas.tab`: Feature file
- - `*.unused_contigs.out`: Information on contigs that have a mapping information but could not be used in the ordering.
-- `assembly//abacas/nucmer/`: Folder containing the files generated by the NUCmer algorithm used by ABACAS.
+- `/nextclade/`
+ - `*.csv`: Analysis results from Nextlade containing genome clade assignment, mutation calling and sequence quality checks.
-**NB:** The value of `` in the output directory name above is determined by the `--assemblers` parameter (Default: 'spades').
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[ABACAS](https://www.sanger.ac.uk/science/tools/pagit) was developed to rapidly contiguate (align, order, orientate), visualize and design primers to close gaps on shotgun assembled contigs based on a reference sequence.
+[Nextclade](https://github.com/nextstrain/nextclade) performs viral genome clade assignment, mutation calling and sequence quality checks for the consensus sequences generated in this pipeline. Similar to Pangolin, it has been used extensively during the COVID-19 pandemic. A [web application](https://clades.nextstrain.org/) also exists that allows users to upload genome sequences via a web browser.
-### PlasmidID
+### Nanopore: ASCIIGenome
/plasmidid//`
- - `*_final_results.html`: Summary file with reference coverage stats and contigs for visualization.
- - `*_final_results.tab`: Summary file with reference coverage stats and contigs.
- - `images/_.png`: PNG file with the visualization of the alignment between the viral assembly and the reference viral genome.
- - `logs/`: Log files.
+- `/asciigenome//`
+ - `*.pdf`: Individual variant screenshots with annotation tracks in PDF format.
-**NB:** The value of `` in the output directory name above is determined by the `--assemblers` parameter (Default: 'spades').
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[PlasmidID](https://github.com/BU-ISCIII/plasmidID) was used to graphically represent the alignment of the reference genome relative to a given assembly. This helps to visualize the coverage of the reference genome in the assembly. To find more information about the output files refer to the [documentation](https://github.com/BU-ISCIII/plasmidID/wiki/Understanding-the-image:-track-by-track).
+As described in the documentation, [ASCIIGenome](https://asciigenome.readthedocs.io/en/latest/) is a command-line genome browser that can be run from a terminal window and is solely based on ASCII characters. The closest program to ASCIIGenome is probably [samtools tview](http://www.htslib.org/doc/samtools-tview.html) but ASCIIGenome offers much more flexibility, similar to popular GUI viewers like the [IGV](https://software.broadinstitute.org/software/igv/) browser. We are using the batch processing mode of ASCIIGenome in this pipeline to generate individual screenshots for all of the variant sites reported for each sample in the VCF files. This is incredibly useful to be able to quickly QC the variants called by the pipeline without having to tediously load all of the relevant tracks into a conventional genome browser. Where possible, the BAM read alignments, VCF variant file, primer BED file and GFF annotation track will be represented in the screenshot for contextual purposes. The screenshot below shows a SNP called relative to the MN908947.3 SARS-CoV-2 reference genome that overlaps the ORF7a protein and the nCoV-2019_91_LEFT primer from the ARIC v3 protocol.
-### Assembly QUAST
+
/quast/`
- - `report.html`: Results report in HTML format. Also available in various other file formats i.e. `report.pdf`, `report.tex`, `report.tsv` and `report.txt`.
+- `/`
+ - `variants_long_table.csv`: Long format table collating per-sample information for individual variants, functional effect prediction and lineage analysis.
-**NB:** The value of `` in the output directory name above is determined by the `--assemblers` parameter (Default: 'spades').
+**NB:** The value of `` in the output directory name above is determined by the `--artic_minion_caller` parameter (Default: 'nanopolish').
-[QUAST](http://bioinf.spbau.ru/quast) is used to generate a single report with which to evaluate the quality of the _de novo_ assemblies across all of the samples provided to the pipeline. The HTML results can be opened within any browser (we recommend using Google Chrome). Please see the [QUAST output docs](http://quast.sourceforge.net/docs/manual.html#sec3) for more detailed information regarding the output files.
+Create variants long format table collating per-sample information for individual variants ([`BCFTools`](http://samtools.github.io/bcftools/bcftools.html)), functional effect prediction ([`SnpSift`](http://snpeff.sourceforge.net/SnpSift.html)) and lineage analysis ([`Pangolin`](https://github.com/cov-lineages/pangolin)).
-
+The more pertinent variant information is summarised in this table to make it easier for researchers to assess the impact of variants found amongst the sequenced sample(s). An example of the fields included in the table are shown below:
-## Illumina: Workflow reporting and genomes
+```bash
+SAMPLE,CHROM,POS,REF,ALT,FILTER,DP,REF_DP,ALT_DP,AF,GENE,EFFECT,HGVS_C,HGVS_P,HGVS_P_1LETTER,CALLER,LINEAGE
+SAMPLE1_PE,MN908947.3,241,C,T,PASS,489,4,483,0.99,orf1ab,upstream_gene_variant,c.-25C>T,.,.,ivar,B.1
+SAMPLE1_PE,MN908947.3,1875,C,T,PASS,92,62,29,0.32,orf1ab,missense_variant,c.1610C>T,p.Ala537Val,p.A537V,ivar,B.1
+SAMPLE1_PE,MN908947.3,3037,C,T,PASS,213,0,213,1.0,orf1ab,synonymous_variant,c.2772C>T,p.Phe924Phe,p.F924F,ivar,B.1
+SAMPLE1_PE,MN908947.3,11719,G,A,PASS,195,9,186,0.95,orf1ab,synonymous_variant,c.11454G>A,p.Gln3818Gln,p.Q3818Q,ivar,B.1
+```
-### MultiQC
+## Nanopore: Workflow reporting
+
+### Nanopore: MultiQC
/`
- `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser.
- `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline.
- - `summary_variants_metrics_mqc.csv`: file containing a selection of read alignment and variant calling metrics. The same metrics will also be added to the top of the MultiQC report.
- - `summary_assembly_metrics_mqc.csv`: file containing a selection of read alignment and _de novo_ assembly related metrics. The same metrics will also be added to the top of the MultiQC report.
+ - `summary_variants_metrics_mqc.csv`: file containing a selection of read alignmnet and variant calling metrics. The same metrics will also be added to the top of the MultiQC report.
-[MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarizing all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory.
+
-Results generated by MultiQC collate pipeline QC from FastQC, fastp, Cutadapt, Bowtie 2, Kraken 2, samtools, picard CollectMultipleMetrics, BCFTools, SnpEff and QUAST.
+Results generated by MultiQC collate pipeline QC from pycoQC, samtools, mosdepth, BCFTools, SnpEff and QUAST.
-The default [`multiqc config file`](https://github.com/nf-core/viralrecon/blob/master/assets/multiqc_config_illumina.yaml) has been written in a way in which to structure these QC metrics to make them more interpretable in the final report.
+The default [`multiqc config file`](https://github.com/nf-core/viralrecon/blob/master/assets/multiqc_config_nanopore.yaml) has been written in a way in which to structure these QC metrics to make them more interpretable in the final report.
The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see .
@@ -918,4 +982,4 @@ A number of genome-specific files are generated by the pipeline because they are
-[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage.
+[Nextflow](https://www.nextflow.io/docs/latest/tracing.html) provides excellent functionality for generating various reports relevant to the running and execution of the pipeline. This will allow you to troubleshoot errors with the running of the pipeline, and also provide you with other information such as launch commands, run times and resource usage.
\ No newline at end of file
diff --git a/bu_isciii/assets/reports/md/wgmlst_chewbbaca.md b/bu_isciii/assets/reports/md/wgmlst_chewbbaca.md
new file mode 100644
index 00000000..45e68e2d
--- /dev/null
+++ b/bu_isciii/assets/reports/md/wgmlst_chewbbaca.md
@@ -0,0 +1,65 @@
+# Output description for cgMLST/wgMLST with ChewBBACA
+
+## Pipeline description
+
+MLST service performs Multi-Locus Sequence Typing of the samples with the _de novo_ assembly genomes of the samples. It uses [ChewBBACA](https://chewbbaca.readthedocs.io/en/latest/index.html) to generate the schemas (if necessary) and perform the allele calling, and [GrapeTree](https://enterobase.readthedocs.io/en/latest/grapetree/grapetree-about.html) to generate the minimun spanning tree.
+
+### Pipeline overview
+
+- ChewBBACA - v3.3.3 - Schema generation and allele calling
+- GRapeTree - v2.2 - Minimum spanning tree
+
+> [!WARNING]
+> Needs the _de novo_ assembly of the samples to be performed.
+
+## Output directory
+
+- `02-chewbbaca`: Results from ChewBBACA analysis:
+ - `prep_schema`: Schema files prepared for chewbbaca.
+ - `analyze_schema/schema_report.html`: HTML report with the evaluation of the schema used for the analysis.
+ - `allele_calling`:
+ - `cds_coordinates.tsv`: Contains the coordinates (genome unique identifier, contig identifier, start position, stop position, protein identifier attributed by chewBBACA, and coding strand (chewBBACA<=3.2.0 assigns 1 to the forward strand and 0 to the reverse strand and chewBBACA>=3.3.0 assigns 1 and -1 to the forward and reverse strands, respectively)) of the CDSs identified in each genome.
+ - `loci_summary_stats.tsv`: Contains the classification type counts (EXC, INF, PLOT3, PLOT5, LOTSC, NIPH, NIPHEM, ALM, ASM, PAMA, LNF) and the total number of classified CDSs (non-LNF) per locus.
+ - `paralogous_counts.tsv`: Contains the list of paralogous loci and the number of times those loci matched a CDS that was also similar to other loci in the schema.
+ - `results_alleles.tsv`: Contains the allelic profiles determined for the input samples. The first column has the identifiers of the genome assemblies for which the allele call was performed. The remaining columns contain the allele call data for loci present in the schema, with the column headers being the locus identifiers. The INF- prefix in the allelic number indicates that such allele was newly inferred in that genome, and the number following the prefix is the ID attributed to such allele. For the PLOT classification, in the allelic profile output, a locus can be classified as PLOT5 or PLOT3 depending whether the CDS in the genome under analysis matching the schema locus is located in the 5' end or 3' end (respectively) of the contig. All other annotations are identical to what was described above.
+ - `results_statistics.tsv`: Contains the classification type counts (EXC, INF, PLOT3, PLOT5, LOTSC, NIPH, NIPHEM, ALM, ASM, PAMA, LNF), the total number of invalid CDSs, the total number of classified CDSs (non-LNF) and the total number of predicted CDSs per genome. The column headers stand for:
+ - EXC - EXaCt matches (100% DNA identity) with previously identified alleles.
+ - INF - INFerred new alleles that had no exact match in the schema but are highly similar to loci in the schema. The INF- prefix in the allele identifier indicates that such allele was newly inferred in that genome, and the number following the prefix is the allele identifier attributed to such allele. Inferred alleles are added to the FASTA file of the locus they share high similarity with.
+ - LNF - Locus Not Found. No alleles were found for the number of loci in the schema shown. This means that, for those loci, there were no BLAST hits or they were not within the BSR threshold for allele assignment.
+ - PLNF - Probable Locus Not Found. Attributed when a locus is not found during execution modes 1, 2 and 3. Those modes do not perform the complete analysis, that is only performed in mode 4 (default), and the distinct classification indicates that a more thorough analysis might have found a match for the loci that were not found.
+ - PLOT3/PLOT5 - Possible Locus On the Tip of the query genome contigs (see image below). A locus is classified as PLOT when the CDS of the query genome has a BLAST hit with a known larger allele that covers the CDS sequence entirely and the unaligned regions of the larger allele exceed one of the query genome contigs ends (a locus can be classified as PLOT5 or PLOT3 depending on whether the CDS in the genome under analysis matching the schema locus is located in the 5’ end or 3’ end (respectively) of the contig). This could be an artifact caused by genome fragmentation resulting in a shorter CDS prediction by Prodigal. To avoid locus misclassification, loci in such situations are classified as PLOT.
+ - LOTSC - A locus is classified as LOTSC when the contig of the query genome is smaller than the matched allele.
+ - NIPH - Non-Informative Paralogous Hit (see image below). When ≥2 CDSs in the query genome match one locus in the schema with a BSR > 0.6, that locus is classified as NIPH. This suggests that such locus can have paralogous (or orthologous) loci in the query genome and should be removed from the analysis due to the potential uncertainty in allele assignment (for example, due to the presence of multiple copies of the same mobile genetic element (MGE) or as a consequence of gene duplication followed by pseudogenization). A high number of NIPH may also indicate a poorly assembled genome due to a high number of smaller contigs which result in partial CDS predictions. These partial CDSs may contain conserved domains that match multiple loci.
+ - NIPHEM - similar to the NIPH classification, but specifically referring to exact matches. Whenever several CDSs from the same genome match a single or multiple alleles of the same locus with 100% DNA similarity during the first DNA sequence comparison, the NIPHEM tag is attributed.
+ - PAMA - PAralogous MAtch. Attributed to CDSs that are highly similar to more than one locus. This type of classification allows the identification of groups of similar loci in the schema that are classified as paralogous loci and listed in the paralogous_counts.tsv and paralogous_loci.tsv files.
+ - ALM - Alleles 20% Larger than the length Mode of the distribution of the matched loci (CDS length > (locus length mode + locus length mode * 0.2)) (see image below). This determination is based on the currently identified set of alleles for a given locus. It is important to remember that, although infrequently, the mode may change as more alleles for a given locus are called and added to a schema.
+ - ASM - similar to ALM but for Alleles 20% Smaller than the length Mode distribution of the matched loci (CDS length < (locus length mode - locus length mode * 0.2)). As with ALMs it is important to remember that, although infrequently, the mode may change as more alleles for a given locus are called and added to a schema.
+
+
+
+ - `invalid_cds.txt`: Contains the list of alleles predicted by Prodigal that were excluded based on the minimum sequence size value and presence of ambiguous bases.
+ - `logging_info.txt`: Contains summary information about the allele calling process.
+ - `paralogous_loci.tsv`: Contains the sets of paralogous loci identified per genome (genome identifier, identifiers of the paralogous loci and the coordinates of the CDS that is similar to the group of paralogous loci).
+ - `results_contigsInfo.tsv`: Contains the loci coordinates in the genomes analyzed. The first column contains the identifier of the genome used in the allele calling and the other columns (with loci names in the headers) the locus coordinate information or the classification attributed by chewBBACA if it was not an exact match or inferred allele.
+ - `allele_calling_evaluation`:
+ - `allelecall_report.html`: A HTML report, that contains the following components:
+ - A table with the total number of samples, total number of loci, total number of coding sequences (CDSs) extracted from the samples, total number of CDSs classified and totals per classification type.
+ - A tab panel with stacked bar charts for the classification type counts per sample and per locus.
+ - A tab panel with detailed sample and locus statistics.
+ - If a TSV file with annotations is provided to the --annotations parameter, the report will also include a table with the provided annotations. Otherwise, it will display a warning informing that no annotations were provided.
+ - A Heatmap chart representing the loci presence-absence matrix for all samples in the dataset.
+ - A Heatmap chart representing the allelic distance matrix for all samples in the dataset.
+ - A tree drawn with Phylocanvas.gl based on the Neighbor-Joining (NJ) tree computed by FastTree.
+ - `cgMLST_MSA.fasta`: contains the MSA of the core loci. For each locus in the core genome, the alleles found in all samples are translated and aligned with MAFFT. The alignment files are concatenated to generate the full alignment.
+ - `cgMLST_profiles.tsv`: contains the allelic profiles for the set of core loci.
+ - `distance_matrix_symmetric.tsv`: contains the symmetric distance matrix. The distances are computed by determining the number of allelic differences from the set of core loci (shared by 100% of the samples) between each pair of samples.
+ - `masked_profiles.tsv`: contains the masked allelic profiles (results from masking the allelic profiles in the results_alleles.tsv file generated by the AlleleCall module).
+ - `presence_absence.tsv`: Contains the loci presence-absence matrix.
+ - `report_bundle.js`: A JavaScript bundle file necessary to visualize the report.
+
+> [!NOTE]
+> For more information, see the [ChewBBACA's documentation](https://chewbbaca.readthedocs.io/en/latest/index.html) in the `User Guide` secton
+
+- `03-grapetree`:
+ - `tree.svg`: Minimum Spannig Tree plot in SVG (Scalable Vector Graphics) format. Branches longer than = 700 are shown shortenned.
+ - `tree.nwk`: Newick tree from the Minimum Spannig Tree.
diff --git a/bu_isciii/assets/reports/results/assembly.md b/bu_isciii/assets/reports/results/assembly.md
new file mode 100644
index 00000000..2685350a
--- /dev/null
+++ b/bu_isciii/assets/reports/results/assembly.md
@@ -0,0 +1,68 @@
+# Assembly
+
+Here, we describe the results from the Assembly pipeline for de novo genome assembly and annotation.
+
+* **assemblies**: a symbolic link to the raw reads associated with the resolution.
+* **kmerfinder_summary.csv**: a .csv file containing the main results from kmerfinder. For each sample, you should check that both the best hit and the second hit reported by kmerfinder correspond to the species name indicated by the researcher when requesting the service. If the second hit is associated to a different species, check other metrics like %GC or %genome fraction in the MultiQC report, since this might reveal a contamination in that sample.
+ * *sample_name*: sample name.
+ * *07-kmerfinder_best_hit_# Assembly*: RefSeq assembly accession ID.
+ * *07-kmerfinder_best_hit_Accession Number*: accession number of entry ID in fasta file.
+ * *07-kmerfinder_best_hit_Depth*: is the number of matched kmers in the query sequence divided by the total number of Kmers in the template. For read files this estimates the sequencing depth.
+ * *07-kmerfinder_best_hit_Description*: additional descriptions available in fasta file, or in the case of organism databases the identifier lines of fasta files.
+ * *07-kmerfinder_best_hit_Expected*: is the expected score, i.e.the expected total number of matching Kmers between query and template (randomly selected).
+ * *07-kmerfinder_best_hit_Num*: is the sequence number of accession entry in the KmerFinder database.
+ * *07-kmerfinder_best_hit_Query_Coverage*: is the percentage of input query/reads Kmers that match the template.
+ * *07-kmerfinder_best_hit_Score*: is the total number of matching Kmers between the query and the template.
+ * *07-kmerfinder_best_hit_Species*: Species name.
+ * *07-kmerfinder_best_hit_TAXID*: NCBI's TaxID number of the hit.
+ * *07-kmerfinder_best_hit_TAXID Species*: NCBI's species TaxID number of the hit (sometimes bacterial strain or substrain TaxIDs can be given above).
+ * *07-kmerfinder_best_hit_Taxonomy*: complete taxonomy of the hit.
+ * *07-kmerfinder_best_hit_Template_Coverage*: is the template/genome coverage.
+ * *07-kmerfinder_best_hit_Template_length*: is the number of Kmers in the template.
+ * *07-kmerfinder_best_hit_p_value*: is the p-value corresponding to the obtained q_value.
+ * *07-kmerfinder_best_hit_q_value*: is the quantile in a standard Pearson Chi-square test, to test whether the current template is a significant hit.
+ * *07-kmerfinder_best_hit_tot_depth*: depth value based on all query kmers that can be found in the template sequence.
+ * *07-kmerfinder_best_hit_tot_query_Coverage*: is calculated based on the ratio of the score and the number of kmers in the query sequence, where the score includes kmers matched before.
+ * *07-kmerfinder_best_hit_tot_template_Coverage*: is calculated based on ratio of the score and the number of unique kmers in the template sequence, where the score includes kmers matched before.
+ * *07-kmerfinder_second_hit_# Assembly*: RefSeq assembly accession ID.
+ * *07-kmerfinder_second_hit_Accession Number*: accession number of entry ID in fasta file.
+ * *07-kmerfinder_second_hit_Depth*: is the number of matched kmers in the query sequence divided by the total number of Kmers in the template. For read files this estimates the sequencing depth.
+ * *07-kmerfinder_second_hit_Description*: additional descriptions available in fasta file, or in the case of organism databases the identifier lines of fasta files.
+ * *07-kmerfinder_second_hit_Expected*: is the expected score, i.e.the expected total number of matching Kmers between query and template (randomly selected).
+ * *07-kmerfinder_second_hit_Num*: is the sequence number of accession entry in the KmerFinder database.
+ * *07-kmerfinder_second_hit_Query_Coverage*: is the percentage of input query Kmers that match the template.
+ * *07-kmerfinder_second_hit_Score*: is the total number of matching Kmers between the query and the template.
+ * *07-kmerfinder_second_hit_Species*: Species name.
+ * *07-kmerfinder_second_hit_TAXID*: NCBI's TaxID number of the hit.
+ * *07-kmerfinder_second_hit_TAXID Species*: NCBI's species TaxID number of the hit (sometimes bacterial strain or substrain TaxIDs can be given above).
+ * *07-kmerfinder_second_hit_Taxonomy*: complete taxonomy of the hit.
+ * *07-kmerfinder_second_hit_Template_Coverage*: is the template coverage.
+ * *07-kmerfinder_second_hit_Template_length*: is the number of Kmers in the template.
+ * *07-kmerfinder_second_hit_p_value*: is the p-value corresponding to the obtained q_value.
+ * *07-kmerfinder_second_hit_q_value*: is the quantile in a standard Pearson Chi-square test, to test whether the current template is a significant hit.
+ * *07-kmerfinder_second_hit_tot_depth*: depth value based on all query kmers that can be found in the template sequence.
+ * *07-kmerfinder_second_hit_tot_query_Coverage*: is calculated based on the ratio of the score and the number of kmers in the query sequence, where the score includes kmers matched before.
+ * *07-kmerfinder_second_hit_tot_template_Coverage*: is calculated based on ratio of the score and the number of unique kmers in the template sequence, where the score includes kmers matched before.
+ * *Total_hits_07_kmerfinder*: number of total hits.
+* **multiqc_report.html**: an interactive report containing the results from MultiQC (kmerfinder, QUAST, quality control, etc.)
+* **quast_GCF_XXXXXXXXX.X_ASMXXXXXv2_report.html**: an interactive report obtained after the execution of QUAST, providing different metrics about the assembly QC against the reference.
+* **quast_global_report.html**: an interactive report obtained after the execution of QUAST, providing different metrics about the global assembly QC.
+* **summary_assembly_metrics_mqc.csv**: a custom table containing most relevant assembly QC metrics.
+ * *Sample*: sample ID.
+ * *Input reads*: number of input reads for each sample.
+ * *Trimmed reads (fastp)*: number of trimmed reads.
+ * *Contigs*: number of contigs.
+ * *Largest contig*: length of the largest contig.
+ * *N50*: is the contig length such that using longer or equal length contigs produces half of the bases of the assembly. Usually there is no value that produces exactly 50%, so the technical definition is the maximum length x such that using contigs of length at least x accounts for at least 50% of the total assembly length.
+ * *% Genome fraction*: the total number of aligned bases in the reference, divided by the genome size.
+ * *Best hit (Kmerfinder)*: best hit species name.
+ * *Best hit assembly ID (Kmerfinder)*: best hit RefSeq assembly accession ID.
+ * *Best hit query coverage (Kmerfinder)*: best hit query coverage.
+ * *Best hit depth (Kmerfinder)*: best hit depth.
+ * *Second hit (Kmerfinder)*: second hit species name.
+ * *Second hit assembly ID (Kmerfinder)*: second hit RefSeq assembly accession ID.
+ * *Second hit query coverage (Kmerfinder)*: second hit query coverage.
+ * *Second hit depth (Kmerfinder)*: second hit depth.
+
+> [!WARNING]
+> Software's versions used in this analysis can be obtained from the `MultiQC` report.
\ No newline at end of file
diff --git a/bu_isciii/assets/reports/results/exomeeb.md b/bu_isciii/assets/reports/results/exomeeb.md
new file mode 100644
index 00000000..5e9dab93
--- /dev/null
+++ b/bu_isciii/assets/reports/results/exomeeb.md
@@ -0,0 +1,154 @@
+# ExomeEB
+
+This markdown briefly describes the files found in `RESULTS/` folder for ExomeEB service.
+
+## exomiser.html
+
+This file includes information regarding variant annotation, effect prediction and inheritance typing:
+
+
+
+
+
+## picard_hsmetrics.csv
+
+This table includes mapping quality metrics from sarek's pipeline results, with the following columns:
+- SAMPLE
+- MEAN TARGET COVERAGE: The mean coverage of a target region.
+- PCT USABLE BASES ON TARGET: The number of aligned, de-duped, on-bait bases out of the PF bases available (those that pass the vendor's filter).
+- FOLD ENRICHMENT: The fold by which the baited region has been amplified above genomic background.
+- PCT TARGET BASES 10X: The fraction of all target bases achieving 10X or greater coverage.
+- PCT TARGET BASES 20X: The fraction of all target bases achieving 20X or greater coverage.
+- PCT TARGET BASES 30X: The fraction of all target bases achieving 30X or greater coverage.
+- PCT TARGET BASES 40X: The fraction of all target bases achieving 40X or greater coverage.
+- PCT TARGET BASES 50X: The fraction of all target bases achieving 50X or greater coverage.
+
+You may find further documentation for the metrics in this table [here](http://broadinstitute.github.io/picard/picard-metric-definitions.html#HsMetrics).
+
+## variants_annot_highmoderate.tab
+
+This table includes all the variants from VEP and Exomiser annotation with a predicted high or moderate effect.
+
+ID: Variant identifier
+Chrom: Chromosome number
+Pos: Reference position according to hg19
+Ref: Nucleotides found in the reference genome
+Alt: Nucleotides found in the individual's genome
+Filter: Indicates whether it has passed the filter or not
+Sample_GT: Genotype of sample
+Sample_DP: Coverage depth of sample
+Sample_AD: Allelic Depth of Genotype of sample
+Sample_GQ: Quality of Genotype of sample
+Gene: Gene affected by the variant
+Location: Location of the variant in the genome
+Allele: Alternative allele of the variant
+Feature: Genomic element affected by the variant
+Feature_type: Type of genomic element affected by the variant
+Consequence: Functional consequence of the variant
+cDNA_position: Position of the variant in the cDNA sequence
+CDS_position: Position of the variant in the gene's coding sequence
+Protein_position: Position of the variant in the protein encoded by the gene
+Amino_acids: Amino acids affected by the variant
+Codons: Codons affected by the variant
+Existing_variation: Existing genetic variation at this position
+Impact: Fundamental impact of the variant
+Distance: Distance of the variant to the nearest exon
+Strand: DNA strand on which the variant is located
+Flags: Indicators of additional variant features
+Variant_class: Variant class
+Symbol: Gene symbol affected by the variant
+Symbol_source: Source of the variant symbol
+HGNC_ID: Unique identifier in the HGNC database
+Biotype: Gene biotype
+Canonical: Indicates if it is the canonical transcript of the gene
+Mane_sel: Indicates if the transcript is identified by Mane
+Mane_plus: Indicates if the transcript is identified by Mane+
+TSL: Indicates if the transcript is identified by TSL
+Appris: Indicates if the transcript is identified by Appris
+ENSP: Unique identifier of the transcript in Ensembl
+Swissprot: Unique identifier of the transcript in Swissprot
+TREMBL: Unique identifier of the transcript in TREMBL
+Uniparc: Unique identifier of the protein in the Uniparc database
+Uniparc_Isoform: Unique identifier of the protein isoform in Uniparc
+Gene_pheno: Indicates if the gene is associated with a phenotype
+SIFT: SIFT pathogenicity of the variant
+PolyPhen: PolyPhen pathogenicity of the variant
+Exon: Indicates if the variant is located in an exon
+Intron: Indicates if the variant is located in an intron
+Domains: Protein domains affected by the variant
+miRNA: miRNA binding to the region affected by the variant
+HGVSc: HGVS notation for the variant in cDNA sequence
+HGVSp: HGVS notation for the variant in protein sequence
+HGVS_offset: Offset of the variant in cDNA or protein sequence
+AF: Allelic frequency of the variant in the general population
+AFR_AF: Allelic frequency of the variant in the African population
+AMR_AF: Allelic frequency of the variant in the American population
+EAS_AF: Allelic frequency of the variant in the East Asian population
+EUR_AF: Allelic frequency of the variant in the European population
+SAS_AF: Allelic frequency of the variant in the South Asian population
+AA_AF: Allelic frequency of the variant in the African American population
+EA_AF: Allelic frequency of the variant in the European American population
+gnomAD_AF: Allelic frequency of the variant in the gnomAD database
+gnomAD_AFR_AF: Allelic frequency of the variant in the African population in gnomAD
+gnomAD_AMR_AF: Allele frequency of the variant in the American population in gnomAD
+gnomAD_ASJ_AF: Allele frequency of the variant in the Asian and Japanese population in gnomAD
+gnomAD_EAS_AF: Allele frequency of the variant in the East Asian population in gnomAD
+gnomAD_FIN_AF: Allele frequency of the variant in the Finnish population in gnomAD
+gnomAD_NFE_AF: Allele frequency of the variant in the non-Finnish European population in gnomAD
+gnomAD_OTH_AF: Allele frequency of the variant in other populations in gnomAD
+gnomAD_SAS_AF: Allele frequency of the variant in the South Asian population in gnomAD
+MAX_AF: Maximum allele frequency of the variant in all populations
+MAX_AF_POPS: Population with the maximum allele frequency of the variant
+CLIN_SIG: Clinical significance of the variant
+SOMATIC: Indicates if the variant is somatic or germline
+PHENO: Phenotype associated with the variant
+HGNC_ID: Unique identifier of the gene affected by the variant
+PUBMED: Scientific articles describing the variant
+MOTIF_NAME: Transcription factor binding motif affected by the variant
+MOTIF_POS: Position of the transcription factor binding motif affected
+HIGH_INF_POS: High information position in the transcription factor binding motif
+MOTIF_SCORE_CHANGE: Change in the transcription factor binding motif score caused by the variant
+TRANSCRIPTION_FACTORS: Transcription factors that bind to the affected transcription factor binding motif
+HGVSp_snpEff: HGVS notation for the variant in the protein sequence in snpEff
+SIFT_score: SIFT score of the pathogenicity of the variant
+SIFT_pred: SIFT prediction of the pathogenicity of the variant
+Polyphen2_HDIV_score: PolyPhen2_HDIV score of the pathogenicity of the variant
+Polyphen2_HDIV_pred: PolyPhen2_HDIV prediction of the pathogenicity of the variant
+Polyphen2_HVAR_score: PolyPhen2_HVAR score of the pathogenicity of the variant
+Polyphen2_HVAR_pred: PolyPhen2_HVAR prediction of the pathogenicity of the variant
+MutationTaster_score: MutationTaster score of the pathogenicity of the variant
+MutationTaster_pred: MutationTaster prediction of the pathogenicity of the variant
+MutationAssessor_score: MutationAssessor score of the pathogenicity of the variant
+MutationAssessor_pred: MutationAssessor prediction of the pathogenicity of the variant
+FATHMM_score: FATHMM score of the pathogenicity of the variant
+FATHMM_pred: FATHMM prediction of the pathogenicity of the variant
+HGVSp: HGVS notation for the variant in the protein sequence
+HGVS_offset: Displacement of the variant in the cDNA or protein sequence
+PROVEAN_score: PROVEAN score of the pathogenicity of the variant
+PROVEAN_pred: PROVEAN prediction of the pathogenicity of the variant
+VEST4_score: VEST4 score of the pathogenicity of the variant
+MetaSVM_score: MetaSVM score of the pathogenicity of the variant
+MetaSVM_pred: MetaSVM prediction of the pathogenicity of the variant
+MetaLR_score: MetaLR score of the pathogenicity of the variant
+MetaLR_pred: MetaLR prediction of the pathogenicity of the variant
+CADD_raw: Raw CADD score
+CADD_phred: CADD Phred score
+CADD_raw_hg19: Raw CADD score for the hg19 genome version
+CADD_phred_hg19: CADD Phred score for the hg19 genome version
+GERP++_NR: GERP++ score for the non-coding region
+GERP++_RS: GERP++ score for the synonymous region
+phyloP100way_vertebrate: phyloP score for 100 vertebrate species
+phastCons100way_vertebrate: phastCons score for 100 vertebrate species
+clinvar_trait: Clinical trait or condition associated with the variant
+clinvar_id: Unique identifier of the variant in ClinVar
+clinvar_OMIM_id: Unique identifier of the variant in OMIM
+OMIM_id: Unique identifier of the disease in OMIM
+Function_description: Description of the function of the gene affected by the variant
+Disease_description: Description of the disease associated with the variant
+HPO_id: Unique identifier of the phenotype in the HPO database
+HPO_name: Name of the phenotype in the HPO database
+
+## multiqc_report.html
+
+Most of sarek's QC results are visualised in this report and further statistics are available in the report data directory.
+Results generated by MultiQC collect pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see http://multiqc.info.
diff --git a/bu_isciii/assets/reports/results/irma_output.md b/bu_isciii/assets/reports/results/irma_output.md
new file mode 100644
index 00000000..ef4d79f3
--- /dev/null
+++ b/bu_isciii/assets/reports/results/irma_output.md
@@ -0,0 +1,27 @@
+# IRMA-output
+
+This markdown briefly describes the files found in `RESULTS/` folder for IRMA services. As described [here]()
+
+## **`krona_results.html`**
+
+Includes the multiQC html report from MAG, you can find a further description in [MAG](https://github.com/BU-ISCIII/buisciii-tools/blob/main/bu_isciii/assets/reports/md/mag.md).
+
+## Files in `fragment_name/`
+
+Depending on the virus types found in your samples, you will have one folder for each fragment found (e.g. A_H1 for Influenza A H1_N1). In these folders you will have a multi-fasta file for each of the fragments found, which will include all the sequences for that specific fragment found by IRMA in the samples. Here's an example:
+```
+A_H1/A_HA.txt
+>Sample1_HA
+XXXXXXXX-SEQUENCE_OF_FRAGMENT_HA_FOR_SAMPLE1-XXXXXXXX
+>Sample2_HA
+XXXXXXXX-SEQUENCE_OF_FRAGMENT_HA_FOR_SAMPLE2-XXXXXXXX
+>Sample3_HA
+XXXXXXXX-SEQUENCE_OF_FRAGMENT_HA_FOR_SAMPLE3-XXXXXXXX
+```
+
+## **`all_samples_completo.txt`**
+
+This file is a multi-fasta that includes all the assembly sequences generated by IRMA for all the fragments found in all the samples. The structure content of this file is the same as the previously described.
+
+In case of Influenza services (FLU), you will find a file named `flu_type_summary.txt`, which will include a summary of the different types of influenza found in your samples. This is the most relevant information to be included in your report.
+
diff --git a/bu_isciii/assets/reports/results/mag.md b/bu_isciii/assets/reports/results/mag.md
index 1cfd12ca..ce968514 100644
--- a/bu_isciii/assets/reports/results/mag.md
+++ b/bu_isciii/assets/reports/results/mag.md
@@ -1,6 +1,26 @@
## MAG
+
Here we describe the results from the MAG pipeline for multispecies metagenomic analysis.
-* krona_results.html : Final HTML report with the top 5 species most present in all samples.
+### MAG - TAXONIMIC ANALYSIS
+
+* `krona_results.html` : Final HTML report with the top 5 species most present in all samples.
+
+> [!WARNING]
+> Software's versions used in this analysis can be obtained from the `MultiQC` report.
+
+### MAG - COMPLETE ANALYSIS
+
+* `mag_all/krona/${sample_name}.${tool}.report.html`: A Krona interactive visualization report for the each sample based on Kraken2 (or other) taxonomic classification mehtod.
+* `mag_all/quast/${sample_name}.${tool}.report.html`: A Quast report for the assembly quality control of each sample assembled using MEGAHIT, SPAdes or other.
+* `mag_all/multiqc_report.html`: A combined report generated by MultiQC summarizing various quality control results for all samples.
+
+## Taxprofiler
+
+Here we describe the results from the (nf-core/taxprofiler)[https://nf-co.re/taxprofiler/1.1.8] pipeline for multispecies taxonomic classification and profiling of shorgun short- and long-read.
+
+* `taxprofiler/multiqc_report.html`: Final HTML report collecting numerical stats from each module executed in this pipeline.
+* `taxprofiler/krona/database_*.html`: Interactive HTML files generated by Krona, displaying the results of taxonomic classification for supported tools (Kraken2, Centrifuge, Kaiju, and MALT)
-*Warning:* Software's versions used in this analysis can be obtained from the `MultiQC` report.
+> [!WARNING]
+> Software's versions used in this analysis can be obtained from the `MultiQC` report.
\ No newline at end of file
diff --git a/bu_isciii/assets/reports/results/pikavirus.md b/bu_isciii/assets/reports/results/pikavirus.md
index 59f8faab..a5565e91 100644
--- a/bu_isciii/assets/reports/results/pikavirus.md
+++ b/bu_isciii/assets/reports/results/pikavirus.md
@@ -1,4 +1,5 @@
## PikaVirus
+
Here we describe the results from the PikaVirus pipeline for viral presence discovery.
* filtered_all_samples_virus_table.xlsx: Results from PikaVirus.
diff --git a/bu_isciii/assets/reports/results/plasmidid.md b/bu_isciii/assets/reports/results/plasmidid.md
new file mode 100644
index 00000000..047dddc5
--- /dev/null
+++ b/bu_isciii/assets/reports/results/plasmidid.md
@@ -0,0 +1,27 @@
+# PlasmidID
+
+Here we describe the results from the Viralrecon pipeline for viral genome reconstruction.
+
+> [!WARNING]
+> Some of the files listed here may not be in your `RESULTS` folder. It will depend on the analysis you requested.
+
+## Summary report
+
+A summary report consolidating all samples in the analysis is created.
+
+- `NO_GROUP_final_results.html`: report with same info as table below that can be viewed using chrome.
+- `NO_GROUP_final_results.tab`: plasmid info for each sample. Header columns are described here:
+ - id: plasmid unique identifier for each entry.
+ - length: The length of the plasmid sequence.
+ - species description: A description of the species from which the sequence originates.
+ - fraction_covered: The fraction of the sequence that is covered by alignments.
+ - contig_name: The name of the contigs associated with the sequence.
+ - percentage: The percentage of the genome or sequence that is covered.
+ - images: Links or references to related images or visual data.
+
+## Circos images
+
+Circos is used for creating one image for each identified plasmid and a summary image with all the plasmids identified in one figure. A manual for image interpretation can be found [here](https://github.com/BU-ISCIII/plasmidID/wiki/Understanding-the-image:-track-by-track) and a manual about how to select the correct plasmid can be found [here](https://github.com/BU-ISCIII/plasmidID/wiki/How-to-chose-the-right-plasmids).
+
+- `images/SAMPLE_NAME_PLASMID_individual.png`: circos image for individual plasmidID
+- `images/SAMPLE_NAME_summary_image.png`: summary image
\ No newline at end of file
diff --git a/bu_isciii/assets/reports/results/rnaseq_deg.md b/bu_isciii/assets/reports/results/rnaseq_deg.md
new file mode 100644
index 00000000..8c39926f
--- /dev/null
+++ b/bu_isciii/assets/reports/results/rnaseq_deg.md
@@ -0,0 +1,68 @@
+# mRNAseq (DEG): results
+
+Here we describe the results from the mRNAseq pipeline for transcriptomic analysis and differential gene expression.
+
+> [!WARNING]
+> Some of the files listed here may not be in your `RESULTS` folder. It will depend on the analysis you requested.
+
+## Alignment and quantification
+
+
+> [!WARNING]
+> Please note that the files in the RESULTS directory are still pending determination by our administrative team. We are currently discussing which generic/common files will be included in this section to ensure that you receive the most relevant and useful information for your analysis.
+
+## Differential expression analysis with DESEQ2
+
+
+> [!WARNING]
+> Please note that the files in the RESULTS directory are still pending determination by our administrative team. We are currently discussing which generic/common files will be included in this section to ensure that you receive the most relevant and useful information for your analysis.
+
+## Interpretation of Differential Expression Results:
+
+For each comparison conducted, a separate folder has been created, following the naming convention outlined below:
+
+ 1_Treatment1_Control1/
+ 2_Treatment2_Control2/
+ 3_Treatment3_Control3/
+ 4_Treatment1_Treatment2_Treatment3_Control1_Control2_Control3/
+
+Within each folder, you will find the results of the differential expression analysis. When interpreting the results, it's important to note that:
+
+ A positive log2FC indicates that the mRNA is overexpressed in the Treatment group compared to the Control group.
+ Conversely, a negative log2FC suggests that the mRNA is downregulated in the Treatment group compared to the Control group.
+
+Additionally, the file `normalized_expression.xlsx` contains the normalized expression counts matrix, providing further insight into the expression levels of the mRNAs across the experimental conditions.
+
+> [!WARNING]
+> Software's versions used in this analysis can be obtained from the `MultiQC` report.
+
+### Description of output files
+- `Output files
+### Nanopore: pycoQC -- `variants/
+` in the output directory name above is determined by the `--variant_caller` parameter (Default: 'ivar' for '--protocol amplicon' and 'bcftools' for '--protocol metagenomic').
+- `pycoqc/`
+ - `*.html` and `.json` file that includes a run summary and graphical representation of various QC metrics including distribution of read length, distribution of read quality scores, mean read quality per sequence length, output per channel over experiment time and percentage of reads per barcode.
-[BCFTools](http://samtools.github.io/bcftools/bcftools.html) is used in the variant calling and _de novo_ assembly steps of this pipeline to obtain basic statistics from the VCF output. It can also used be used to generate a consensus sequence by integrating variant calls into the reference genome. In this pipeline, we use `samtools mpileup` to create a mask using low coverage positions, and `bedtools maskfasta` to mask the genome sequences based on these intervals. Finally, `bcftools consensus` is used to generate the consensus by projecting the high allele frequency variants onto the masked genome reference sequence.
+[PycoQC](https://github.com/a-slide/pycoQC) compute metrics and generate QC plots using the sequencing summary information generated by basecalling/demultiplexing tools such as Guppy e.g. distribution of read length, read length over time, number of reads per barcode and other general stats.
-### QUAST
+Output files
-**NB:** The value of `
Output files
-- `variants/Output files
-- `variants/
Output files
-- `variants/Output files
-- `variants/Output files
-- `assembly/cutadapt/log/` - - `*.cutadapt.log`: Cutadapt log file generated from stdout. -- `assembly/cutadapt/fastqc/` - - `*_fastqc.html`: FastQC report of the trimmed reads. - - `*_fastqc.zip`: Zip archive containing the FastQC report. +- `
Output files
-- `assembly/spades/Output files
-- `assembly/unicycler/` - - `*.scaffolds.fa.gz`: Unicycler scaffold assembly. - - `*.assembly.gfa.gz`: Unicycler assembly graph in GFA format. -- `assembly/unicycler/bandage/` - - `*.png`: Bandage visualisation for Unicycler assembly graph in PNG format. - - `*.svg`: Bandage visualisation for Unicycler assembly graph in SVG format. +- `Output files
-- `assembly/minia/` - - `*.contigs.fa`: Minia scaffold assembly. - - `*.unitigs.fa`: Minia unitigs fasta file. - - `*.h5`: Minia h5 output file. +- `Output files
-- `assembly/Output files
-- `assembly/Output files
-- `assembly/
Output files
-- `assembly/Output files
-- `multiqc/` +- `multiqc/" + ) else: email_data["email_notes"] = bu_isciii.utils.ask_for_some_text( msg="Write email notes" - ) + ).replace("\n", "
") email_data["user_data"] = self.resolution_info["service_user_id"] email_data["service_id"] = self.service_name.split("_", 5)[0] diff --git a/bu_isciii/clean.py b/bu_isciii/clean.py index 70cb1c1d..f1f14fc6 100644 --- a/bu_isciii/clean.py +++ b/bu_isciii/clean.py @@ -31,6 +31,7 @@ def __init__( option=None, api_user=None, api_password=None, + conf=None, ): # access the api with the resolution name to obtain the data # ask away if no input given @@ -40,15 +41,13 @@ def __init__( self.resolution_id = resolution_id # Obtain info from iskylims api - self.conf = bu_isciii.config_json.ConfigJson().get_configuration("cleanning") - conf_api = bu_isciii.config_json.ConfigJson().get_configuration( - "xtutatis_api_settings" - ) + self.conf = conf.get_configuration("cleanning") + conf_api = conf.get_configuration("xtutatis_api_settings") rest_api = bu_isciii.drylab_api.RestServiceApi( conf_api["server"], conf_api["api_url"], api_user, api_password ) self.resolution_info = rest_api.get_request( - request_info="service-data", safe=False, resolution=self.resolution_id + request_info="service-data", safe=True, resolution=self.resolution_id ) self.service_folder = self.resolution_info["resolutions"][0][ "resolution_full_number" @@ -56,7 +55,9 @@ def __init__( self.services_requested = self.resolution_info["resolutions"][0][ "available_services" ] - self.service_samples = self.resolution_info["samples"] + self.service_samples = [ + sample_id["sample_name"] for sample_id in self.resolution_info["samples"] + ] if ask_path and path is None: stderr.print( @@ -77,7 +78,10 @@ def __init__( sys.exit() else: self.path = bu_isciii.utils.get_service_paths( - "services_and_colaborations", self.resolution_info, "non_archived_path" + conf, + "services_and_colaborations", + self.resolution_info, + "non_archived_path", ) self.full_path = os.path.join(self.path, self.service_folder) @@ -92,14 +96,13 @@ def __init__( self.delete_files = self.get_clean_items(self.services_to_clean, type="files") # self.delete_list = [item for item in self.delete_list if item] self.nocopy = self.get_clean_items(self.services_to_clean, type="no_copy") - self.service_samples = self.resolution_info.get("Samples", None) if option is None: self.option = bu_isciii.utils.prompt_selection( "Options", [ "full_clean", - "rename_nocopy", + "rename", "clean", "revert_renaming", "show_removable", @@ -126,10 +129,16 @@ def get_clean_items(self, services_ids, type="files"): for service in services_ids: try: items = service_conf.get_find_deep(service, type) - if len(clean_items_list) == 0 and len(items) > 0: - clean_items_list = items - elif len(items) > 0: - clean_items_list.append(items) + if items is None: + stderr.print( + "[red]ERROR: Service type %s not found in services json file for service %s." + % (type, service) + ) + sys.exit() + else: + for item in items: + if item not in clean_items_list: + clean_items_list.append(item) except KeyError as e: stderr.print( "[red]ERROR: Service id %s not found in services json file." @@ -310,10 +319,9 @@ def purge_files(self): files_to_delete = [] for sample_info in self.service_samples: for file in self.delete_files: - file_to_delete = file.replace( - "sample_name", sample_info["sample_name"] - ) - files_to_delete.append(file_to_delete) + file_to_delete = file.replace("sample_name", sample_info) + if file_to_delete not in files_to_delete: + files_to_delete.append(file_to_delete) path_content = self.scan_dirs(to_find=files_to_delete) for file in path_content: os.remove(file) @@ -369,7 +377,7 @@ def delete_work(self): else: stderr.print("There is no work folder here") - def delete_rename(self, verbose=True, sacredtexts=["lablog", "logs"], add="_DEL"): + def delete(self, verbose=True, sacredtexts=["lablog", "logs"], add="_DEL"): """ Description: Remove both files and purge folders defined for the service, and rename to tag. @@ -390,10 +398,8 @@ def delete_rename(self, verbose=True, sacredtexts=["lablog", "logs"], add="_DEL" # Purge folders if self.delete_folders != "": self.purge_folders(sacredtexts=sacredtexts, add=add, verbose=verbose) - # Rename to tag. - self.rename(add=add, to_find=self.delete_folders, verbose=verbose) else: - stderr.print("No folders to remove or rename") + stderr.print("No folders to remove") # Purge work self.delete_work() # Delete files @@ -430,8 +436,10 @@ def full_clean(self): Perform and handle the whole cleaning of the service """ - self.delete_rename() + self.delete() self.rename(to_find=self.nocopy, add="_NC", verbose=True) + if self.delete_folders != "": + self.rename(add="_DEL", to_find=self.delete_folders, verbose=True) def handle_clean(self): """ @@ -443,9 +451,11 @@ def handle_clean(self): self.show_nocopy() if self.option == "full_clean": self.full_clean() - if self.option == "rename_nocopy": + if self.option == "rename": self.rename(to_find=self.nocopy, add="_NC", verbose=True) + if self.delete_folders != "": + self.rename(add="_DEL", to_find=self.delete_folders, verbose=True) if self.option == "clean": - self.delete_rename() + self.delete() if self.option == "revert_renaming": self.revert_renaming() diff --git a/bu_isciii/conf/configuration.json b/bu_isciii/conf/configuration.json old mode 100755 new mode 100644 index 7a39ff99..3551c276 --- a/bu_isciii/conf/configuration.json +++ b/bu_isciii/conf/configuration.json @@ -2,7 +2,11 @@ "global": { "data_path": "/data/bi", "archived_path": "/archived/bi", - "yaml_conf_path": "~/buisciii_config.yml" + "yaml_conf_path": "~/buisciii_config.yml", + "permissions": { + "directory_chmod": "2775", + "file_chmod": "664" + } }, "sftp_copy": { "protocol": "rsync", @@ -16,7 +20,8 @@ "'.nextflow*'", "'*_DEL'", "'*.R'", - "'*.py'" + "'*.py'", + "'*.sbatch'" ] }, "xtutatis_api_settings": { @@ -35,7 +40,6 @@ "delivery_template_path_file": "templates/jinja_template_delivery.j2", "html_template_path_file": "templates/html_service_template.html", "path_to_css": "assets/css", - "wkhtmltopdf_path": "/data/bi/pipelines/miniconda3/envs/buisciii-tools/bin/wkhtmltopdf", "email_host": "mx2.isciii.es", "email_port": "587", "email_host_user": "bioinformatica@isciii.es", @@ -54,7 +58,7 @@ ], "scratch_path": "/scratch/bi/", "srun_settings": { - "--partition": "middle_idx", + "--partition": "middle_obx,middle_idx", "--time": "24:00:00", "--chdir": "/scratch/bi/" } diff --git a/bu_isciii/conf/configuration_dev.json b/bu_isciii/conf/configuration_dev.json new file mode 100755 index 00000000..68e948cb --- /dev/null +++ b/bu_isciii/conf/configuration_dev.json @@ -0,0 +1,66 @@ +{ + "global": { + "data_path": "tests/data/bi", + "archived_path": "tests/archived/bi", + "yaml_conf_path": "~/buisciii_config.yml" + }, + "sftp_copy": { + "protocol": "rsync", + "options": ["-rlpv", "--update", "-L", "--inplace"], + "exclusions": [ + "'*_NC'", + "'*lablog*'", + "'work'", + "'00-reads'", + "'*.sh'", + "'.nextflow*'", + "'*_DEL'", + "'*.R'", + "'*.py'", + "'*.sbatch'" + ] + }, + "xtutatis_api_settings": { + "api_url": "/drylab/api/", + "server": "http://iskylims.isciiides.es" + }, + "api_settings": { + "server": "http://iskylims.isciiides.es", + "api_url": "/drylab/api/" + }, + "bioinfo_doc": { + "bioinfodoc_path": "tests/bioinfo_doc/", + "services_path": "services", + "service_folder": ["service_info", "result"], + "service_info_template_path_file": "templates/jinja_template_service_info.j2", + "delivery_template_path_file": "templates/jinja_template_delivery.j2", + "html_template_path_file": "templates/html_service_template.html", + "path_to_css": "assets/css", + "email_host": "mx2.isciii.es", + "email_port": "587", + "email_host_user": "bioinformatica@isciii.es", + "email_use_tls": "True" + }, + "new_service": { + "fastq_repo": "tests/fastq_repo" + }, + "scratch_copy": { + "protocol": "rsync", + "options": ["-rlpv"], + "exclusions": [ + "'*_NC'", + "'service_info.txt'", + "'work'" + ], + "scratch_path": "tests/scratch/bi/", + "srun_settings": { + "--partition": "middle_obx,middle_idx", + "--time": "24:00:00", + "--chdir": "tests/scratch/bi/" + } + }, + "archive": { + "protocol": "rsync", + "options": ["-rv"] + } +} diff --git a/bu_isciii/config_json.py b/bu_isciii/config_json.py index cafad194..5035ec93 100644 --- a/bu_isciii/config_json.py +++ b/bu_isciii/config_json.py @@ -24,7 +24,6 @@ def get_configuration(self, topic): def get_find(self, topic, found): """ - Owner: Pablo Description: Obtain from topic any forward items from json data """ diff --git a/bu_isciii/copy_sftp.py b/bu_isciii/copy_sftp.py index 13cd40f7..9474b59b 100644 --- a/bu_isciii/copy_sftp.py +++ b/bu_isciii/copy_sftp.py @@ -33,6 +33,7 @@ def __init__( sftp_folder=None, api_user=None, api_password=None, + conf=None, ): if resolution_id is None: self.resolution_id = bu_isciii.utils.prompt_resolution_id() @@ -40,10 +41,8 @@ def __init__( self.resolution_id = resolution_id # Load conf - self.conf = bu_isciii.config_json.ConfigJson().get_configuration("sftp_copy") - conf_api = bu_isciii.config_json.ConfigJson().get_configuration( - "xtutatis_api_settings" - ) + self.conf = conf.get_configuration("sftp_copy") + conf_api = conf.get_configuration("xtutatis_api_settings") # Obtain info from iskylims api rest_api = bu_isciii.drylab_api.RestServiceApi( @@ -51,10 +50,12 @@ def __init__( ) self.resolution_info = rest_api.get_request( - request_info="service-data", safe=False, resolution=self.resolution_id + request_info="service-data", safe=True, resolution=self.resolution_id ) if sftp_folder is None: - self.sftp_folder = bu_isciii.utils.get_sftp_folder(self.resolution_info)[0] + self.sftp_folder = bu_isciii.utils.get_sftp_folder( + conf, self.resolution_info + )[0] else: self.sftp_folder = sftp_folder @@ -64,9 +65,7 @@ def __init__( self.services_requested = self.resolution_info["resolutions"][0][ "available_services" ] - self.sftp_options = bu_isciii.config_json.ConfigJson().get_find( - "sftp_copy", "options" - ) + self.sftp_options = conf.get_find("sftp_copy", "options") self.services_to_copy = bu_isciii.utils.get_service_ids(self.services_requested) self.last_folders = self.get_last_folders( @@ -89,7 +88,10 @@ def __init__( sys.exit() else: self.path = bu_isciii.utils.get_service_paths( - "services_and_colaborations", self.resolution_info, "non_archived_path" + conf, + "services_and_colaborations", + self.resolution_info, + "non_archived_path", ) self.full_path = os.path.join(self.path, self.service_folder) @@ -110,8 +112,9 @@ def get_last_folders(self, services_ids, type="last_folder"): last_folders_list = [] for service in services_ids: try: - items = service_conf.get_find_deep(service, type) - last_folders_list.append(items) + item = service_conf.get_find_deep(service, type) + if item not in last_folders_list: + last_folders_list.append(item) except KeyError as e: stderr.print( "[red]ERROR: Service id %s not found in services json file." diff --git a/bu_isciii/new_service.py b/bu_isciii/new_service.py index c9eca73d..0bcf6556 100755 --- a/bu_isciii/new_service.py +++ b/bu_isciii/new_service.py @@ -34,6 +34,7 @@ def __init__( ask_path=False, api_user=None, api_password=None, + conf=None, ): if resolution_id is None: self.resolution_id = bu_isciii.utils.prompt_resolution_id() @@ -46,16 +47,14 @@ def __init__( self.no_create_folder = no_create_folder # Load conf - self.conf = bu_isciii.config_json.ConfigJson().get_configuration("new_service") - conf_api = bu_isciii.config_json.ConfigJson().get_configuration( - "xtutatis_api_settings" - ) + self.conf = conf.get_configuration("new_service") + conf_api = conf.get_configuration("xtutatis_api_settings") # Obtain info from iskylims api self.rest_api = bu_isciii.drylab_api.RestServiceApi( conf_api["server"], conf_api["api_url"], api_user, api_password ) self.resolution_info = self.rest_api.get_request( - request_info="service-data", safe=False, resolution=self.resolution_id + request_info="service-data", safe=True, resolution=self.resolution_id ) self.service_folder = self.resolution_info["resolutions"][0][ "resolution_full_number" @@ -82,7 +81,10 @@ def __init__( sys.exit() else: self.path = bu_isciii.utils.get_service_paths( - "services_and_colaborations", self.resolution_info, "non_archived_path" + conf, + "services_and_colaborations", + self.resolution_info, + "non_archived_path", ) self.full_path = os.path.join(self.path, self.service_folder) @@ -123,13 +125,13 @@ def copy_template(self): ) services_ids = bu_isciii.utils.get_service_ids(self.services_requested) services_json = bu_isciii.service_json.ServiceJson() - if len(services_ids) == 1: + for service_id in services_ids: try: - service_template = services_json.get_find(services_ids[0], "template") + service_template = services_json.get_find(service_id, "template") except KeyError as e: stderr.print( "[red]ERROR: Service id %s not found in services json file." - % services_ids[0] + % service_id ) stderr.print("traceback error %s" % e) sys.exit() @@ -151,13 +153,6 @@ def copy_template(self): stderr.print("[red]ERROR: Copying template failed.") stderr.print("traceback error %s" % e) sys.exit() - else: - stderr.print( - "[red] ERROR: I'm not already prepared for handling more than one error at the same time, sorry!" - "Please re-run and select one of the service ids." - ) - sys.exit(1) - return False return True def create_samples_id(self): @@ -212,11 +207,9 @@ def create_symbolic_links(self): ) except OSError as e: stderr.print( - "[red]ERROR: Symbolic links creation failed for sample %s." - % sample["sampleName"] + "[red]ERROR: Symbolic links creation failed for file %s." % file ) stderr.print("Traceback: %s" % e) - sys.exit() def samples_json(self): json_samples = json.dumps(self.service_samples, indent=4) @@ -235,16 +228,7 @@ def create_new_service(self): self.create_samples_id() self.create_symbolic_links() self.samples_json() - self.rest_api.put_request( - "update-state", "resolution", self.resolution_id, "state", "in_progress" - ) - else: - stderr.print( - "[yellow]WARN: No samples recorded in service: " + self.resolution_id - ) - if bu_isciii.utils.prompt_yn_question("Do you want to proceed?: "): - self.create_folder() - self.copy_template() + if self.resolution_info["service_state"] != "in_progress": self.rest_api.put_request( "update-state", "resolution", @@ -252,6 +236,22 @@ def create_new_service(self): "state", "in_progress", ) + + else: + stderr.print( + "[yellow]WARN: No samples recorded in service: " + self.resolution_id + ) + if bu_isciii.utils.prompt_yn_question("Do you want to proceed?: "): + self.create_folder() + self.copy_template() + if self.resolution_info["service_state"] != "in_progress": + self.rest_api.put_request( + "update-state", + "resolution", + self.resolution_id, + "state", + "in_progress", + ) else: stderr.print("Directory not created. Bye!") sys.exit(1) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index d31ca0cd..529493ab 100755 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -36,6 +36,7 @@ def __init__( ask_path=False, api_user=None, api_password=None, + conf=None, ): if resolution_id is None: self.resolution_id = bu_isciii.utils.prompt_resolution_id() @@ -55,17 +56,15 @@ def __init__( else: self.direction = direction # Load conf - conf_api = bu_isciii.config_json.ConfigJson().get_configuration( - "xtutatis_api_settings" - ) + conf_api = conf.get_configuration("xtutatis_api_settings") # Obtain info from iskylims api rest_api = bu_isciii.drylab_api.RestServiceApi( conf_api["server"], conf_api["api_url"], api_user, api_password ) - self.conf = bu_isciii.config_json.ConfigJson().get_configuration("scratch_copy") + self.conf = conf.get_configuration("scratch_copy") self.resolution_info = rest_api.get_request( - request_info="service-data", safe=False, resolution=self.resolution_id + request_info="service-data", safe=True, resolution=self.resolution_id ) self.service_folder = self.resolution_info["resolutions"][0][ "resolution_full_number" @@ -91,7 +90,10 @@ def __init__( sys.exit() else: self.path = bu_isciii.utils.get_service_paths( - "services_and_colaborations", self.resolution_info, "non_archived_path" + conf, + "services_and_colaborations", + self.resolution_info, + "non_archived_path", ) self.full_path = os.path.join(self.path, self.service_folder) @@ -185,6 +187,16 @@ def revert_copy_scratch(self): sync_source_contents=False, ) self.srun_command(self.srun_settings, rsync_command) + + # After successful rsync, apply correct permissions + conf = bu_isciii.config_json.ConfigJson() + permissions_config = conf.get_configuration("global").get( + "permissions" + ) + bu_isciii.utils.remake_permissions( + self.full_path, permissions_config + ) + stderr.print( "[green]Successfully copied the directory to %s" % dest_folder, @@ -199,7 +211,7 @@ def revert_copy_scratch(self): except Exception as e: stderr.print(e) stderr.print( - "[red]ERROR: Copy of the directory %s failed" + "[red]ERROR: Copy of directory %s failed" % self.scratch_tmp_path, highlight=False, ) @@ -239,17 +251,17 @@ def remove_scratch(self): if self.service_folder in scratch_folder: shutil.rmtree(scratch_folder) stderr.print( - "[green]Successfully removed the directory %s" % scratch_folder, + "[green]Successfully removed directory %s" % scratch_folder, highlight=False, ) else: log.error( - f"Directory path not the same as service resolution. Skip folder copy '{scratch_folder}'" + f"Directory path is not the same as service resolution. Skip folder copy '{scratch_folder}'" ) stderr.print( "[red]ERROR: Directory " + scratch_folder - + " not the same as " + + " is not the same as " + self.scratch_tmp_path, highlight=False, ) diff --git a/bu_isciii/service_json.py b/bu_isciii/service_json.py index d9f2a280..bbe8a111 100644 --- a/bu_isciii/service_json.py +++ b/bu_isciii/service_json.py @@ -38,7 +38,6 @@ def get_service_configuration(self, service): def get_find(self, service, found): """ - Owner: Pablo Description: Obtain from service any forward items from json data """ diff --git a/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/01-preproQC/lablog b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/01-preproQC/lablog index 66e8db87..0062b7b6 100644 --- a/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/01-preproQC/lablog +++ b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/01-preproQC/lablog @@ -1,7 +1,7 @@ -#module load FastQC +#module load singularity mkdir logs scratch_dir=$(echo $PWD | sed "s/\/data\/bi\/scratch_tmp/\/scratch/g") -cat ../samples_id.txt | while read in; do echo "mkdir $in; srun --partition short_idx --cpus-per-task 8 --time 01:00:00 --chdir $scratch_dir --output logs/FASTQC.${in}.%j.log fastqc -o $in --nogroup -t 8 -k 8 ../00-reads/"$in"_R1.fastq.gz ../00-reads/"$in"_R2.fastq.gz &"; done > _01_rawfastqc.sh +cat ../samples_id.txt | while read in; do echo "mkdir $in; srun --partition short_idx --cpus-per-task 8 --time 01:00:00 --chdir $scratch_dir --output logs/FASTQC.${in}.%j.log singularity exec -B ${scratch_dir}/../../../ -B /srv/fastq_repo/ /data/bi/pipelines/singularity-images/fastqc:0.11.9--hdfd78af_1 fastqc -o ${scratch_dir}/$in --nogroup -t 8 -k 8 ${scratch_dir}/../00-reads/"$in"_R1.fastq.gz ${scratch_dir}/../00-reads/"$in"_R2.fastq.gz &"; done > _01_rawfastqc.sh diff --git a/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/02-preprocessing/lablog b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/02-preprocessing/lablog index 46ab047b..4d355c58 100644 --- a/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/02-preprocessing/lablog +++ b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/02-preprocessing/lablog @@ -1,4 +1,4 @@ -# module load fastp +# module load singularity mkdir logs scratch_dir=$(echo $(pwd) | sed 's@/data/bi/scratch_tmp/@/scratch/@g') -cat ../samples_id.txt | xargs -I @@ echo "mkdir @@; srun --chdir ${scratch_dir} --mem 10G --time 1:00:00 --job-name FP.@@ --output logs/FP.@@.%j.log --partition short_idx --cpus-per-task 5 fastp --in1 ../00-reads/@@_R1.fastq.gz --in2 ../00-reads/@@_R2.fastq.gz --thread 5 --cut_front --cut_tail --cut_mean_quality 15 --qualified_quality_phred 15 --trim_poly_x --length_required 50 --detect_adapter_for_pe --json @@/@@_fastp.json --html @@/@@_fastp.html --out1 @@/@@_R1_filtered.fastq.gz --out2 @@/@@_R2_filtered.fastq.gz --unpaired1 @@/@@_R1_unpaired.fastq.gz --unpaired2 @@/@@_R2_unpaired.fastq.gz &" > _01_fastp.sh +cat ../samples_id.txt | xargs -I @@ echo "mkdir @@; srun --chdir ${scratch_dir} --mem 10G --time 1:00:00 --job-name FP.@@ --output logs/FP.@@.%j.log --partition short_idx --cpus-per-task 5 singularity exec -B ${scratch_dir}/../../../ -B /srv/fastq_repo/ /data/bi/pipelines/singularity-images/fastp:0.20.0--hdbcaa40_0 fastp --in1 ${scratch_dir}/../00-reads/@@_R1.fastq.gz --in2 ${scratch_dir}/../00-reads/@@_R2.fastq.gz --thread 5 --cut_front --cut_tail --cut_mean_quality 15 --qualified_quality_phred 15 --trim_poly_x --length_required 50 --detect_adapter_for_pe --json ${scratch_dir}/@@/@@_fastp.json --html ${scratch_dir}/@@/@@_fastp.html --out1 ${scratch_dir}/@@/@@_R1_filtered.fastq.gz --out2 ${scratch_dir}/@@/@@_R2_filtered.fastq.gz --unpaired1 ${scratch_dir}/@@/@@_R1_unpaired.fastq.gz --unpaired2 ${scratch_dir}/@@/@@_R2_unpaired.fastq.gz &" > _01_fastp.sh diff --git a/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/03-procQC/lablog b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/03-procQC/lablog index f0636764..c49d7076 100644 --- a/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/03-procQC/lablog +++ b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/03-procQC/lablog @@ -1,7 +1,7 @@ -#module load FastQC +#module load singularity mkdir logs scratch_dir=$(echo $PWD | sed "s/\/data\/bi\/scratch_tmp/\/scratch/g") -cat ../samples_id.txt | while read in; do echo "mkdir $in; srun --partition short_idx --chdir $scratch_dir --output logs/FASTQC.${in}.%j.log fastqc -o $in --nogroup -t 8 -k 8 ../02-preprocessing/${in}/${in}_R1_filtered.fastq.gz ../02-preprocessing/${in}/${in}_R2_filtered.fastq.gz &"; done > _01_rawfastqc.sh +cat ../samples_id.txt | while read in; do echo "mkdir $in; srun --partition short_idx --chdir $scratch_dir --output logs/FASTQC.${in}.%j.log singularity exec -B ${scratch_dir}/../../../ /data/bi/pipelines/singularity-images/fastqc:0.11.9--hdfd78af_1 fastqc -o ${scratch_dir}/$in --nogroup -t 8 -k 8 ${scratch_dir}/../02-preprocessing/${in}/${in}_R1_filtered.fastq.gz ${scratch_dir}/../02-preprocessing/${in}/${in}_R2_filtered.fastq.gz &"; done > _01_rawfastqc.sh diff --git a/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/04-irma/create_irma_vcf.py b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/04-irma/create_irma_vcf.py new file mode 100644 index 00000000..1cd41672 --- /dev/null +++ b/bu_isciii/templates/IRMA/ANALYSIS/ANALYSIS01_FLU_IRMA/04-irma/create_irma_vcf.py @@ -0,0 +1,1020 @@ +# imports +from Bio import SeqIO +import statistics +import argparse +import sys + + +def parse_args(args=None): + Description = "Convert alignment between IRMA consensus and reference fasta to VCF file using IRMA stats" + Epilog = """Example usage: python create_irma_vcf.py -a