Merge pull request #240 from NationalGenomicsInfrastructure/monthly

Monthly merge April 2024

env_vars/site_upps_env_all.yml

@@ -59,3 +59,7 @@ archive_upload_port: 10480
 # arteria archive verify
 archive_verify_port: 10490
 archive_verify_path_to_verify_root: "{{ static_ngi_pipeline_nobackup }}/arteria/pdc_archive_dump/{{ deployment_environment }}"
+
+# snpseq-metadata-service
+snpseq_metadata_service_port: 10555
+snpseq_metadata_runfolder_path: "{{ static_runfolder_path }}"

host_vars/deploy/main.yml

@@ -47,7 +47,12 @@ ngi_pipeline_conf: "{{ root_path }}/conf/"
 ngi_resources: "{{ root_path }}/resources/"
 
 sarek_tag: "2.7"
-sarek_dest: "{{ sw_path }}/sarek"
+sarek_dest: "{{ sw_path }}/sarek/{{ sarek_tag | replace('.', '_') }}"
+
+rnaseq_tag: "3.3"
+rnaseq_dest: "{{ sw_path }}/rnaseq/{{ rnaseq_tag | replace('.', '_') }}"
+
+demultiplex_tag: "1.4.1"
 
 # File with tools/software version in the deployed env
 deployed_tool_versions: "{{ ngi_resources }}/deployed_tools.{{ site }}.version"

install.yml

@@ -93,6 +93,7 @@
     - { role: arteria-checkqc-ws, tags: arteria-checkqc }
     - { role: arteria-delivery-ws, tags: arteria-delivery }
     - { role: arteria-sequencing-report-ws, tags: arteria-sequencing-report }
+    - { role: snpseq-metadata-service, tags: snpseq-metadata-service }
     - { role: standalone_scripts, tags: standalone_scripts }
     - { role: delivery, tags: delivery }
     - { role: misc-tools, tags: misc-tools }

roles/arteria-checkqc-ws/defaults/main.yml

@@ -2,4 +2,4 @@ arteria_service_name: arteria-checkqc-ws
 arteria_checkqc_service_log: "{{ arteria_service_log_dir }}/{{ arteria_service_name }}.log"
 
 checkqc_repo: https://github.com/Molmed/checkQC.git
-checkqc_version: v4.0.0
+checkqc_version: v4.0.2

roles/arteria-sequencing-report-ws/defaults/main.yml

@@ -1,10 +1,12 @@
 ---
 
 seqreport_service_repo: https://github.com/arteria-project/sequencing-report-service.git
-seqreport_service_version: v1.4.2
+seqreport_service_version: v1.5.0
 
 arteria_service_name: arteria-sequencing-report-ws
 arteria_sequencing_report_wrapper: "{{ arteria_service_config_root }}/arteria_sequencing_report_wrapper.sh"
+arteria_service_pipeline_config_dir: "{{ arteria_service_config_root }}/pipeline_configs"
+arteria_service_nextflow_config_dir: "{{ arteria_service_config_root }}/nextflow_configs"
 
 seqreports_nf_profile: singularity,uppmax
 fastq_ref_genomes_path: /sw/data/fastq_screen_data/20220330/miarka/FastQ_Screen_Genomes

roles/arteria-sequencing-report-ws/files/nextflow_configs/demultiplex.config

@@ -0,0 +1,50 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Config file for defining DSL2 per module options and publishing paths
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Available keys to override module options:
+        ext.args   = Additional arguments appended to command in module.
+        ext.args2  = Second set of arguments appended to command in module (multi-tool modules).
+        ext.args3  = Third set of arguments appended to command in module (multi-tool modules).
+        ext.prefix = File name prefix for output files.
+----------------------------------------------------------------------------------------
+*/
+
+process {
+
+    withName: BCL2FASTQ {
+        ext.args = {[
+            params.ext_args ? "${params.ext_args}" : "",
+        ].join(" ").trim()}
+        publishDir = [
+            [
+                path: { "${params.outdir}/Unaligned/" },
+                pattern: "**_S[1-9]*_*.fastq.gz",
+                mode: "link",
+            ],
+            [
+                path: { "${params.outdir}/Unaligned/" },
+                pattern: "Undetermined_S0_*.fastq.gz",
+                mode: "link",
+            ],
+            [
+                // Gather and write Reports and Stats
+                path: { "${params.outdir}/Unaligned/" },
+                mode: "link",
+                pattern: "Stats",
+                saveAs: {filename -> filename.split("/")[-1] }
+            ],
+            [
+                // Gather and write InterOp files
+                path: { "${params.outdir}/InterOp" },
+                mode: "link",
+                pattern: "**.bin",
+            ],
+        ]
+    }
+
+    withName: MD5SUM {
+        publishDir = []
+    }
+}
+

roles/arteria-sequencing-report-ws/tasks/install.yml

@@ -39,7 +39,7 @@
   args:
     chdir: "{{ tempfolder.path }}"
   environment:
-    NXF_SINGULARITY_CACHEDIR: "{{ ngi_containers }}/seqreports"
+    NXF_SINGULARITY_CACHEDIR: "{{ ngi_containers }}/biocontainers"
     NXF_HOME: "{{ nextflow_dest }}/workfiles"
     NXF_JAVA_HOME: "{{ nextflow_env.NXF_JAVA_HOME }}"
     NXF_TEMP: "{{ tempfolder.path }}"

roles/arteria-sequencing-report-ws/tasks/install_config.yml

@@ -20,6 +20,35 @@
     src: checkqc.config
     dest: "{{ arteria_service_config_root }}/checkqc.config"
 
+- name: ensure {{ arteria_service_pipeline_config_dir }} exists
+  file:
+    state: directory
+    path: "{{ arteria_service_pipeline_config_dir }}"
+
+- name: deploying {{ arteria_service_name }} configs for pipelines
+  template:
+    src: "{{ item }}"
+    dest: "{{ arteria_service_pipeline_config_dir }}/{{ item | basename | splitext | first }}"
+  with_fileglob:
+    - "../templates/pipeline_configs/*.j2"
+
+- name: deploy {{ arteria_service_name }} pipeline config schema
+  copy:
+    src: "{{ arteria_service_src_path }}/config/pipeline_config/schema.json"
+    dest: "{{ arteria_service_pipeline_config_dir }}/schema.json"
+
+- name: ensure {{ arteria_service_nextflow_config_dir }} exists
+  file:
+    state: directory
+    path: "{{ arteria_service_nextflow_config_dir }}"
+
+- name: deploying {{ arteria_service_name }} nextflow configs for pipelines
+  template:
+    src: "{{ item }}"
+    dest: "{{ arteria_service_nextflow_config_dir }}/{{ item | basename }}"
+  with_fileglob:
+    - "../files/nextflow_configs/*.config"
+
 - name: Deploy arteria-delivery wrapper
   template:
     src: arteria_sequencing_report_wrapper.sh.j2

roles/arteria-sequencing-report-ws/templates/pipeline_configs/demultiplex.yml.j2

@@ -0,0 +1,26 @@
+---
+
+main_workflow_path: "{{ sw_path }}/demultiplex/{{ demultiplex_tag | replace('.', '_') }}/main.nf"
+# Note that in the following sections it is possible to do variable subsitution
+# on the following variables:
+# - {runfolder_name}
+# - {runfolder_path}
+# - {current_year}
+# - {input_samplesheet_path}
+nextflow_parameters:
+  config: "{{ arteria_service_nextflow_config_dir }}/demultiplex.config"
+  profile: "singularity,uppmax"
+environment:
+  NXF_TEMP: {{nextflow_env.NXF_TEMP}}
+  NXF_WORK: "{runfolder_path}/.nextflow"
+  NXF_SINGULARITY_CACHEDIR: {{ ngi_containers }}/biocontainers
+  NXF_ANSI_LOG: "false"
+pipeline_parameters:
+  input: "{input_samplesheet_path}"
+  outdir: "{runfolder_path}"
+  project: {{uppmax_project}}
+  demultiplexer: "bcl2fastq"
+  skip_tools: "fastp,falco,multiqc"
+input_samplesheet_content: |
+  id,samplesheet,lane,flowcell
+  {runfolder_name},{runfolder_path}/SampleSheet.csv,,{runfolder_path}

roles/arteria-sequencing-report-ws/templates/pipeline_configs/seqreports.yml.j2

@@ -0,0 +1,25 @@
+---
+
+main_workflow_path: "{{arteria_service_src_path}}/seqreports/main.nf"
+# Note that in the following sections it is possible to do variable
+# substitution on the following variables:
+# - {runfolder_name}
+# - {runfolder_path}
+# - {current_year}
+# - {input_samplesheet_path}
+nextflow_parameters:
+  config: "{{arteria_service_src_path}}/seqreports/nextflow.config"
+  profile: {{seqreports_nf_profile}}
+environment:
+  NXF_TEMP: {{nextflow_env.NXF_TEMP}}
+  NXF_WORK: "{runfolder_path}/.nextflow"
+  NXF_SINGULARITY_CACHEDIR: "{{ ngi_containers }}/biocontainers"
+  NXF_ANSI_LOG: "false"
+pipeline_parameters:
+  run_folder: "{runfolder_path}"
+  project: {{uppmax_project}}
+  config_dir: "{{arteria_service_src_path}}/seqreports/config/tool_config/"
+  fastqscreen_databases: {{fastq_ref_genomes_path}}
+  checkqc_config: "{{arteria_service_config_root}}/checkqc.config"
+  result_dir: "{runfolder_path}/seqreports"
+  script_dir: "{{arteria_service_src_path}}/seqreports/bin/"

roles/arteria-sequencing-report-ws/templates/seqreport_app.config.j2

@@ -3,34 +3,10 @@
 
 port: {{ seqreport_service_port }}
 db_connection_string: sqlite:///{{ db_path }}/sequencing_reports.db
-reports_dir: ${DEFAULT:runfolder_path}/seqreports/
+reports_dir: "{runfolder_path}/seqreports/"
 monitored_directories:
     - {{ static_runfolder_path }}
 alembic_scripts: {{ arteria_service_src_path }}/alembic/
 alembic_log_config_path: {{ arteria_service_config_root}}/logger.config
-process_queue_check_interval: 5 # in seconds
 nextflow_log_dirs: {{ arteria_service_log_dir }}/nxf_logs
-
-nextflow_config:
-    main_workflow_path: {{ arteria_service_src_path }}/seqreports/main.nf
-    nf_config: {{ arteria_service_src_path }}/seqreports/nextflow.config
-    nf_profile: {{ seqreports_nf_profile }}
-    environment:
-        NXF_TEMP: {{ nextflow_env.NXF_TEMP }}
-        NXF_WORK: ${DEFAULT:runfolder_path}/seqreports/nextflow
-        NXF_SINGULARITY_CACHEDIR: "{{ ngi_containers }}/seqreports"
-        NXF_ANSI_LOG: false
-    # Note that in the parameters section it is possible to do variable
-    # subsitution on the following variables.
-    # loading the config.
-    # - ${DEFAULT:runfolder_name}
-    # - ${DEFAULT:runfolder_path}
-    # - ${DEFAULT:current_year}
-    parameters:
-        run_folder: ${DEFAULT:runfolder_path}
-        project: {{ uppmax_project }}
-        config_dir: {{ arteria_service_src_path }}/seqreports/config/tool_config/
-        fastqscreen_databases: {{ fastq_ref_genomes_path }}
-        checkqc_config: {{ arteria_service_config_root }}/checkqc.config
-        result_dir: ${DEFAULT:runfolder_path}/seqreports
-        script_dir: {{ arteria_service_src_path }}/seqreports/bin/
+pipeline_config_dir: {{ arteria_service_pipeline_config_dir }}

roles/nf-core/defaults/main.yml

@@ -1,6 +1,6 @@
 nf_core_env_name: nf-core-env
 nf_core_env: "{{ sw_path }}/anaconda/envs/{{ nf_core_env_name }}"
-nf_core_version: 2.7.2
+nf_core_version: 2.13.1
 nf_core_container_repo: docker://nfcore
 nf_core_vars:
   NFCORE_NO_VERSION_CHECK: 1
@@ -15,7 +15,7 @@ nf_core_vars:
 biocontainers_dirname: biocontainers
 pipelines:
   - name: rnaseq
-    release: 3.3
+    release: "{{ rnaseq_tag }}"
     container_dir: "{{ biocontainers_dirname }}"
   - name: sarek
     release: "{{ sarek_tag }}"
@@ -51,6 +51,9 @@ pipelines:
   - name: scrnaseq
     release: 2.3.0
     container_dir: "{{ biocontainers_dirname }}"
+  - name: demultiplex
+    release: "{{ demultiplex_tag }}"
+    container_dir: "{{ biocontainers_dirname }}"
 
 nf_core_delivery_readmes:
   sarek:

roles/nf-core/tasks/pipeline.yml

@@ -15,18 +15,20 @@
     loop_var: cachedir
 
 - name: nf-core download {{ pipeline }} v{{ release }}
-  command: "nf-core -v
+  command: >
+    nf-core -v
     download {{ pipeline }}
+    -d
     --compress none
     --outdir {{ sw_path }}/{{ pipeline }}
     --revision {{ release }}
-    --container singularity
-    --singularity-cache-only"
+    --container-system singularity
+    --container-cache-utilisation amend
   environment: "{{ nf_core_vars }}"
   ignore_errors: true
   args:
     chdir: "{{ sw_path }}"
-    creates: "{{ sw_path }}/{{ pipeline }}"
+    creates: "{{ sw_path }}/{{ pipeline }}/{{ release | replace('.', '_') }}"
 
 - name: pull pipeline-specific images for {{ pipeline }} v{{ release }}
   command: "singularity pull --name nfcore-{{ container.0 }}-{{ release }}.{{ container.1 }}.img {{ nf_core_container_repo }}/{{ container.0 }}:{{ release }}.{{ container.1 }}"
@@ -50,7 +52,7 @@
   lineinfile:
     dest: "{{ ngi_pipeline_conf }}/{{ bash_env_script }}"
     line: >
-          alias {{ pipeline }}='nextflow run {{ sw_path }}/{{ pipeline }}/workflow/
+          alias {{ pipeline }}='nextflow run {{ sw_path }}/{{ pipeline }}/{{ release | replace('.', '_') }}/
           -profile uppmax \
           -c {{ ngi_pipeline_conf }}/nextflow_miarka_{{ site }}.config \
           -c {{ ngi_pipeline_conf }}/{{ pipeline }}_{{ site }}.config'

roles/ngi_pipeline/templates/miarka_ngi_config.yaml.j2

@@ -33,7 +33,7 @@ piper:
     gatk_key: "{{ ngi_resources }}/piper/{{ gatk_key }}"
 
 sarek:
-    command: {{ sarek_dest }}/workflow/main.nf
+    command: {{ sarek_dest }}/main.nf
     tools:
       - haplotypecaller
       - snpeff
@@ -75,7 +75,7 @@ analysis:
             analysis_engine: ngi_pipeline.engines.qc_ngi
         RNA-seq:
             analysis_engine: ngi_pipeline.engines.rna_ngi
-            ngi_nf_path: {{ sw_path }}/rnaseq/workflow/
+            ngi_nf_path: {{ rnaseq_dest }}/
             sthlm_ngi_conf: {{ rna_seq_conf }}
             upps_ngi_conf: {{ rna_seq_conf }}
         wgs_germline:

roles/ngi_reports/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 ngi_reports_repo: https://github.com/NationalGenomicsInfrastructure/ngi_reports.git
 ngi_reports_dest: "{{ sw_path }}/ngi_reports"
-ngi_reports_version: e09e3f720199b9875f30d4429b4bf35211479747
+ngi_reports_version: 8bfd3bbab27cb6385edd6c128ef2e480d5af774c
 ngi_reports_log: "{{ ngi_log_path }}/ngi_reports.log"
 
 ngi_mdx_outline_repo: https://github.com/NationalGenomicsInfrastructure/mdx_outline.git

roles/snpseq-metadata-service/defaults/main.yml

@@ -0,0 +1,16 @@
+snpseq_metadata_env_name: snpseq_metadata
+snpseq_metadata_env_path: "{{ anaconda_path }}/envs/{{ snpseq_metadata_env_name }}"
+
+snpseq_metadata_repo: https://github.com/Molmed/snpseq_metadata.git
+snpseq_metadata_version: 2763cee
+xml_schema_source_url: ftp://ftp.ebi.ac.uk/pub/databases/ena/doc/xsd/sra_1_5
+
+snpseq_metadata_src_path: "{{ sw_path }}/metadata/snpseq_metadata"
+
+snpseq_metadata_service_repo: https://github.com/Molmed/snpseq-metadata-service.git
+snpseq_metadata_service_version: 1.1.0
+
+snpseq_metadata_service_log_dir: "{{ static_ngi_log_path }}/metadata/snpseq_metadata_service/{{ deployment_environment }}"
+snpseq_metadata_service_src_path: "{{ sw_path }}/metadata/snpseq_metadata_service"
+snpseq_metadata_service_config_root: "{{ ngi_pipeline_conf }}/metadata/snpseq_metadata_service"
+

roles/snpseq-metadata-service/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - { role: anaconda, tags: anaconda }
+  - { role: setup_base_config, tags: setup_base_config }

roles/snpseq-metadata-service/tasks/install_config.yml

@@ -0,0 +1,32 @@
+---
+
+- name: ensure snpseq-metadata-service conf dir exists
+  file:
+    state: directory
+    path: "{{ snpseq_metadata_service_config_root }}"
+
+- name: deploy snpseq-metadata-service app config
+  template:
+    src: snpseq_metadata_service_app.config.j2
+    dest: "{{ snpseq_metadata_service_config_root }}/app.yaml"
+
+- name: deploy snpseq-metadata-service logger config
+  template:
+    src: snpseq_metadata_service_logger.config.j2
+    dest: "{{ snpseq_metadata_service_config_root }}/logger.yaml"
+
+- name: modify uppsala's supervisord conf to set environment and program commands
+  ini_file:
+    dest: "{{ supervisord_conf }}"
+    section: program:snpseq-metadata-service-{{ deployment_environment }}
+    option: "{{ item.opt }}"
+    value: "{{ item.val }}"
+    backup: no
+  loop:
+    - opt: command
+      val: "{{ snpseq_metadata_env_path}}/bin/metadata-service -c {{ snpseq_metadata_service_config_root }}"
+    - opt: autorestart
+      val: true
+    - opt: directory
+      val: "{{ snpseq_metadata_service_log_dir }}"
+