Merge branch 'nf-core:master' into nanoqc

.github/CODEOWNERS

@@ -12,3 +12,4 @@ modules/nf-core/pints/caller/** @edmundmiller
 modules/nf-core/preseq/ccurve/** @edmundmiller
 modules/nf-core/preseq/lcextrap/** @edmundmiller
 subworkflows/nf-core/homer/** @edmundmiller
+modules/nf-core/**/Dockerfile @edmundmiller @maxulysse

.github/renovate.json5

@@ -1,79 +1,14 @@
 {
-    $schema: "https://docs.renovatebot.com/renovate-schema.json",
-    extends: ["config:best-practices"],
-    vulnerabilityAlerts: {
-        labels: ["security"],
-        schedule: "at any time",
-    },
-    prHourlyLimit: 3,
-    platform: "github",
-    onboarding: false,
-    dependencyDashboard: true,
-    dependencyDashboardTitle: "Renovate Dashboard 🤖",
-    suppressNotifications: ["prIgnoreNotification"],
-    rebaseWhen: "conflicted",
-    reviewers: ["team:modules"],
-    assigneesFromCodeOwners: true,
-    gitAuthor: "nf-core-bot <[email protected]>",
-    prHeader: "## WARNING\n - [ ] I have made sure to update the singularity and conda images",
-    "pre-commit": {
-        enabled: true,
-    },
-    customManagers: [
-    // TODO waiting on https://github.com/nf-core/modules/pull/4080
-        // {
-        //     customType: "regex",
-        //     description: "Process Containers dependencies",
-        //     fileMatch: ["(^|/)main\\.nf$"],
-        //     matchStrings: [
-        //         "\\s+'(?<depName>\\S+):(?<currentValue>.*\\-.*)' }\"\n",
-        //     ],
-        //     datasourceTemplate: "docker",
-        // },
-        // {
-        //     customType: "regex",
-        //     description: "Upgrade conda dependencies",
-        //     fileMatch: ["(^|/)environment(.*).ya?ml$"],
-        //     matchStrings: [
-        //         '#\\s*renovate\\s+datasource=conda\\s+depName=(?<depName>.*?)\\s+-\\s*[\\w-]+\\s*==?\\s*"?(?<currentValue>.*)"?',
-        //         '# renovate: datasource=conda depName=(?<depName>.*?)\\s+-\\s*[\\w-]+\\s*==?\\s*"?(?<currentValue>.*)"?',
-        //     ],
-        //     datasourceTemplate: "conda",
-        // },
-        // {
-        //     customType: "regex",
-        //     description: "Upgrade pypi dependencies inside conda environment files",
-        //     fileMatch: ["(^|/)environment(.*).ya?ml$"],
-        //     matchStrings: [
-        //         "# renovate datasource=pypi\\s+-\\s*(?<depName>[\\w-]+)\\s*(\\[[\\w,\\s]+\\])?\\s*==?\\s*(?<currentValue>.*)",
-        //         "# renovate: datasource=pypi\\s+-\\s*(?<depName>[\\w-]+)\\s*(\\[[\\w,\\s]+\\])?\\s*==?\\s*(?<currentValue>.*)",
-        //     ],
-        //     datasourceTemplate: "pypi",
-        // },
-        ],
-    packageRules: [
-        {
-            matchDatasources: ["docker"],
-            registryUrls: ["quay.io"],
-        },
-        {
-            matchManagers: ["github-actions"],
-            automerge: true,
-            pinDigest: {
-                enabled: false,
-            },
-            matchPackageNames: ["/^nf-core//", "/^seqera/"],
-        },
-        {
-            matchManagers: ["github-actions"],
-            groupName: "GitHub Actions",
-            automerge: true,
-            schedule: ["before 9am"],
-        },
-        {
-            automerge: true,
-            schedule: ["before 9am on Monday"],
-            matchPackageNames: ["/renovate-config-validator/"],
-        },
+    "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+    "extends": [
+        "github>nf-core/ops//.github/renovate/default.json5",
+        "github>nf-core/ops//.github/renovate/process_updaters.json5"
     ],
+    "prHeader": "## WARNING\n - [ ] I have made sure to update the singularity and conda images",
+    "packageRules": [
+        {
+            "matchDatasources": ["conda", "pypi", "docker"],
+            "automerge": false
+        }
+    ]
 }

.github/workflows/test.yml

@@ -1,5 +1,9 @@
 name: Run tests
 on:
+  push:
+    branches:
+      # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging
+      - "renovate/**" # branches Renovate creates
   pull_request:
     branches: [master]
   merge_group:
@@ -135,7 +139,7 @@ jobs:
           echo ${{ steps.outputs.outputs.subworkflows }}
 
   nf-core-lint-modules:
-    runs-on: ${{ github.event.inputs.runners  || 'self-hosted' }}
+    runs-on: ${{ github.event.inputs.runners  || 'ubuntu-latest' }}
     name: nf-core-lint-modules
     needs: [pytest-changes, nf-test-changes]
     if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }}
@@ -219,7 +223,7 @@ jobs:
         run: nf-core subworkflows lint ${{ matrix.tags }}
 
   pytest:
-    runs-on: ${{ github.event.inputs.runners  || 'self-hosted' }}
+    runs-on: ${{ github.event.inputs.runners  || 'ubuntu-latest' }}
     name: pytest
     needs: [pytest-changes]
     if: needs.pytest-changes.outputs.tags != '[]'
@@ -485,7 +489,7 @@ jobs:
             !${{ github.workspace }}/.singularity
 
   nf-test:
-    runs-on: ${{ github.event.inputs.runners  || 'self-hosted' }}
+    runs-on: ${{ github.event.inputs.runners  || 'ubuntu-latest' }}
     name: nf-test
     needs: [nf-test-changes]
     if: ( needs.nf-test-changes.outputs.paths != '[]' )

.github/workflows/wave.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - "**/environment.yml"
+      - "**/Dockerfile"
 
 # TODO On complete call testing CI
 # TODO Skip testing CI if any changes to environment.yml
@@ -18,38 +19,44 @@ jobs:
     runs-on: ubuntu-latest
     # Only run on Pull Requests within the same repository, and not from forks
     if: github.event.pull_request.head.repo.full_name == github.repository
-
+    outputs:
+      conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }}
+      docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
-      - name: Calculate file differences
-        id: diff
+      - name: Find conda differences
+        id: conda-diff
         uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45
         with:
           json: true
-          quotepath: false
-          # TODO Add Dockerfiles
+          escape_json: false
           files: |
             modules/**/environment.yml
-      - name: Debug
-        run: echo ${{ steps.diff.outputs.all_changed_files }}
-      - id: set-matrix
-        run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT"
 
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      - name: Find Dockerfile differences
+        id: docker-diff
+        uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45
+        with:
+          json: true
+          escape_json: false
+          files: |
+            modules/**/Dockerfile
 
-  build:
+  conda-wave:
     # NOTE This should get skipped because generate-matrix won't run
     # if: github.repository == 'nf-core/modules'
-    if: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}"
+    if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }}
     needs: generate-matrix
-    name: build
+    name: Build Conda-based ${{ matrix.profile }} Container
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
-      matrix: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}"
+      max-parallel: 4
+      matrix:
+        files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}"
+        profile: [docker, singularity]
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
@@ -59,38 +66,64 @@ jobs:
           sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
           chmod +x /usr/local/bin/wave
 
-      - name: Create a registry name
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
-        id: registry-name
-        with:
-          result-encoding: string
-          script: |
-            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_');
-
-      - name: Build container
+      - name: Build ${{ matrix.profile }} container
         # FIXME Hack while iron out the CI
         continue-on-error: true
-        if: matrix.profile == 'docker'
+        env:
+          PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }}
         run: |
           wave --conda-file "${{ matrix.files }}" \
+              $PROFILE \
               --freeze \
               --await \
               --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
               --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
 
-      - name: Build Singularity
-        # FIXME Hack while iron out the CI
-        continue-on-error: true
-        if: matrix.profile == 'singularity'
+  docker-wave:
+    # NOTE This should get skipped because generate-matrix won't run
+    # if: github.repository == 'nf-core/modules'
+    if: ${{ needs.generate-matrix.outputs.docker-matrix != '[]' }}
+    needs: generate-matrix
+    name: Build Dockerfile-based ${{ matrix.profile }} Container
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        files: "${{ fromJson(needs.generate-matrix.outputs.docker-matrix) }}"
+        profile: [docker, singularity]
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: Install wave-cli
         run: |
-          wave --conda-file "${{ matrix.files }}" \
+          wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64
+          sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
+          chmod +x /usr/local/bin/wave
+
+      - name: Create a registry name
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+        id: registry-name
+        with:
+          result-encoding: string
+          script: |
+            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/Dockerfile', '').replace('/', '_');
+
+      - name: Build ${{ matrix.profile }} container
+        # NOTE If you're changing a Dockerfile and this is running, try to update the Dockerfile to build with wave
+        continue-on-error: false
+        env:
+          PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }}
+        run: |
+          wave -f "${{ matrix.files }}" \
+              $PROFILE \
               --freeze \
               --await \
+              --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \
+              --cache-repository quay.io/nf-core/wave-cache \
               --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
-              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \
-              --singularity
-
-    # TODO Build from Dockerfiles
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
 
   # bump-versions:
   #   needs: generate-matrix

modules/nf-core/biscuit/align/environment.yml

@@ -2,5 +2,5 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - bioconda::biscuit=1.1.0.20220707
-  - bioconda::samtools=1.16.1
+  - bioconda::biscuit=1.5.0.20240506
+  - bioconda::samtools=1.21

modules/nf-core/biscuit/align/main.nf

@@ -4,12 +4,13 @@ process BISCUIT_ALIGN {
 
     conda "${moduleDir}/environment.yml"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0':
-        'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }"
+        'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data':
+        'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }"
 
     input:
     tuple val(meta), path(reads)
-    path index
+    tuple val(meta2), path(fasta)
+    tuple val(meta3), path(index)
 
     output:
     tuple val(meta), path("*.bam"), emit: bam
@@ -23,18 +24,34 @@ process BISCUIT_ALIGN {
     def args = task.ext.args ?: ''
     def args2 = task.ext.args2 ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
-    def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.9),1)
-    def samtools_cpus = task.cpus-biscuit_cpus
+    def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max()
+    def samtools_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus)
     """
-    INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'`
+    ln -sf \$(readlink $fasta) $index/$fasta
 
     biscuit align \\
         $args \\
         -@ $biscuit_cpus \\
-        \$INDEX \\
+        $index/$fasta \\
         $reads \\
-        | samtools sort $args2 --threads $samtools_cpus --write-index -o ${prefix}.bam##idx##${prefix}.bam.bai -
+        | samtools sort $args2 --threads $samtools_cpus --write-index -o ${prefix}.bam -O BAM -
 
+    samtools index ${prefix}.bam
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        biscuit: \$( biscuit version |& sed '1!d; s/^.*BISCUIT Version: //' )
+        samtools: \$( samtools --version |& sed '1!d; s/^.*samtools //' )
+    END_VERSIONS
+    """
+
+    stub:
+    def args = task.ext.args ?: ''
+    def args2 = task.ext.args2 ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    touch ${prefix}.bam
+    touch ${prefix}.bam.bai
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/nf-core/biscuit/align/meta.yml

@@ -27,12 +27,24 @@ input:
     - reads:
         type: file
         description: |
-          List of input fastq files of size 1 and 2 for single-end and paired-end data,
+          List of input FastQ files of size 1 and 2 for single-end and paired-end data,
           respectively.
-  - - index:
+  - - meta2:
+        type: map
+        description: |
+          Groovy Map containing sample information
+          e.g. [ id:'test', single_end:false ]
+    - fasta:
+        type: file
+        description: Input genome fasta file
+  - - meta3:
+        type: map
+        description: |
+          Groovy Map containing sample information
+          e.g. [ id:'test', single_end:false ]
+    - index:
         type: directory
-        description: Biscuit genome index directory (generated with 'biscuit index')
-        pattern: "BiscuitIndex"
+        description: Directory containing biscuit genome index
 output:
   - bam:
       - meta:
@@ -63,3 +75,4 @@ authors:
   - "@njspix"
 maintainers:
   - "@njspix"
+  - "@sateeshperi"