Merge branch 'master' into pypgx/runngspipeline

.github/scripts/wave_singularity.py

@@ -2,53 +2,86 @@
 # /// script
 # requires-python = ">=3.10"
 # dependencies = [
-#     "httpx",
+#     "requests",
+#     "rich",
 # ]
 # ///
 
 import logging
 
-import httpx
+import requests
+import rich_click as click
+from rich.logging import RichHandler
 
+# Replace the basic logger setup with rich logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(message)s",
+    handlers=[
+        RichHandler(
+            rich_tracebacks=True,
+            show_time=False,
+            markup=True,
+        )
+    ],
+)
 logger = logging.getLogger(__name__)
+click.rich_click.SHOW_ARGUMENTS = True
 
-image_url = "oras://community.wave.seqera.io/library/pybedtools_bedtools_htslib_pip_pypints:aa20de1f1b5ddb30"
-
-if image_url.startswith("oras://"):
-    image_url = image_url.replace("oras://", "")
-
-wave_api_url = "https://wave.seqera.io"
-url = f"{wave_api_url}/v1alpha1/inspect"
-
-# if platform_pat:
-#     data["toweraccesstoken"] = platform_pat
-# else:
-# TODO
-logger.warning("'platform_pat' not set, no auth to wave back end")
-
-try:
-    logger.info(f"calling image inspect at {url} for image url {image_url}")
-    response = httpx.post(
-        url=url,
-        json={"containerImage": image_url},
-        headers={"content-type": "application/json"},
-    )
-
-    data = response.json()
-    logger.debug(data)
-    layers = data.get("container", {}).get("manifest", {}).get("layers", [])
-    is_singularity = len(layers) == 1 and layers[0].get("mediaType", "").endswith(".sif")
-    if not is_singularity:
-        print(layers)
-        raise ValueError("not a singularity image")
-    if "digest" not in layers[0]:
-        print(layers)
-        raise ValueError("no 'digest' in first layer found")
-
-    digest = layers[0]["digest"].replace("sha256:", "")
-    container_url = f"https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/{digest[:2]}/{digest}/data"
-    print(container_url)
-
-except httpx.RequestError as exc:
-    print(f"An error occurred while requesting {exc.request.url!r}.")
-    print("No singularity image for you")
+
+@click.command()
+@click.option(
+    "--platform-pat",
+    envvar="SEQERA_ACCESS_TOKEN",
+    show_envvar=True,
+    help="Platform authentication token",
+)
+@click.argument("image_name")
+def main(image_name, platform_pat):
+    """Script to return a HTTPS Singularity image URL from Seqera Containers."""
+
+    if image_name.startswith("oras://"):
+        image_name = image_name.replace("oras://", "")
+
+    wave_api_url_base = "https://wave.seqera.io"
+    wave_api_url = f"{wave_api_url_base}/v1alpha1/inspect"
+    container_url_base = "https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256"
+
+    request_data = {"containerImage": image_name}
+    if platform_pat:
+        request_data["toweraccesstoken"] = platform_pat
+    else:
+        logger.debug("'--platform-pat' / '$TOWER_ACCESS_TOKEN' not set, no auth to wave back end")
+
+    try:
+        logger.debug(f"Calling image inspect at {wave_api_url} for image url '{image_name}'")
+        response = requests.post(
+            url=wave_api_url,
+            json=request_data,
+            headers={"content-type": "application/json"},
+        )
+
+        data = response.json()
+        logger.debug(data)
+        layers = data.get("container", {}).get("manifest", {}).get("layers", [])
+        is_singularity = len(layers) == 1 and layers[0].get("mediaType", "").endswith(".sif")
+        logger.debug(layers)
+        if not is_singularity:
+            raise ValueError("Not a singularity image")
+        if "digest" not in layers[0]:
+            raise ValueError("no 'digest' in first layer found")
+
+        digest = layers[0]["digest"].replace("sha256:", "")
+        container_url = f"{container_url_base}/{digest[:2]}/{digest}/data"
+        print(container_url)
+
+    except requests.RequestException as exc:
+        raise ValueError(f"An error occurred while requesting {wave_api_url}\n {exc}")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except ValueError as exc:
+        logger.error(f"[red]{exc}[/red]")
+        exit(1)

.github/workflows/test.yml

@@ -436,7 +436,7 @@ jobs:
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
       - name: Set up miniconda
-        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
+        uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3
         with:
           miniconda-version: "latest"
           channels: conda-forge,bioconda
@@ -538,6 +538,10 @@ jobs:
             path: modules/nf-core/custom/dumpsoftwareversions
           - profile: conda
             path: modules/nf-core/deepcell/mesmer
+          - profile: conda
+            path: modules/nf-core/deepsomatic
+          - profile: singularity
+            path: modules/nf-core/deepsomatic
           - profile: conda
             path: modules/nf-core/deepvariant
           - profile: conda
@@ -550,6 +554,8 @@ jobs:
             path: modules/nf-core/deepvariant/rundeepvariant
           - profile: conda
             path: modules/nf-core/deepvariant/vcfstatsreport
+          - profile: conda
+            path: modules/nf-core/doubletdetection
           - profile: conda
             path: modules/nf-core/ensemblvep/vep
           - profile: conda
@@ -662,6 +668,15 @@ jobs:
             path: subworkflows/nf-core/fastq_align_bwa
           - profile: conda
             path: subworkflows/nf-core/fasta_newick_epang_gappa
+          - profile: conda
+            path: modules/nf-core/xeniumranger/relabel
+          - profile: conda
+            path: modules/nf-core/xeniumranger/rename
+          - profile: conda
+            path: modules/nf-core/xeniumranger/resegment
+          - profile: conda
+            path: modules/nf-core/xeniumranger/import-segmentation
+
     env:
       NXF_ANSI_LOG: false
       NFTEST_VER: "0.9.0"
@@ -711,7 +726,7 @@ jobs:
 
       - name: Set up miniconda
         if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
+        uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3
         with:
           miniconda-version: "latest"
           auto-update-conda: true

.github/workflows/wave.yml

@@ -0,0 +1,141 @@
+name: Wave
+# When environment.yml is changed
+on:
+  pull_request:
+    paths:
+      - "**/environment.yml"
+
+# TODO On complete call testing CI
+# TODO Skip testing CI if any changes to environment.yml
+
+env:
+  WAVE_VER: "1.5.0"
+  NFTEST_VER: "0.9.1"
+
+jobs:
+  generate-matrix:
+    name: generate-matrix
+    runs-on: ubuntu-latest
+    if: github.repository == 'nf-core/modules'
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: Calculate file differences
+        id: diff
+        uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45
+        with:
+          json: true
+          quotepath: false
+          # TODO Add Dockerfiles
+          files: |
+            modules/**/environment.yml
+      - name: Debug
+        run: echo ${{ steps.diff.outputs.all_changed_files }}
+      - id: set-matrix
+        run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT"
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+  build:
+    # NOTE This should get skipped because generate-matrix won't run
+    # if: github.repository == 'nf-core/modules'
+    if: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}"
+    needs: generate-matrix
+    name: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: Install wave-cli
+        run: |
+          wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64
+          sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
+          chmod +x /usr/local/bin/wave
+
+      - name: Create a registry name
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+        id: registry-name
+        with:
+          result-encoding: string
+          script: |
+            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_');
+
+      - name: Build container
+        if: matrix.profile == 'docker'
+        run: |
+          wave --conda-file "${{ matrix.files }}" \
+              --freeze \
+              --await \
+              --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
+
+      - name: Build Singularity
+        if: matrix.profile == 'singularity'
+        run: |
+          wave --conda-file "${{ matrix.files }}" \
+              --freeze \
+              --await \
+              --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \
+              --singularity
+
+    # TODO Build from Dockerfiles
+
+  # bump-versions:
+  #   needs: generate-matrix
+  #   name: bump-versions
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+  #       with:
+  #         distribution: "temurin"
+  #         java-version: "17"
+
+  #     - uses: nf-core/setup-nextflow@v2
+
+  #     - uses: nf-core/setup-nf-test@v1
+  #       with:
+  #         version: ${{ env.NFTEST_VER }}
+
+  #     - name: Bump Snapshot Versions
+  #       env:
+  #         # NFT_DIFF: "pdiff"
+  #         # NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+  #         SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+  #         SENTIEON_AUTH_MECH: "GitHub Actions - token"
+  #       run: |
+  #         # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker
+  #         if [ "${{ matrix.profile }}" == "docker" ]; then
+  #           PROFILE="docker_self_hosted"
+  #         else
+  #           PROFILE=${{ matrix.profile }}
+  #         fi
+
+  #         NFT_WORKDIR=~ \
+  #         nf-test test \
+  #           --profile=${{ matrix.profile }} \
+  #           --tap=test.tap \
+  #           # --ci \
+  #           --verbose \
+  #           --only-changed \
+  #           # --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \
+  #           # --filter ${{ matrix.filter }} \
+  #           --follow-dependencies \
+  #           --tag version \
+  #           --update-snapshot
+
+  #     - name: Commit & push version bumps
+  #       run: |
+  #         git config user.email "[email protected]"
+  #         git config user.name "nf-core-bot"
+  #         git config push.default upstream
+  #         git add .
+  #         git status
+  #         git commit -m "[automated] Bump versions snapshot"
+  #         git push

.pre-commit-config.yaml

@@ -17,13 +17,9 @@ repos:
         files: ^subworkflows/nf-core/.*/meta\.yml$
         args: ["--schemafile", "subworkflows/yaml-schema.json"]
       - id: check-github-workflows
-  - repo: https://github.com/renovatebot/pre-commit-hooks
-    rev: 38.46.0
-    hooks:
-      - id: renovate-config-validator
   # use ruff for python files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.2
+    rev: v0.7.3
     hooks:
       - id: ruff
         files: \.py$

.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+    "recommendations": ["nextflow.nextflow", "nf-core.nf-core-extensionpack"]
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "nextflow.formatting.harshilAlignment": true
+}

modules/nf-core/ale/environment.yml

@@ -5,4 +5,5 @@ channels:
   - bioconda
   - tanghaibao
 dependencies:
+  # renovate: datasource=conda depName=bioconda/ale
   - "bioconda::ale=20180904"

modules/nf-core/anndata/barcodes/environment.yml

@@ -0,0 +1,5 @@
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - conda-forge::anndata=0.10.9

modules/nf-core/anndata/barcodes/main.nf

@@ -0,0 +1,36 @@
+process ANNDATA_BARCODES {
+    tag "$meta.id"
+    label 'process_single'
+
+    conda "${moduleDir}/environment.yml"
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'oras://community.wave.seqera.io/library/anndata:0.10.9--d13580e4b297da7c':
+        'community.wave.seqera.io/library/anndata:0.10.9--1eab54e300e1e584' }"
+
+    input:
+    tuple val(meta), path(h5ad), path(barcodes)
+
+    output:
+    tuple val(meta), path("*.h5ad"), emit: h5ad
+    path "versions.yml"            , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    prefix = task.ext.prefix ?: "${meta.id}"
+    template 'barcodes.py'
+
+    stub:
+    prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    touch ${prefix}.h5ad
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        python: \$(python3 -c 'import platform; print(platform.python_version())')
+        anndata: \$(python3 -c 'import anndata as ad; print(ad.__version__)')
+        pandas: \$(python3 -c 'import pandas as pd; print(pd.__version__)')
+    END_VERSIONS
+    """
+}