Merge remote-tracking branch 'origin/main' into spatially_variable_genes

openproblems-bio · Sep 4, 2024 · 339ba74 · 339ba74
2 parents 3854158 + 9493835
commit 339ba74
Show file tree

Hide file tree

Showing 231 changed files with 2,045 additions and 498 deletions.
diff --git a/.github/workflows/viash-test.yml b/.github/workflows/viash-test.yml
@@ -3,34 +3,15 @@ name: viash test
 on:
   pull_request:
   push:
-    branches: [ '**' ]
+    branches: [ main ]
 
 jobs:
-  run_ci_check_job:
-    runs-on: ubuntu-latest
-    outputs:
-      run_ci: ${{ steps.github_cli.outputs.check }}
-    steps:
-      - name: 'Check if branch has an existing pull request and the trigger was a push'
-        id: github_cli
-        run: |
-          pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url')
-          # If the branch has a PR and this run was triggered by a push event, do not run
-          if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then
-            echo "check=false" >> $GITHUB_OUTPUT
-          else
-            echo "check=true" >> $GITHUB_OUTPUT
-          fi
-        env:
-          GITHUB_TOKEN: ${{ secrets.GTHB_PAT }}
 
   # phase 1
   list:
-    needs: run_ci_check_job
     env:
       s3_bucket: s3://openproblems-data/resources_test/
     runs-on: ubuntu-latest
-    if: "needs.run_ci_check_job.outputs.run_ci == 'true'"
 
     outputs:
       matrix: ${{ steps.set_matrix.outputs.matrix }}

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
+README.html
+README_files/
 *.DS_Store
 *__pycache__
 *.h5ad

diff --git a/CONTRIBUTING.qmd b/CONTRIBUTING.qmd
@@ -185,7 +185,7 @@ functionality:
       path: script.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     setup:
       - type: python
         packages: [scikit-learn]

diff --git a/src/common/check_dataset_schema/config.vsh.yaml b/src/common/check_dataset_schema/config.vsh.yaml
@@ -36,7 +36,7 @@ functionality:
       path: test.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     test_setup:
       - type: python
         packages: viashpy

diff --git a/src/common/check_yaml_schema/config.vsh.yaml b/src/common/check_yaml_schema/config.vsh.yaml
@@ -18,7 +18,7 @@ functionality:
       path: script.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     setup:
       - type: python
         pypi:

diff --git a/src/common/create_component/script.py b/src/common/create_component/script.py
@@ -141,11 +141,11 @@ def generate_resources(par, script_path) -> str:
 def generate_docker_platform(par) -> str:
   """Set up the docker platform for Python."""
   if par["language"] == "python":
-    image_str = "ghcr.io/openproblems-bio/base_python:1.0.4"
+    image_str = "openproblems/base_python:1.0.0"
     setup_type = "python"
     package_example = "scib==1.1.5"
   elif par["language"] == "r":
-    image_str = "ghcr.io/openproblems-bio/base_r:1.0.4"
+    image_str = "openproblems/base_r:1.0.0"
     setup_type = "r"
     package_example = "tidyverse"
   return strip_margin(f'''\

diff --git a/src/common/create_task_readme/config.vsh.yaml b/src/common/create_task_readme/config.vsh.yaml
@@ -48,7 +48,7 @@ functionality:
       dest: openproblems-v2/_viash.yaml
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         packages: [dplyr, purrr, rlang, glue, yaml, fs, cli, igraph, rmarkdown, processx]

diff --git a/src/common/extract_metadata/config.vsh.yaml b/src/common/extract_metadata/config.vsh.yaml
@@ -31,7 +31,7 @@ functionality:
       path: test.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     test_setup:
       - type: python
         packages: viashpy

diff --git a/src/common/extract_scores/config.vsh.yaml b/src/common/extract_scores/config.vsh.yaml
@@ -26,7 +26,7 @@ functionality:
       path: script.R
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ tidyverse ]

diff --git a/src/common/helper_functions/read_anndata_partial.py b/src/common/helper_functions/read_anndata_partial.py
@@ -0,0 +1,77 @@
+import warnings
+from pathlib import Path
+import anndata as ad
+import h5py
+from scipy.sparse import csr_matrix
+from anndata.experimental import read_elem, sparse_dataset
+
+
+def read_anndata(
+    file: str,
+    backed: bool = False,
+    **kwargs
+) -> ad.AnnData:
+    """
+    Read anndata file
+    :param file: path to anndata file in h5ad format
+    :param kwargs: AnnData parameter to group mapping
+    """
+    assert Path(file).exists(), f'File not found: {file}'
+
+    f = h5py.File(file, 'r')
+    kwargs = {x: x for x in f} if not kwargs else kwargs
+    if len(f.keys()) == 0:
+        return ad.AnnData()
+    # check if keys are available
+    for name, slot in kwargs.items():
+        if slot not in f:
+            warnings.warn(
+                f'Cannot find "{slot}" for AnnData parameter `{name}` from "{file}"'
+            )
+    adata = read_partial(f, backed=backed, **kwargs)
+    if not backed:
+        f.close()
+
+    return adata
+
+
+def read_partial(
+    group: h5py.Group,
+    backed: bool = False,
+    force_sparse_types: [str, list] = None,
+    **kwargs
+) -> ad.AnnData:
+    """
+    Partially read h5py groups
+    :params group: file group
+    :params force_sparse_types: encoding types to convert to sparse_dataset via csr_matrix
+    :params backed: read sparse matrix as sparse_dataset
+    :params **kwargs: dict of slot_name: slot, by default use all available slot for the h5py file
+    :return: AnnData object
+    """
+    if force_sparse_types is None:
+        force_sparse_types = []
+    elif isinstance(force_sparse_types, str):
+        force_sparse_types = [force_sparse_types]
+    slots = {}
+    if backed:
+        print('Read as backed sparse matrix...')
+
+    for slot_name, slot in kwargs.items():
+        print(f'Read slot "{slot}", store as "{slot_name}"...')
+        if slot not in group:
+            warnings.warn(f'Slot "{slot}" not found, skip...')
+            slots[slot_name] = None
+        else:
+            elem = group[slot]
+            iospec = ad._io.specs.get_spec(elem)
+            if iospec.encoding_type in ("csr_matrix", "csc_matrix") and backed:
+                slots[slot_name] = sparse_dataset(elem)
+            elif iospec.encoding_type in force_sparse_types:
+                slots[slot_name] = csr_matrix(read_elem(elem))
+                if backed:
+                    slots[slot_name] = sparse_dataset(slots[slot_name])
+            else:
+                slots[slot_name] = read_elem(elem)
+    return ad.AnnData(**slots)
+
diff --git a/src/common/ontology/check_obsolete_terms/config.vsh.yaml b/src/common/ontology/check_obsolete_terms/config.vsh.yaml
@@ -70,7 +70,7 @@ functionality:
     - path: /resources_test/common/cellxgene_census
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         packages: [ dplyr, tidyr, tibble, ontologyIndex, processx ]
diff --git a/src/common/process_task_results/generate_qc/config.vsh.yaml b/src/common/process_task_results/generate_qc/config.vsh.yaml
@@ -33,7 +33,7 @@ functionality:
       path: script.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
   - type: nextflow
     directives:
       label: [lowmem, lowtime, lowcpu]
diff --git a/src/common/process_task_results/get_api_info/config.vsh.yaml b/src/common/process_task_results/get_api_info/config.vsh.yaml
@@ -8,7 +8,7 @@ functionality:
       path: script.R
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ purrr, dplyr, yaml, rlang, processx ]

diff --git a/src/common/process_task_results/get_dataset_info/config.vsh.yaml b/src/common/process_task_results/get_dataset_info/config.vsh.yaml
@@ -11,7 +11,7 @@ functionality:
       dest: test_file.yaml
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ purrr, yaml, rlang, processx ]

diff --git a/src/common/process_task_results/get_method_info/config.vsh.yaml b/src/common/process_task_results/get_method_info/config.vsh.yaml
@@ -11,7 +11,7 @@ functionality:
       dest: test_file.yaml
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ purrr, yaml, rlang, processx ]

diff --git a/src/common/process_task_results/get_metric_info/config.vsh.yaml b/src/common/process_task_results/get_metric_info/config.vsh.yaml
@@ -11,7 +11,7 @@ functionality:
       dest: test_file.yaml
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ purrr, yaml, rlang, processx ]

diff --git a/src/common/process_task_results/get_results/config.vsh.yaml b/src/common/process_task_results/get_results/config.vsh.yaml
@@ -42,7 +42,7 @@ functionality:
       path: script.R
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ purrr, yaml, rlang, dplyr, tidyr, readr, lubridate, dynutils, processx ]

diff --git a/src/common/process_task_results/get_task_info/config.vsh.yaml b/src/common/process_task_results/get_task_info/config.vsh.yaml
@@ -11,7 +11,7 @@ functionality:
       dest: test_file.yaml
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ purrr, yaml, rlang, processx ]

diff --git a/src/common/process_task_results/yaml_to_json/config.vsh.yaml b/src/common/process_task_results/yaml_to_json/config.vsh.yaml
@@ -11,6 +11,6 @@ functionality:
       dest: test_file.yaml
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
   - type: nextflow
   - type: native
diff --git a/src/datasets/loaders/cellxgene_census/config.vsh.yaml b/src/datasets/loaders/cellxgene_census/config.vsh.yaml
@@ -151,7 +151,7 @@ functionality:
       path: test.py
 platforms:
   - type: docker
-    #image: ghcr.io/openproblems-bio/base_python:1.0.4
+    #image: openproblems/base_python:1.0.0
     image: python:3.11
     setup:
       - type: python

diff --git a/src/datasets/loaders/cellxgene_census_from_source_h5ad/config.vsh.yaml b/src/datasets/loaders/cellxgene_census_from_source_h5ad/config.vsh.yaml
@@ -114,7 +114,7 @@ functionality:
       path: test.py
 platforms:
   - type: docker
-    #image: ghcr.io/openproblems-bio/base_python:1.0.4
+    #image: openproblems/base_python:1.0.0
     image: python:3.11
     setup:
       - type: python

diff --git a/src/datasets/loaders/openproblems_neurips2021_bmmc/config.vsh.yaml b/src/datasets/loaders/openproblems_neurips2021_bmmc/config.vsh.yaml
@@ -68,7 +68,7 @@ functionality:
     #   path: /resources_test/common/openproblems_neurips2021/neurips2021_bmmc_cite.h5ad
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
   - type: nextflow
     directives:
-      label: [highmem, midcpu , midtime]
+      label: [highmem, midcpu, midtime]
diff --git a/src/datasets/loaders/openproblems_neurips2022_pbmc/config.vsh.yaml b/src/datasets/loaders/openproblems_neurips2022_pbmc/config.vsh.yaml
@@ -74,7 +74,7 @@ functionality:
     #   path: /resources_test/common/openproblems_neurips2021/neurips2021_bmmc_cite.h5ad
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
   - type: nextflow
     directives:
       label: [ highmem, midcpu, midtime]
diff --git a/src/datasets/loaders/openproblems_v1/config.vsh.yaml b/src/datasets/loaders/openproblems_v1/config.vsh.yaml
@@ -72,7 +72,7 @@ functionality:
       path: test.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     setup:
       - type: apt
         packages: git

diff --git a/src/datasets/loaders/openproblems_v1_multimodal/config.vsh.yaml b/src/datasets/loaders/openproblems_v1_multimodal/config.vsh.yaml
@@ -80,7 +80,7 @@ functionality:
       path: test.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     setup:
       - type: apt
         packages: git

diff --git a/src/datasets/normalization/atac_tfidf/config.vsh.yaml b/src/datasets/normalization/atac_tfidf/config.vsh.yaml
@@ -12,7 +12,7 @@ functionality:
       path: script.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     setup:
       - type: python
         packages:

diff --git a/src/datasets/normalization/l1_sqrt/config.vsh.yaml b/src/datasets/normalization/l1_sqrt/config.vsh.yaml
@@ -16,12 +16,12 @@ functionality:
       path: script.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     setup:
       - type: python
         packages:
           - scprep
-          - numpy
+          - numpy<2
   - type: nextflow
     directives: 
       label: [midtime, midmem, midcpu]
diff --git a/src/datasets/normalization/log_cp/config.vsh.yaml b/src/datasets/normalization/log_cp/config.vsh.yaml
@@ -12,7 +12,7 @@ functionality:
       description: "Number of counts per cell. When set to -1, will use None."
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
   - type: nextflow
     directives: 
       label: [midtime, midmem, midcpu]
diff --git a/src/datasets/normalization/log_scran_pooling/config.vsh.yaml b/src/datasets/normalization/log_scran_pooling/config.vsh.yaml
@@ -7,7 +7,7 @@ functionality:
       path: script.R
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_r:1.0.4
+    image: openproblems/base_r:1.0.0
     setup:
       - type: r
         cran: [ Matrix, rlang, scran, BiocParallel ]

diff --git a/src/datasets/normalization/prot_clr/config.vsh.yaml b/src/datasets/normalization/prot_clr/config.vsh.yaml
@@ -16,7 +16,7 @@ functionality:
       path: script.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
     setup:
       - type: python
         packages:

diff --git a/src/datasets/normalization/sqrt_cp/config.vsh.yaml b/src/datasets/normalization/sqrt_cp/config.vsh.yaml
@@ -12,7 +12,7 @@ functionality:
       description: "Number of counts per cell"
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
   - type: nextflow
     directives: 
       label: [midtime, midmem, midcpu]
diff --git a/src/datasets/processors/hvg/config.vsh.yaml b/src/datasets/processors/hvg/config.vsh.yaml
@@ -7,7 +7,7 @@ functionality:
       path: script.py
 platforms:
   - type: docker
-    image: ghcr.io/openproblems-bio/base_python:1.0.4
+    image: openproblems/base_python:1.0.0
   - type: nextflow
     directives:
       label: [midtime, highmem, midcpu]