Update components (#3)

* Update docker setup * remove accelerator parameter * clean up docker setup * update common resources * update ci * update scripts * move dataset simulator to process datasets * change to viash 0.9 * update readme * update test resources dest * add organisation * fix typo * update common submodule * update submodule * Update test.yml --------- Co-authored-by: Robrecht Cannoodt <[email protected]>
openproblems-bio · Jul 15, 2024 · e2acdec · e2acdec
1 parent aa8ff52
commit e2acdec
Show file tree

Hide file tree

Showing 55 changed files with 957 additions and 1,091 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,114 +1,21 @@
-name: main build
+name: Build
 
 on:
   push:
     branches: [ 'main' ]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: |
+          The version of the project to build. Example: `1.0.3`.
+          
+          If not provided, a development build with a version name
+          based on the branch name will be built. Otherwise, a release
+          build with the provided version will be built.
+        required: false
 
 jobs:
-  # phase 1
-  list:
-    runs-on: ubuntu-latest
-
-    outputs:
-      component_matrix: ${{ steps.set_matrix.outputs.matrix }}
-      cache_key: ${{ steps.cache.outputs.cache_key }}
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - name: Remove target folder from .gitignore
-      run: |
-        # allow publishing the target folder
-        sed -i 's#^/target/$##g' .gitignore
-
-    - uses: viash-io/viash-actions/ns-build@v5
-      with:
-        config_mod: .functionality.version := 'main_build'
-        parallel: true
-
-    # - name: Build nextflow schemas
-    #   uses: viash-io/viash-actions/pro/build-nextflow-schemas@v4
-    #   with:
-    #     workflows: src
-    #     components: src
-    #     viash_pro_token: ${{ secrets.GTHB_PAT }}
-    #     tools_version: 'main_build'
-
-    # - name: Build parameter files
-    #   uses: viash-io/viash-actions/pro/build-nextflow-params@v4
-    #   with:
-    #     workflows: src
-    #     components: src
-    #     viash_pro_token: ${{ secrets.GTHB_PAT }}
-    #     tools_version: 'main_build'
-
-    - name: Deploy to target branch
-      uses: peaceiris/actions-gh-pages@v4
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: .
-        publish_branch: main_build
-
-    - id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        platform: docker
-        src: src
-        format: json
-
-    - id: set_matrix
-      run: |
-        echo "matrix=$(jq -c '[ .[] | 
-          { 
-            "name": (.functionality.namespace + "/" + .functionality.name),
-            "dir": .info.config | capture("^(?<dir>.*\/)").dir
-          }
-        ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
-  # phase 2
   build:
-    needs: list
-
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        component: ${{ fromJson(needs.list.outputs.component_matrix) }}
-
-    steps:
-    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
-    - uses: data-intuitive/reclaim-the-bytes@v2
-
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - name: Build container
-      uses: viash-io/viash-actions/ns-build@v5
-      with:
-        config_mod: .functionality.version := 'main_build'
-        platform: docker
-        src: ${{ matrix.component.dir }}
-        setup: build
-
-    - name: Login to container registry
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ secrets.GTHB_USER }}
-        password: ${{ secrets.GTHB_PAT }}
-
-    - name: Push container
-      uses: viash-io/viash-actions/ns-build@v5
-      with:
-        config_mod: .functionality.version := 'main_build'
-        platform: docker
-        src: ${{ matrix.component.dir }}
-        setup: push
+    uses: openproblems-bio/actions/.github/workflows/build.yml@main
+    with:
+      version: ${{ github.event.inputs.version }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,110 +1,10 @@
-name: viash test
+name: Test
 
 on:
-  pull_request:
   push:
-    branches: [ '**' ]
+    branches: ["main"]
+  pull_request:
 
 jobs:
-  run_ci_check_job:
-    runs-on: ubuntu-latest
-    outputs:
-      run_ci: ${{ steps.github_cli.outputs.check }}
-    steps:
-      - name: 'Check if branch has an existing pull request and the trigger was a push'
-        id: github_cli
-        run: |
-          pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url')
-          # If the branch has a PR and this run was triggered by a push event, do not run
-          if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then
-            echo "check=false" >> $GITHUB_OUTPUT
-          else
-            echo "check=true" >> $GITHUB_OUTPUT
-          fi
-        env:
-          GITHUB_TOKEN: ${{ secrets.GTHB_PAT }}
-
-  # phase 1
-  list:
-    needs: run_ci_check_job
-    env:
-      s3_bucket: s3://openproblems-data/resources_test/
-    runs-on: ubuntu-latest
-    if: ${{needs.run_ci_check_job.outputs.run_ci == 'true'}}
-
-    outputs:
-      matrix: ${{ steps.set_matrix.outputs.matrix }}
-      cache_key: ${{ steps.cache.outputs.cache_key }}
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0 
-        submodules: true
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
-      id: cache
-      with:
-        s3_bucket: $s3_bucket
-        dest_path: resources_test
-        cache_key_prefix: resources_test__
-
-    - id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        platform: docker
-        format: json
-
-    - id: ns_list_filtered
-      uses: viash-io/viash-actions/project/detect-changed-components@v5
-      with:
-        input_file: "${{ steps.ns_list.outputs.output_file }}"
-
-    - id: set_matrix
-      run: |
-        echo "matrix=$(jq -c '[ .[] | 
-          { 
-            "name": (.functionality.namespace + "/" + .functionality.name),
-            "config": .info.config
-          }
-        ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
-  # phase 2
-  viash_test:
-    needs: list
-    if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        component: ${{ fromJson(needs.list.outputs.matrix) }}
-
-    steps:
-    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
-    - uses: data-intuitive/reclaim-the-bytes@v2
-
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    # use cache
-    - name: Cache resources data
-      uses: actions/cache@v4
-      timeout-minutes: 10
-      with:
-        path: resources_test
-        key: ${{ needs.list.outputs.cache_key }}
-
-    - name: Run test
-      timeout-minutes: 30
-      run: |
-        VIASH_TEMP=$RUNNER_TEMP/viash viash test \
-          "${{ matrix.component.config }}" \
-          --cpus 2 \
-          --memory "5gb"
-
+  build:
+    uses: openproblems-bio/actions/.github/workflows/test.yml@main
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ Estimation of cell type proportions per spot in 2D space from spatial
 transcriptomic data coupled with corresponding single-cell data
 
 Path to source:
-[`src`](https://github.com/openproblems-bio/task-spatial-decomposition/tree/main/src)
+[`src`](https://github.com/openproblems-bio/task_spatial_decomposition/src)
 
 ## Motivation
 
@@ -83,7 +83,7 @@ flowchart LR
 A subset of the common dataset.
 
 Example file:
-`resources_test/spatial_decomposition/cxg_mouse_pancreas_atlas/dataset_simulated.h5ad`
+`resources_test/common/cxg_mouse_pancreas_atlas/dataset.h5ad`
 
 Format:
 
@@ -92,7 +92,7 @@ Format:
     AnnData object
      obs: 'cell_type', 'batch'
      var: 'hvg', 'hvg_score'
-     obsm: 'X_pca', 'coordinates', 'proportions_true'
+     obsm: 'X_pca'
      layers: 'counts'
      uns: 'cell_type_names', 'dataset_id', 'dataset_name', 'dataset_url', 'dataset_reference', 'dataset_summary', 'dataset_description', 'dataset_organism'
 
@@ -108,9 +108,7 @@ Slot description:
 | `obs["batch"]`               | `string`  | A batch identifier. This label is very context-dependent and may be a combination of the tissue, assay, donor, etc. |
 | `var["hvg"]`                 | `boolean` | Whether or not the feature is considered to be a ‘highly variable gene’.                                            |
 | `var["hvg_score"]`           | `double`  | A ranking of the features by hvg.                                                                                   |
-| `obsm["X_pca"]`              | `double`  | The resulting PCA embedding.                                                                                        |
-| `obsm["coordinates"]`        | `double`  | (*Optional*) XY coordinates for each spot.                                                                          |
-| `obsm["proportions_true"]`   | `double`  | (*Optional*) True cell type proportions for each spot.                                                              |
+| `obsm["X_pca"]`              | `double`  | (*Optional*) The resulting PCA embedding.                                                                           |
 | `layers["counts"]`           | `integer` | Raw counts.                                                                                                         |
 | `uns["cell_type_names"]`     | `string`  | (*Optional*) Cell type names corresponding to values in `cell_type`.                                                |
 | `uns["dataset_id"]`          | `string`  | A unique identifier for the dataset.                                                                                |

diff --git a/_viash.yaml b/_viash.yaml
@@ -1,18 +1,25 @@
-viash_version: 0.8.6
+viash_version: 0.9.0-RC6
 
-# name: task_spatial_decomposition
-# description: |
-#   An OpenProblems benchmark task for spatial decomposition.
+name: task_spatial_decomposition
+description: |
+  An OpenProblems benchmark task for spatial decomposition.
+license: MIT
+keywords: [single-cell, openproblems, benchmark, spatial decomposition]
+links:
+  issue_tracker: https://github.com/openproblems-bio/task_spatial_decomposition/issues
+  repository: https://github.com/openproblems-bio/task_spatial_decomposition
+  docker_registry: ghcr.io
+info:
+  test_resources:
+    - type: s3
+      path: s3://openproblems-data/resources_test/common/
+      dest: resources_test/common
+    - type: s3
+      path: s3://openproblems-data/resources_test/spatial_decomposition/
+      dest: resources_test/spatial_decomposition
 
-source: src
-target: target
+organization: openproblems-bio
+version: dev
 
 config_mods: |
-  .functionality.version := 'dev'
-  .platforms[.type == 'docker'].target_registry := 'ghcr.io'
-  .platforms[.type == 'docker'].target_organization := 'openproblems-bio/task-spatial-decomposition'
-  .platforms[.type == 'docker'].target_image_source := 'https://github.com/openproblems-bio/task-spatial-decomposition'
-  .platforms[.type == "nextflow"].directives.tag := "$id"
-  .platforms[.type == "nextflow"].auto.simplifyOutput := false
-  .platforms[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
-  .platforms[.type == "nextflow"].config.script := "process.errorStrategy = 'ignore'"
+  .runners[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
diff --git a/common b/common
diff --git a/scripts/add_component.sh b/scripts/add_component.sh
diff --git a/scripts/add_control_method.sh b/scripts/add_control_method.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+task_name="spatial_decomposition"
+component_name="my_control_method"
+component_lang="python" # change this to "r" if need be
+
+common/create_component/create_component \
+  --task $task_name \
+  --language "$component_lang" \
+  --name "$component_name" \
+  --api_file src/api/comp_control_method.yaml \
+  --output "src/control_methods/$component_name"
diff --git a/scripts/add_method.sh b/scripts/add_method.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+task_name="spatial_decomposition"
+component_name="my_method"
+component_lang="python" # change this to "r" if need be
+
+common/create_component/create_component \
+  --task $task_name \
+  --language "$component_lang" \
+  --name "$component_name" \
+  --api_file src/api/comp_method.yaml \
+  --output "src/methods/$component_name"
diff --git a/scripts/add_metric.sh b/scripts/add_metric.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+task_name="spatial_decomposition"
+component_name="my_metric"
+component_lang="python" # change this to "r" if need be
+
+common/create_component/create_component \
+  --task $task_name \
+  --language "$component_lang" \
+  --name "$component_name" \
+  --api_file src/api/comp_metric.yaml \
+  --output "src/metrics/$component_name"