Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update components #3

Merged
merged 15 commits into from
Jul 15, 2024
121 changes: 14 additions & 107 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -1,114 +1,21 @@
name: main build
name: Build

on:
push:
branches: [ 'main' ]
workflow_dispatch:
inputs:
version:
description: |
The version of the project to build. Example: `1.0.3`.
If not provided, a development build with a version name
based on the branch name will be built. Otherwise, a release
build with the provided version will be built.
required: false

jobs:
# phase 1
list:
runs-on: ubuntu-latest

outputs:
component_matrix: ${{ steps.set_matrix.outputs.matrix }}
cache_key: ${{ steps.cache.outputs.cache_key }}

steps:
- uses: actions/checkout@v4
with:
submodules: true

- uses: viash-io/viash-actions/setup@v5

- name: Remove target folder from .gitignore
run: |
# allow publishing the target folder
sed -i 's#^/target/$##g' .gitignore
- uses: viash-io/viash-actions/ns-build@v5
with:
config_mod: .functionality.version := 'main_build'
parallel: true

# - name: Build nextflow schemas
# uses: viash-io/viash-actions/pro/build-nextflow-schemas@v4
# with:
# workflows: src
# components: src
# viash_pro_token: ${{ secrets.GTHB_PAT }}
# tools_version: 'main_build'

# - name: Build parameter files
# uses: viash-io/viash-actions/pro/build-nextflow-params@v4
# with:
# workflows: src
# components: src
# viash_pro_token: ${{ secrets.GTHB_PAT }}
# tools_version: 'main_build'

- name: Deploy to target branch
uses: peaceiris/actions-gh-pages@v4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: .
publish_branch: main_build

- id: ns_list
uses: viash-io/viash-actions/ns-list@v5
with:
platform: docker
src: src
format: json

- id: set_matrix
run: |
echo "matrix=$(jq -c '[ .[] |
{
"name": (.functionality.namespace + "/" + .functionality.name),
"dir": .info.config | capture("^(?<dir>.*\/)").dir
}
]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
# phase 2
build:
needs: list

runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
component: ${{ fromJson(needs.list.outputs.component_matrix) }}

steps:
# Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
- uses: data-intuitive/reclaim-the-bytes@v2

- uses: actions/checkout@v4
with:
submodules: true

- uses: viash-io/viash-actions/setup@v5

- name: Build container
uses: viash-io/viash-actions/ns-build@v5
with:
config_mod: .functionality.version := 'main_build'
platform: docker
src: ${{ matrix.component.dir }}
setup: build

- name: Login to container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GTHB_USER }}
password: ${{ secrets.GTHB_PAT }}

- name: Push container
uses: viash-io/viash-actions/ns-build@v5
with:
config_mod: .functionality.version := 'main_build'
platform: docker
src: ${{ matrix.component.dir }}
setup: push
uses: openproblems-bio/actions/.github/workflows/build.yml@main
with:
version: ${{ github.event.inputs.version }}
110 changes: 5 additions & 105 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -1,110 +1,10 @@
name: viash test
name: Test

on:
pull_request:
push:
branches: [ '**' ]
branches: ["main"]
pull_request:

jobs:
run_ci_check_job:
runs-on: ubuntu-latest
outputs:
run_ci: ${{ steps.github_cli.outputs.check }}
steps:
- name: 'Check if branch has an existing pull request and the trigger was a push'
id: github_cli
run: |
pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url')
# If the branch has a PR and this run was triggered by a push event, do not run
if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then
echo "check=false" >> $GITHUB_OUTPUT
else
echo "check=true" >> $GITHUB_OUTPUT
fi
env:
GITHUB_TOKEN: ${{ secrets.GTHB_PAT }}

# phase 1
list:
needs: run_ci_check_job
env:
s3_bucket: s3://openproblems-data/resources_test/
runs-on: ubuntu-latest
if: ${{needs.run_ci_check_job.outputs.run_ci == 'true'}}

outputs:
matrix: ${{ steps.set_matrix.outputs.matrix }}
cache_key: ${{ steps.cache.outputs.cache_key }}

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true

- uses: viash-io/viash-actions/setup@v5

- uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
id: cache
with:
s3_bucket: $s3_bucket
dest_path: resources_test
cache_key_prefix: resources_test__

- id: ns_list
uses: viash-io/viash-actions/ns-list@v5
with:
platform: docker
format: json

- id: ns_list_filtered
uses: viash-io/viash-actions/project/detect-changed-components@v5
with:
input_file: "${{ steps.ns_list.outputs.output_file }}"

- id: set_matrix
run: |
echo "matrix=$(jq -c '[ .[] |
{
"name": (.functionality.namespace + "/" + .functionality.name),
"config": .info.config
}
]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT

# phase 2
viash_test:
needs: list
if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
component: ${{ fromJson(needs.list.outputs.matrix) }}

steps:
# Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
- uses: data-intuitive/reclaim-the-bytes@v2

- uses: actions/checkout@v4
with:
submodules: true

- uses: viash-io/viash-actions/setup@v5

# use cache
- name: Cache resources data
uses: actions/cache@v4
timeout-minutes: 10
with:
path: resources_test
key: ${{ needs.list.outputs.cache_key }}

- name: Run test
timeout-minutes: 30
run: |
VIASH_TEMP=$RUNNER_TEMP/viash viash test \
"${{ matrix.component.config }}" \
--cpus 2 \
--memory "5gb"

build:
uses: openproblems-bio/actions/.github/workflows/test.yml@main
10 changes: 4 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Estimation of cell type proportions per spot in 2D space from spatial
transcriptomic data coupled with corresponding single-cell data

Path to source:
[`src`](https://github.com/openproblems-bio/task-spatial-decomposition/tree/main/src)
[`src`](https://github.com/openproblems-bio/task_spatial_decomposition/src)

## Motivation

Expand Down Expand Up @@ -83,7 +83,7 @@ flowchart LR
A subset of the common dataset.

Example file:
`resources_test/spatial_decomposition/cxg_mouse_pancreas_atlas/dataset_simulated.h5ad`
`resources_test/common/cxg_mouse_pancreas_atlas/dataset.h5ad`

Format:

Expand All @@ -92,7 +92,7 @@ Format:
AnnData object
obs: 'cell_type', 'batch'
var: 'hvg', 'hvg_score'
obsm: 'X_pca', 'coordinates', 'proportions_true'
obsm: 'X_pca'
layers: 'counts'
uns: 'cell_type_names', 'dataset_id', 'dataset_name', 'dataset_url', 'dataset_reference', 'dataset_summary', 'dataset_description', 'dataset_organism'

Expand All @@ -108,9 +108,7 @@ Slot description:
| `obs["batch"]` | `string` | A batch identifier. This label is very context-dependent and may be a combination of the tissue, assay, donor, etc. |
| `var["hvg"]` | `boolean` | Whether or not the feature is considered to be a ‘highly variable gene’. |
| `var["hvg_score"]` | `double` | A ranking of the features by hvg. |
| `obsm["X_pca"]` | `double` | The resulting PCA embedding. |
| `obsm["coordinates"]` | `double` | (*Optional*) XY coordinates for each spot. |
| `obsm["proportions_true"]` | `double` | (*Optional*) True cell type proportions for each spot. |
| `obsm["X_pca"]` | `double` | (*Optional*) The resulting PCA embedding. |
| `layers["counts"]` | `integer` | Raw counts. |
| `uns["cell_type_names"]` | `string` | (*Optional*) Cell type names corresponding to values in `cell_type`. |
| `uns["dataset_id"]` | `string` | A unique identifier for the dataset. |
Expand Down
35 changes: 21 additions & 14 deletions _viash.yaml
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
viash_version: 0.8.6
viash_version: 0.9.0-RC6

# name: task_spatial_decomposition
# description: |
# An OpenProblems benchmark task for spatial decomposition.
name: task_spatial_decomposition
description: |
An OpenProblems benchmark task for spatial decomposition.
license: MIT
keywords: [single-cell, openproblems, benchmark, spatial decomposition]
links:
issue_tracker: https://github.com/openproblems-bio/task_spatial_decomposition/issues
repository: https://github.com/openproblems-bio/task_spatial_decomposition
docker_registry: ghcr.io
info:
test_resources:
- type: s3
path: s3://openproblems-data/resources_test/common/
dest: resources_test/common
- type: s3
path: s3://openproblems-data/resources_test/spatial_decomposition/
dest: resources_test/spatial_decomposition

source: src
target: target
organization: openproblems-bio
version: dev

config_mods: |
.functionality.version := 'dev'
.platforms[.type == 'docker'].target_registry := 'ghcr.io'
.platforms[.type == 'docker'].target_organization := 'openproblems-bio/task-spatial-decomposition'
.platforms[.type == 'docker'].target_image_source := 'https://github.com/openproblems-bio/task-spatial-decomposition'
.platforms[.type == "nextflow"].directives.tag := "$id"
.platforms[.type == "nextflow"].auto.simplifyOutput := false
.platforms[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
.platforms[.type == "nextflow"].config.script := "process.errorStrategy = 'ignore'"
.runners[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
2 changes: 1 addition & 1 deletion common
21 changes: 0 additions & 21 deletions scripts/add_component.sh

This file was deleted.

12 changes: 12 additions & 0 deletions scripts/add_control_method.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

task_name="spatial_decomposition"
component_name="my_control_method"
component_lang="python" # change this to "r" if need be

common/create_component/create_component \
--task $task_name \
--language "$component_lang" \
--name "$component_name" \
--api_file src/api/comp_control_method.yaml \
--output "src/control_methods/$component_name"
12 changes: 12 additions & 0 deletions scripts/add_method.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

task_name="spatial_decomposition"
component_name="my_method"
component_lang="python" # change this to "r" if need be

common/create_component/create_component \
--task $task_name \
--language "$component_lang" \
--name "$component_name" \
--api_file src/api/comp_method.yaml \
--output "src/methods/$component_name"
12 changes: 12 additions & 0 deletions scripts/add_metric.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

task_name="spatial_decomposition"
component_name="my_metric"
component_lang="python" # change this to "r" if need be

common/create_component/create_component \
--task $task_name \
--language "$component_lang" \
--name "$component_name" \
--api_file src/api/comp_metric.yaml \
--output "src/metrics/$component_name"
Loading