diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..9a8a64b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,24 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: [bug] +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..a49eab2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..c17d3c0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: [enhancement] +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..3717137 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ +## Describe your changes + + + +## Checklist before requesting a review +- [ ] I have performed a self-review of my code + +- Check the correct box. Does this PR contain: + - [ ] Breaking changes + - [ ] New functionality + - [ ] Major changes + - [ ] Minor changes + - [ ] Bug fixes + +- [ ] Proposed changes are described in the CHANGELOG.md + +- [ ] CI Tests succeed and look good! \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b6b568d..b33cbdb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,122 +1,21 @@ -name: build +name: Build on: push: branches: [ 'main' ] workflow_dispatch: inputs: - target_branch: - description: 'Branch to deploy to. If not specified, `build-${BRANCH_NAME}` will be used.' - required: false version: - description: 'Version name to use for the build. If not specified, `build-${BRANCH_NAME}` will be used.' + description: | + The version of the project to build. Example: `1.0.3`. + + If not provided, a development build with a version name + based on the branch name will be built. Otherwise, a release + build with the provided version will be built. required: false jobs: - # phase 1 - list: - runs-on: ubuntu-latest - - outputs: - target_branch: ${{ steps.defaults.outputs.target_branch }} - version: ${{ steps.defaults.outputs.version }} - component_matrix: ${{ steps.set_matrix.outputs.matrix }} - - steps: - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - - uses: viash-io/viash-actions/setup@v5 - - - name: Determine version tag from branch name - id: defaults - run: | - BRANCH_NAME=$(echo $GITHUB_REF | sed 's/refs\/heads\///') - - VERSION=${{ github.event.inputs.version }} - if [ -z "$VERSION" ]; then - VERSION="build-$BRANCH_NAME" - fi - echo "version=$VERSION" >> $GITHUB_OUTPUT - - TARGET_BRANCH=${{ github.event.inputs.target_branch }} - if [ -z "$TARGET_BRANCH" ]; then - TARGET_BRANCH="build-$BRANCH_NAME" - fi - echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT - - - name: Remove target folder from .gitignore - run: | - # allow publishing the target folder - sed -i '/^\/target.*/d' .gitignore - - - uses: viash-io/viash-actions/ns-build@v5 - with: - config_mod: .version := '${{ steps.defaults.outputs.version }}' - parallel: true - - - name: Deploy to target branch - uses: peaceiris/actions-gh-pages@v4 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: . - publish_branch: ${{ steps.defaults.outputs.target_branch }} - - - id: ns_list - uses: viash-io/viash-actions/ns-list@v5 - with: - platform: docker - src: src - format: json - - - id: set_matrix - run: | - echo "matrix=$(jq -c '[ .[] | - { - "name": (.namespace + "/" + .name), - "dir": .info.config | capture("^(?.*\/)").dir - } - ]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT - - # phase 2 build: - needs: list - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - component: ${{ fromJson(needs.list.outputs.component_matrix) }} - - steps: - # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' - - uses: data-intuitive/reclaim-the-bytes@v2 - - - uses: actions/checkout@v4 - - - uses: viash-io/viash-actions/setup@v5 - - - name: Build container - uses: viash-io/viash-actions/ns-build@v5 - with: - config_mod: .version := '${{ needs.list.outputs.version }}' - platform: docker - src: ${{ matrix.component.dir }} - setup: build - - - name: Login to container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ secrets.GTHB_USER }} - password: ${{ secrets.GTHB_PAT }} - - - name: Push container - uses: viash-io/viash-actions/ns-build@v5 - with: - config_mod: .version := '${{ needs.list.outputs.version }}' - platform: docker - src: ${{ matrix.component.dir }} - setup: push \ No newline at end of file + uses: openproblems-bio/actions/.github/workflows/build.yml@main + with: + version: ${{ github.event.inputs.version }} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0abad5c..96811dd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,113 +1,9 @@ -name: test +name: Test on: - pull_request: push: - branches: [ '**' ] + pull_request: jobs: - run_ci_check_job: - runs-on: ubuntu-latest - outputs: - run_ci: ${{ steps.github_cli.outputs.check }} - steps: - - name: 'Check if branch has an existing pull request and the trigger was a push' - id: github_cli - run: | - pull_request=$(gh pr list -R ${{ github.repository }} -H ${{ github.ref_name }} --json url --state open --limit 1 | jq '.[0].url') - # If the branch has a PR and this run was triggered by a push event, do not run - if [[ "$pull_request" != "null" && "$GITHUB_REF_NAME" != "main" && "${{ github.event_name == 'push' }}" == "true" && "${{ !contains(github.event.head_commit.message, 'ci force') }}" == "true" ]]; then - echo "check=false" >> $GITHUB_OUTPUT - else - echo "check=true" >> $GITHUB_OUTPUT - fi - env: - GITHUB_TOKEN: ${{ secrets.GTHB_PAT }} - - # phase 1 - list: - needs: run_ci_check_job - env: - s3_bucket: s3://openproblems-data/resources_test - runs-on: ubuntu-latest - if: ${{ needs.run_ci_check_job.outputs.run_ci == 'true' }} - - outputs: - matrix: ${{ steps.set_matrix.outputs.matrix }} - cache_key: ${{ steps.cache.outputs.cache_key }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'recursive' - - - uses: christian-ci/action-yaml-github-output@v2 - with: - file_path: _viash.yaml - - - uses: viash-io/viash-actions/setup@v5 - - - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5 - id: cache - with: - s3_bucket: $s3_bucket/$NAME - dest_path: resources - cache_key_prefix: resources__ - - - id: ns_list - uses: viash-io/viash-actions/ns-list@v5 - with: - engine: docker - format: json - - - id: ns_list_filtered - uses: viash-io/viash-actions/project/detect-changed-components@v5 - with: - input_file: "${{ steps.ns_list.outputs.output_file }}" - - - id: set_matrix - run: | - echo "matrix=$(jq -c '[ .[] | - { - "name": (.namespace + "/" + .name), - "config": .info.config - } - ]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT - - # phase 2 - viash_test: - needs: list - if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }} - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - component: ${{ fromJson(needs.list.outputs.matrix) }} - - steps: - # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' - - uses: data-intuitive/reclaim-the-bytes@v2 - - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - - uses: viash-io/viash-actions/setup@v5 - - # use cache - - name: Cache resources data - uses: actions/cache@v4 - timeout-minutes: 10 - with: - path: resources - key: ${{ needs.list.outputs.cache_key }} - - - name: Run test - timeout-minutes: 30 - run: | - VIASH_TEMP=$RUNNER_TEMP/viash viash test \ - "${{ matrix.component.config }}" \ - --cpus 2 \ - --memory "16gb" + build: + uses: openproblems-bio/actions/.github/workflows/test.yml@main \ No newline at end of file diff --git a/.gitignore b/.gitignore index fe80ce0..cc7020c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ resources +resources_test work .nextflow* target diff --git a/.gitmodules b/.gitmodules index a7ec0e3..c07c083 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "common"] path = common - url = git@github.com:openproblems-bio/common-resources.git + url = https://github.com/openproblems-bio/common_resources.git diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..77b0339 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,24 @@ +# task_template x.y.z + +## BREAKING CHANGES + + + +## NEW FUNCTIONALITY + +* Added `control_methods/true_labels` component (PR #5). + +* Added `methods/logistic_regression` component (PR #5). + +* Added `metrics/accuracy` component (PR #5). + +## MAJOR CHANGES + +* Updated `api` files (PR #5). + +## MINOR CHANGES + +* Updated `README.md` (PR #5). + +## BUGFIXES + diff --git a/INSTRUCTIONS.md b/INSTRUCTIONS.md deleted file mode 100644 index 74287af..0000000 --- a/INSTRUCTIONS.md +++ /dev/null @@ -1,73 +0,0 @@ -# Instructions - -This is a guide on what to do after you have created a new task repository from the template. More in depth information about how to create a new task can be found in the [OpenProblems Documentation](https://openproblems.bio/documentation/create_task/). - -## First things first - -* Update the `_viash.yaml` file with the correct task information. -* Update the `src/api/task_info.yaml` file with the information you have provied in the task issue. - -## Resources - -THe OpenProblems team has provided some test resources that can be used to test the task. These resources are stored in the `resources` folder. The `scripts/download_resources.sh` script can be used to download these resources. - -If these resources are not sufficient, you can add more resources to the `resources` folder. The `scripts/download_resources.sh` script can be updated to download these resources. - - - - - - - -#!/bin/bash - -echo "This script is not supposed to be run directly." -echo "Please run the script step-by-step." -exit 1 - -# sync resources -scripts/download_resources.sh - -# create a new component -method_id="my_metric" -method_lang="python" # change this to "r" if need be - -common/create_component/create_component -- \ - --language "$method_lang" \ - --name "$method_id" - -# TODO: fill in required fields in src/task/methods/foo/config.vsh.yaml -# TODO: edit src/task/methods/foo/script.py/R - -# test the component -viash test src/task/methods/$method_id/config.vsh.yaml - -# rebuild the container (only if you change something to the docker platform) -# You can reduce the memory and cpu allotted to jobs in _viash.yaml by modifying .platforms[.type == "nextflow"].config.labels -viash run src/task/methods/$method_id/config.vsh.yaml -- \ - ---setup cachedbuild ---verbose - -# run the method (using parquet as input) -viash run src/task/methods/$method_id/config.vsh.yaml -- \ - --de_train "resources/neurips-2023-kaggle/de_train.parquet" \ - --id_map "resources/neurips-2023-kaggle/id_map.csv" \ - --output "output/prediction.parquet" - -# run the method (using h5ad as input) -viash run src/task/methods/$method_id/config.vsh.yaml -- \ - --de_train_h5ad "resources/neurips-2023-kaggle/2023-09-12_de_by_cell_type_train.h5ad" \ - --id_map "resources/neurips-2023-kaggle/id_map.csv" \ - --output "output/prediction.parquet" - -# run evaluation metric -viash run src/task/metrics/mean_rowwise_error/config.vsh.yaml -- \ - --de_test "resources/neurips-2023-kaggle/de_test.parquet" \ - --prediction "output/prediction.parquet" \ - --output "output/score.h5ad" - -# print score on kaggle test dataset -python -c 'import anndata; print(anndata.read_h5ad("output/score.h5ad").uns)' \ No newline at end of file diff --git a/README.md b/README.md index 0c87796..da3ffe5 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repo is a template to create a new task for the OpenProblems v2. This repo ## Create a repository from this template > [!IMPORTANT] -> Before creating a new repository, make sure you are part of the openProblems task team. This will be done when you create an issue for the task and you got the go ahead to create the task. +> Before creating a new repository, make sure you are part of the OpenProblems task team. This will be done when you create an issue for the task and you get the go ahead to create the task. > For more information on how to create a new task, check out the [Create a new task](https://openproblems.bio/documentation/create_task/) documentation. The instructions below will guide you through creating a new repository from this template ([creating-a-repository-from-a-template](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-repository-from-a-template#creating-a-repository-from-a-template)). @@ -20,8 +20,18 @@ The instructions below will guide you through creating a new repository from thi * Set the repository visibility to public. * Click "Create repository from template". +## Clone the repository + +To clone the repository with the submodule files, you can use the following command: + +```bash +git clone --recursive git@github.com:openproblems-bio/.git +``` +>[!NOTE] +> If somehow there are no files visible in the submodule after cloning using the above command. Check the instructions [here](common/README.md). + ## What to do next -Check out the [instructions](INSTRUCTIONS.md) for more information on how to update the example files and components. These instructions also contain information on how to build out the task and basic commands. +Check out the [instructions](https://github.com/openproblems-bio/common_resources/blob/main/INSTRUCTIONS.md) for more information on how to update the example files and components. These instructions also contain information on how to build out the task and basic commands. -For more information on the OpenProblems v2, check out the [Documentation](https://openproblems.bio/documentation/) on the Open Problems website. +For more information on the OpenProblems v2, check out the [documentation](https://openproblems.bio/documentation/). \ No newline at end of file diff --git a/_viash.yaml b/_viash.yaml index 923e418..6399f74 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -1,16 +1,64 @@ viash_version: 0.9.0-RC6 +# Step 1: Change the name of the task. +# example: task_name_of_this_task name: task_template +organization: openproblems-bio +version: dev +# Step 2: Update the description to a short description of the task. description: | An OpenProblems benchmark task. license: MIT +# Step 3: Add keywords to describe the task. keywords: [single-cell, openproblems, benchmark] +# Step 4: Update the `task_template` to the name of the task from step 1. links: - issue_tracker: https://github.com/openproblems-bio/task_/issues - repository: https://github.com/openproblems-bio/task_ - docker_registry: ghcr.io/openproblems-bio + issue_tracker: https://github.com/openproblems-bio/task_template/issues + repository: https://github.com/openproblems-bio/task_template + docker_registry: ghcr.io +# Step 5: Update the info fields to the text from the task issue. +info: + # A unique, human-readable, short label. Used for creating summary tables and visualisations. + label: Template + description: | + Provide a clear and concise description of your task, detailing the specific problem it aims + to solve. Outline the input data types, the expected output, and any assumptions or constraints. + Be sure to explain any terminology or concepts that are essential for understanding the task. + summary: A one sentence summary of purpose and methodology. Used for creating an overview tables. + motivation: | + Explain the motivation behind your proposed task. Describe the biological or computational + problem you aim to address and why it’s important. Discuss the current state of research in + this area and any gaps or challenges that your task could help address. This section + should convince readers of the significance and relevance of your task. + image: The name of the image file to use for the component on the website. + # Step 6: Replace the task_template to the name of the task in `info.name`. + test_resources: + - type: s3 + path: s3://openproblems-data/resources_test/task_template/ + dest: resources_test/task_template + - type: s3 + path: s3://openproblems-data/resources_test/common/ + dest: resources_test/common -version: dev +# Step 7: Update the authors of the task. +authors: + # Full name of the author, usually in the name of FirstName MiddleName LastName. + - name: Kai Waldrant + # Role of the author. Possible values: + # + # * `"author"`: Authors who have made substantial contributions to the component. + # * `"maintainer"`: The maintainer of the component. + # * `"contributor"`: Authors who have made smaller contributions (such as code patches etc.). + roles: [ "author", "maintainer" ] + # Additional information on the author + info: + github: KaiWaldrant + orcid: 0009-0003-8555-1361 + email: ... + twitter: ... + linkedin: ... +# Step 8: Remove all of the comments of the steps you completed +# Step 9: High five yourself! config_mods: | - .runners[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" } \ No newline at end of file + .runners[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" } diff --git a/common b/common index ecbb47c..f82ff10 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit ecbb47ca0cb36e9350760cf126d5c7e3125f26de +Subproject commit f82ff105c986474651240f4e7aef53fd5019b8a4 diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..2f7ffd3 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1,3 @@ +add_a_method.sh +add_a_control_method.sh +add_a_metric.sh \ No newline at end of file diff --git a/scripts/add_a_control_method.sh b/scripts/add_a_control_method.sh deleted file mode 100644 index d853907..0000000 --- a/scripts/add_a_control_method.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -task_name="task_template" -component_name="my_control_method" -component_lang="python" # change this to "r" if need be - -common/create_component/create_component \ - --task $task_name \ - --language "$component_lang" \ - --name "$component_name" \ - --api_file src/api/comp_control_method.yaml \ - --output "src/control_methods/$component_name" \ No newline at end of file diff --git a/scripts/add_a_method.sh b/scripts/add_a_method.sh deleted file mode 100644 index 8812644..0000000 --- a/scripts/add_a_method.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -task_name="task_template" -component_name="my_method" -component_lang="python" # change this to "r" if need be - -common/create_component/create_component \ - --task $task_name \ - --language "$component_lang" \ - --name "$component_name" \ - --api_file src/api/comp_method.yaml \ - --output "src/methods/$component_name" \ No newline at end of file diff --git a/scripts/add_a_metric.sh b/scripts/add_a_metric.sh deleted file mode 100644 index 71d6067..0000000 --- a/scripts/add_a_metric.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -task_name="task_template" -component_name="my_metric" -component_lang="python" # change this to "r" if need be - -common/create_component/create_component \ - --task $task_name \ - --language "$component_lang" \ - --name "$component_name" \ - --api_file src/api/comp_metric.yaml \ - --output "src/metrics/$component_name" \ No newline at end of file diff --git a/scripts/create_readme.sh b/scripts/create_readme.sh old mode 100644 new mode 100755 index 0857cb6..e5dec6f --- a/scripts/create_readme.sh +++ b/scripts/create_readme.sh @@ -2,5 +2,4 @@ common/create_task_readme/create_task_readme \ --task_dir src \ - --github_url https://github.com/openproblems-bio/task-template \ --output README.md diff --git a/scripts/create_test_resources.sh b/scripts/create_test_resources.sh new file mode 100644 index 0000000..a39f8c4 --- /dev/null +++ b/scripts/create_test_resources.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +RAW_DATA=resources_test/common +DATASET_DIR=resources_test/task_template + +mkdir -p $DATASET_DIR + +# process dataset +echo Running process_dataset +nextflow run . \ + -main-script target/nextflow/workflows/process_datasets/main.nf \ + -profile docker \ + -entry auto \ + --input_states "$RAW_DATA/**/state.yaml" \ + --rename_keys 'input:output_dataset' \ + --settings '{"output_train": "$id/train.h5ad", "output_test": "$id/test.h5ad"}' \ + --publish_dir "$DATASET_DIR" \ + --output_state '$id/state.yaml' + +# run one method +viash run src/methods/logistic_regression/config.vsh.yaml -- \ + --input_train $DATASET_DIR/pancreas/train.h5ad \ + --input_test $DATASET_DIR/pancreas/test.h5ad \ + --output $DATASET_DIR/pancreas/denoised.h5ad + +# run one metric +viash run src/metrics/accuracy/config.vsh.yaml -- \ + --input_predicition $DATASET_DIR/pancreas/predicted.h5ad \ + --input_solution $DATASET_DIR/pancreas/solution.h5ad \ + --output $DATASET_DIR/pancreas/score.h5ad \ No newline at end of file diff --git a/scripts/download_resources.sh b/scripts/download_resources.sh old mode 100644 new mode 100755 index 945c47e..8e50685 --- a/scripts/download_resources.sh +++ b/scripts/download_resources.sh @@ -4,7 +4,6 @@ set -e echo ">> Downloading resources" -viash run common/src/sync_resources/config.vsh.yaml -- \ - --input "s3://openproblems-data/resources_test/common/" \ - --output "resources_test" \ - --delete \ No newline at end of file +# the sync_resources script uses the test_resources S3 URI's in the _viash.yaml to download the resources. +common/sync_resources/sync_resources \ + --delete diff --git a/scripts/run_benchmark.sh b/scripts/run_benchmark.sh new file mode 100644 index 0000000..cc4275e --- /dev/null +++ b/scripts/run_benchmark.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +RUN_ID="run_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="s3://openproblems-data/resources/task_template/results/${RUN_ID}" + +# make sure only log_cp10k is used +cat > /tmp/params.yaml << HERE +input_states: s3://openproblems-data/resources/task_template/datasets/**/state.yaml +rename_keys: 'input_train:output_train;input_test:output_test' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +tw launch https://github.com/openproblems-bio/task_template.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 53907369739130 \ + --compute-env 6TeIFgV5OY4pJCk8I0bfOh \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config common/nextflow_helpers/labels_tw.config \ + --labels task_template,full \ No newline at end of file diff --git a/scripts/run_benchmark_test.sh b/scripts/run_benchmark_test.sh new file mode 100644 index 0000000..6c03d42 --- /dev/null +++ b/scripts/run_benchmark_test.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +cat > /tmp/params.yaml << 'HERE' +input_states: s3://openproblems-data/resources_test/task_template/**/state.yaml +rename_keys: 'input_train:output_train;input_test:output_test' +output_state: "state.yaml" +publish_dir: s3://openproblems-nextflow/temp/task_template/ +HERE + +tw launch https://github.com/openproblems-bio/task_template.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 53907369739130 \ + --compute-env 6TeIFgV5OY4pJCk8I0bfOh \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config common/nextflow_helpers/labels_tw.config \ + --labels task_template,test \ No newline at end of file diff --git a/scripts/test_all_components.sh b/scripts/test_all_components.sh old mode 100644 new mode 100755 diff --git a/src/api/comp_control_method.yaml b/src/api/comp_control_method.yaml index fd3ac29..0bfd973 100644 --- a/src/api/comp_control_method.yaml +++ b/src/api/comp_control_method.yaml @@ -3,27 +3,36 @@ info: type: control_method type_info: label: Control Method - summary: A control method. + summary: Quality control methods for verifying the pipeline. description: | - A control method to predict effects. + This folder contains control components for the task. + These components have the same interface as the regular methods + but also receive the solution object as input. It serves as a + starting point to test the relative accuracy of new methods in + the task, and also as a quality control for the metrics defined + in the task. arguments: - - name: --train_h5ad + - name: --input_train __merge__: file_train_h5ad.yaml - required: false + required: true direction: input - - name: --test_h5ad + - name: --input_test __merge__: file_test_h5ad.yaml required: true direction: input + - name: "--input_solution" + __merge__: file_solution.yaml + direction: input + required: true - name: --output __merge__: file_prediction.yaml required: true direction: output test_resources: - type: python_script - path: /common/src/component_tests/run_and_check_output.py + path: /common/component_tests/run_and_check_output.py - type: python_script path: /common/component_tests/check_method_config.py - path: /common/library.bib - #TODO: - path: fill in e.g. /resources/denoising/pancreas - #TODO: dest: fill in e.g. resources/denoising/pancreas \ No newline at end of file + - path: /resources_test/task_template/pancreas + dest: resources_test/task_template/pancreas \ No newline at end of file diff --git a/src/api/comp_data_processor.yaml b/src/api/comp_data_processor.yaml new file mode 100644 index 0000000..184bc54 --- /dev/null +++ b/src/api/comp_data_processor.yaml @@ -0,0 +1,31 @@ +namespace: "data_processors" +info: + type: data_processor + type_info: + label: Data processor + summary: A data processor. + description: | + A component for processing a Common Dataset into a task-specific dataset. +arguments: + - name: "--input" + __merge__: file_common_dataset.yaml + direction: input + required: true + - name: "--output_train" + __merge__: file_train_h5ad.yaml + direction: output + required: true + - name: "--output_test" + __merge__: file_test_h5ad.yaml + direction: output + required: true + - name: "--output_solution" + __merge__: file_solution.yaml + direction: output + required: true +test_resources: + - path: /resources_test/common/pancreas + dest: resources_test/common/pancreas + - type: python_script + path: /common/component_tests/run_and_check_output.py + diff --git a/src/api/comp_method.yaml b/src/api/comp_method.yaml index 10316b5..651f81e 100644 --- a/src/api/comp_method.yaml +++ b/src/api/comp_method.yaml @@ -7,10 +7,14 @@ info: description: | A method to predict the task effects. arguments: - - name: --train_h5ad + - name: --input_train __merge__: file_train_h5ad.yaml - required: false + required: true + direction: input + - name: "--input_test" + __merge__: file_test_h5ad.yaml direction: input + required: true - name: --output __merge__: file_prediction.yaml required: true @@ -21,5 +25,5 @@ test_resources: - type: python_script path: /common/component_tests/check_method_config.py - path: /common/library.bib - #TODO: - path: fill in e.g. /resources/denoising/pancreas - #TODO: dest: fill in e.g. resources/denoising/pancreas \ No newline at end of file + - path: /resources_test/task_template/pancreas + dest: resources_test/task_template/pancreas \ No newline at end of file diff --git a/src/api/comp_metric.yaml b/src/api/comp_metric.yaml index 9dc8c29..47286ee 100644 --- a/src/api/comp_metric.yaml +++ b/src/api/comp_metric.yaml @@ -3,12 +3,12 @@ info: type: metric type_info: label: Metric - summary: A metric. + summary: A task template metric. description: | A metric for evaluating method predictions. arguments: - - name: "--input_test" - __merge__: file_test_h5ad.yaml + - name: "--input_solution" + __merge__: file_solution.yaml direction: input required: true - name: "--input_prediction" @@ -21,9 +21,9 @@ arguments: required: true test_resources: - type: python_script - path: /common/comp_tests/check_metric_config.py + path: /common/component_tests/check_metric_config.py - type: python_script - path: /common/comp_tests/run_and_check_adata.py + path: /common/component_tests/run_and_check_output.py - path: /common/library.bib - #TODO: - path: fill in e.g. /resources/denoising/pancreas - #TODO: dest: fill in e.g. resources/denoising/pancreas + - path: /resources_test/task_template/pancreas + dest: resources_test/task_template/pancreas diff --git a/src/api/file_common_dataset.yaml b/src/api/file_common_dataset.yaml new file mode 100644 index 0000000..f3bd931 --- /dev/null +++ b/src/api/file_common_dataset.yaml @@ -0,0 +1,71 @@ +type: file +example: "resources_test/common/pancreas/dataset.h5ad" +info: + label: "Common Dataset" + summary: A subset of the common dataset. + slots: + layers: + - type: integer + name: counts + description: Raw counts + required: true + - type: double + name: normalized + description: Normalized expression values + required: true + obs: + - type: string + name: cell_type + description: Cell type information + required: true + - type: string + name: batch + description: Batch information + required: true + var: + - type: boolean + name: hvg + description: Whether or not the feature is considered to be a 'highly variable gene' + required: true + - type: double + name: hvg_score + description: A ranking of the features by hvg. + required: true + obsm: + - type: double + name: X_pca + description: The resulting PCA embedding. + required: true + uns: + - type: string + name: dataset_id + description: "A unique identifier for the dataset" + required: true + - name: dataset_name + type: string + description: Nicely formatted name. + required: true + - type: string + name: dataset_url + description: Link to the original source of the dataset. + required: false + - name: dataset_reference + type: string + description: Bibtex reference of the paper in which the dataset was published. + required: false + - name: dataset_summary + type: string + description: Short description of the dataset. + required: true + - name: dataset_description + type: string + description: Long description of the dataset. + required: true + - name: dataset_organism + type: string + description: The organism of the sample in the dataset. + required: false + - type: string + name: normalization_id + description: "Which normalization was used" + required: true diff --git a/src/api/file_prediction.yaml b/src/api/file_prediction.yaml index b473e75..629d146 100644 --- a/src/api/file_prediction.yaml +++ b/src/api/file_prediction.yaml @@ -1,20 +1,24 @@ #TODO: Change to the required and/or optional fields of the anndata type: file -example: "resources_test/denoising/pancreas/denoised.h5ad" +example: "resources_test/task_template/pancreas/prediction.h5ad" info: label: "Predicted data" summary: A predicted dataset as output by a method. slots: - layers: - - type: integer - name: prediction - description: predicted data + obs: + - type: string + name: label_pred + description: Predicted labels for the test cells. required: true uns: - type: string name: dataset_id description: "A unique identifier for the dataset" required: true + - type: string + name: normalization_id + description: "Which normalization was used" + required: true - type: string name: method_id description: "A unique identifier for the method" diff --git a/src/api/file_score.yaml b/src/api/file_score.yaml index 8aefeba..a2567dc 100644 --- a/src/api/file_score.yaml +++ b/src/api/file_score.yaml @@ -10,6 +10,10 @@ info: name: dataset_id description: "A unique identifier for the dataset" required: true + - type: string + name: normalization_id + description: "Which normalization was used" + required: true - type: string name: method_id description: "A unique identifier for the method" diff --git a/src/api/file_solution.yaml b/src/api/file_solution.yaml new file mode 100644 index 0000000..e160651 --- /dev/null +++ b/src/api/file_solution.yaml @@ -0,0 +1,71 @@ +type: file +example: "resources_test/task_template/pancreas/solution.h5ad" +info: + label: "Solution" + summary: "The solution for the test data" + slots: + layers: + - type: integer + name: counts + description: Raw counts + required: true + - type: double + name: normalized + description: Normalized counts + required: true + obs: + - type: string + name: label + description: Ground truth cell type labels + required: true + - type: string + name: batch + description: Batch information + required: true + var: + - type: boolean + name: hvg + description: Whether or not the feature is considered to be a 'highly variable gene' + required: true + - type: double + name: hvg_score + description: A ranking of the features by hvg. + required: true + obsm: + - type: double + name: X_pca + description: The resulting PCA embedding. + required: true + uns: + - type: string + name: dataset_id + description: "A unique identifier for the dataset" + required: true + - name: dataset_name + type: string + description: Nicely formatted name. + required: true + - type: string + name: dataset_url + description: Link to the original source of the dataset. + required: false + - name: dataset_reference + type: string + description: Bibtex reference of the paper in which the dataset was published. + required: false + - name: dataset_summary + type: string + description: Short description of the dataset. + required: true + - name: dataset_description + type: string + description: Long description of the dataset. + required: true + - name: dataset_organism + type: string + description: The organism of the sample in the dataset. + required: false + - type: string + name: normalization_id + description: "Which normalization was used" + required: true diff --git a/src/api/file_test_h5ad.yaml b/src/api/file_test_h5ad.yaml index d373b84..f53a396 100644 --- a/src/api/file_test_h5ad.yaml +++ b/src/api/file_test_h5ad.yaml @@ -1,6 +1,6 @@ #TODO: Change to the required and/or optional fields of the anndata type: file -example: "resources_test/denoising/pancreas/test.h5ad" +example: "resources_test/task_template/pancreas/test.h5ad" info: label: "Test data" summary: The subset of molecules used for the test dataset @@ -10,36 +10,35 @@ info: name: counts description: Raw counts required: true + - type: double + name: normalized + description: Normalized counts + required: true + obs: + - type: string + name: batch + description: Batch information + required: true + var: + - type: boolean + name: hvg + description: Whether or not the feature is considered to be a 'highly variable gene' + required: true + - type: double + name: hvg_score + description: A ranking of the features by hvg. + required: true + obsm: + - type: double + name: X_pca + description: The resulting PCA embedding. + required: true uns: - type: string name: dataset_id description: "A unique identifier for the dataset" required: true - - name: dataset_name - type: string - description: Nicely formatted name. - required: true - type: string - name: dataset_url - description: Link to the original source of the dataset. - required: false - - name: dataset_reference - type: string - description: Bibtex reference of the paper in which the dataset was published. - required: false - - name: dataset_summary - type: string - description: Short description of the dataset. - required: true - - name: dataset_description - type: string - description: Long description of the dataset. - required: true - - name: dataset_organism - type: string - description: The organism of the sample in the dataset. - required: false - - name: train_sum - type: integer - description: The total number of counts in the training dataset. + name: normalization_id + description: "Which normalization was used" required: true \ No newline at end of file diff --git a/src/api/file_train_h5ad.yaml b/src/api/file_train_h5ad.yaml index 9ec0e86..e9c2bb4 100644 --- a/src/api/file_train_h5ad.yaml +++ b/src/api/file_train_h5ad.yaml @@ -1,19 +1,48 @@ #TODO: Change to the required and/or optional fields of the anndata type: file -example: "resources_test/denoising/pancreas/train.h5ad" +example: "resources_test/task_template/pancreas/train.h5ad" info: label: "Training data" - summary: The subset of molecules used for the training dataset + summary: "The training data in h5ad format" slots: layers: - type: integer name: counts description: Raw counts required: true + - type: double + name: normalized + description: Normalized counts + required: true + obs: + - type: string + name: label + description: Ground truth cell type labels + required: true + - type: string + name: batch + description: Batch information + required: true + var: + - type: boolean + name: hvg + description: Whether or not the feature is considered to be a 'highly variable gene' + required: true + - type: double + name: hvg_score + description: A ranking of the features by hvg. + required: true + obsm: + - type: double + name: X_pca + description: The resulting PCA embedding. + required: true uns: - type: string name: dataset_id description: "A unique identifier for the dataset" required: true - # obs: - # ... \ No newline at end of file + - type: string + name: normalization_id + description: "Which normalization was used" + required: true \ No newline at end of file diff --git a/src/api/task_info.yaml b/src/api/task_info.yaml deleted file mode 100644 index 5166700..0000000 --- a/src/api/task_info.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: A unique identifier. Can only contain lowercase letters, numbers or underscores. -label: A unique, human-readable, short label. Used for creating summary tables and visualisations. -summary: A one sentence summary of purpose and methodology. Used for creating an overview tables. -image: The name of the image file to use for the component on the website. -readme: | - ## Installation - - You need to have Docker, Java, and Viash installed. Follow - [these instructions](https://openproblems.bio/documentation/fundamentals/requirements) - to install the required dependencies. - - ## Add a method - - To add a method to the repository, follow the instructions in the `scripts/add_a_method.sh` script. - - ## Frequently used commands - - To get started, you can run the following commands: - - ```bash - git clone git@github.com:openproblems-bio/.git - - cd - - # download resources - scripts/download_resources.sh - ``` - - To run the benchmark, you first need to build the components. Afterwards, you can run the benchmark: - - ```bash - viash ns build --parallel --setup cachedbuild - - scripts/run_benchmark.sh - ``` - - After adding a component, it is recommended to run the tests to ensure that the component is working correctly: - - ```bash - viash ns test --parallel - ``` - - Optionally, you can provide the `--query` argument to test only a subset of components: - - ```bash - viash ns test --parallel --query "component_name" - ``` -motivation: | - Explain the motivation behind your proposed task. Describe the biological or computational - problem you aim to address and why it’s important. Discuss the current state of research in - this area and any gaps or challenges that your task could help address. This section - should convince readers of the significance and relevance of your task. -description: | - Provide a clear and concise description of your task, detailing the specific problem it aims - to solve. Outline the input data types, the expected output, and any assumptions or constraints. - Be sure to explain any terminology or concepts that are essential for understanding the task. - -authors: - # Full name of the author, usually in the name of FirstName MiddleName LastName. - - name: ... - # Role of the author. Possible values: - # - # * `"author"`: Authors who have made substantial contributions to the component. - # * `"maintainer"`: The maintainer of the component. - # * `"contributor"`: Authors who have made smaller contributions (such as code patches etc.). - roles: [ ... ] - # Additional information on the author - info: - github: ... - orcid: ... - email: ... - twitter: ... - linkedin: ... \ No newline at end of file diff --git a/src/control_methods/my_control_method/config.vsh.yaml b/src/control_methods/true_labels/config.vsh.yaml similarity index 82% rename from src/control_methods/my_control_method/config.vsh.yaml rename to src/control_methods/true_labels/config.vsh.yaml index bba79d9..d32eaec 100644 --- a/src/control_methods/my_control_method/config.vsh.yaml +++ b/src/control_methods/true_labels/config.vsh.yaml @@ -8,21 +8,21 @@ __merge__: ../../api/comp_control_method.yaml # A unique identifier for your component (required). # Can contain only lowercase letters or underscores. -name: my_control_method +name: true_labels # Metadata for your component info: # A relatively short label, used when rendering visualisations (required) - label: My Control Method + label: True Labels # A one sentence summary of how this method works (required). Used when # rendering summary tables. - summary: "FILL IN: A one sentence summary of this method." + summary: "a positive control, solution labels are copied 1 to 1 to the predicted data." # A multi-line description of how this component works (required). Used # when rendering reference documentation. description: | - FILL IN: A (multi-line) description of how this method works. + A positive control, where the solution labels are copied 1 to 1 to the predicted data. # Which normalisation method this component prefers to use (required). - preferred_normalization: log_cp10k + preferred_normalization: counts # Component-specific parameters (optional) # arguments: @@ -43,7 +43,7 @@ resources: engines: # Specifications for the Docker image for this component. - type: docker - image: ghcr.io/openproblems-bio/base_python:1.0.4 + image: ghcr.io/openproblems-bio/base_images/python:1.1.0 # Add custom dependencies here (optional). For more information, see # https://viash.io/reference/config/engines/docker/#setup . # setup: @@ -56,4 +56,4 @@ runners: # Allows turning the component into a Nextflow module / pipeline. - type: nextflow directives: - label: [midtime,midmem,midcpu] + label: [midtime,lowmem,lowcpu] diff --git a/src/control_methods/my_control_method/script.py b/src/control_methods/true_labels/script.py similarity index 51% rename from src/control_methods/my_control_method/script.py rename to src/control_methods/true_labels/script.py index f97215f..0a04aaf 100644 --- a/src/control_methods/my_control_method/script.py +++ b/src/control_methods/true_labels/script.py @@ -4,18 +4,20 @@ # Note: this section is auto-generated by viash at runtime. To edit it, make changes # in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. par = { - 'train_h5ad': 'resources_test/task_template/pancreas/train_h5ad.h5ad', - 'test_h5ad': 'resources_test/task_template/pancreas/test_h5ad.h5ad', + 'input_train': 'resources_test/task_template/pancreas/train.h5ad', + 'input_test': 'resources_test/task_template/pancreas/test.h5ad', + 'input_solution': 'resources_test/task_template/pancreas/solution.h5ad', 'output': 'output.h5ad' } meta = { - 'name': 'my_control_method' + 'name': 'true_labels' } ## VIASH END print('Reading input files', flush=True) -train_h5ad = ad.read_h5ad(par['train_h5ad']) -test_h5ad = ad.read_h5ad(par['test_h5ad']) +input_train = ad.read_h5ad(par['input_train']) +input_test = ad.read_h5ad(par['input_test']) +input_solution = ad.read_h5ad(par['input_solution']) print('Preprocess data', flush=True) # ... preprocessing ... @@ -25,15 +27,19 @@ print('Generate predictions', flush=True) # ... generate predictions ... +obs_label_pred = input_solution.obs["label"] print("Write output AnnData to file", flush=True) output = ad.AnnData( uns={ - 'dataset_id': train_h5ad.uns['dataset_id'], + 'dataset_id': input_train.uns['dataset_id'], + 'normalization_id': input_train.uns['normalization_id'], 'method_id': meta['name'] }, - layers={ - 'prediction': layers_prediction + obs={ + 'label_pred': obs_label_pred } ) +output.obs_names = input_test.obs_names + output.write_h5ad(par['output'], compression='gzip') diff --git a/src/data_processors/process_dataset/config.vsh.yaml b/src/data_processors/process_dataset/config.vsh.yaml new file mode 100644 index 0000000..6f35e96 --- /dev/null +++ b/src/data_processors/process_dataset/config.vsh.yaml @@ -0,0 +1,34 @@ +__merge__: ../../api/comp_data_processor.yaml +name: process_dataset +arguments: + - name: "--method" + type: "string" + description: "The process method to assign train/test." + choices: ["batch", "random"] + default: "batch" + - name: "--obs_label" + type: "string" + description: "Which .obs slot to use as label." + default: "cell_type" + - name: "--obs_batch" + type: "string" + description: "Which .obs slot to use as batch covariate." + default: "batch" + - name: "--seed" + type: "integer" + description: "A seed for the subsampling." + example: 123 +resources: + - type: python_script + path: script.py + - path: /common/helper_functions/subset_anndata.py + +engines: + - type: docker + image: ghcr.io/openproblems-bio/base_images/python:1.1.0 + +runners: + - type: executable + - type: nextflow + directives: + label: [highmem,midcpu,midtime] \ No newline at end of file diff --git a/src/data_processors/process_dataset/script.py b/src/data_processors/process_dataset/script.py new file mode 100644 index 0000000..4348dc8 --- /dev/null +++ b/src/data_processors/process_dataset/script.py @@ -0,0 +1,78 @@ +import sys +import random +import numpy as np +import anndata as ad + +## VIASH START +par = { + 'input': 'resources_test/common/pancreas/dataset.h5ad', + 'method': 'batch', + 'seed': None, + 'obs_batch': 'batch', + 'obs_label': 'cell_type', + 'output_train': 'train.h5ad', + 'output_test': 'test.h5ad', + 'output_solution': 'solution.h5ad' +} +meta = { + 'resources_dir': 'data_processors/process_dataset', + 'config': 'data_processors/process_dataset/.config.vsh.yaml' +} +## VIASH END + +# import helper functions +sys.path.append(meta['resources_dir']) +from subset_anndata import read_config_slots_info, subset_anndata + +# set seed if need be +if par["seed"]: + print(f">> Setting seed to {par['seed']}") + random.seed(par["seed"]) + +print(">> Load data", flush=True) +adata = ad.read_h5ad(par["input"]) +print("input:", adata) + +print(f">> Process data using {par['method']} method") +if par["method"] == "batch": + batch_info = adata.obs[par["obs_batch"]] + batch_categories = batch_info.dtype.categories + test_batches = random.sample(list(batch_categories), 1) + is_test = [ x in test_batches for x in batch_info ] +elif par["method"] == "random": + train_ix = np.random.choice(adata.n_obs, round(adata.n_obs * 0.8), replace=False) + is_test = [ not x in train_ix for x in range(0, adata.n_obs) ] + +# subset the different adatas +print(">> Figuring which data needs to be copied to which output file", flush=True) +# use par arguments to look for label and batch value in different slots +slot_mapping = { + "obs": { + "label": par["obs_label"], + "batch": par["obs_batch"], + } +} +slot_info = read_config_slots_info(meta["config"], slot_mapping) + +print(">> Creating train data", flush=True) +output_train = subset_anndata( + adata[[not x for x in is_test]], + slot_info["output_train"] +) + +print(">> Creating test data", flush=True) +output_test = subset_anndata( + adata[is_test], + slot_info["output_test"] +) + +print(">> Creating solution data", flush=True) +output_solution = subset_anndata( + adata[is_test], + slot_info['output_solution'] +) + +print(">> Writing data", flush=True) +output_train.write_h5ad(par["output_train"]) +output_test.write_h5ad(par["output_test"]) +output_solution.write_h5ad(par["output_solution"]) diff --git a/src/methods/my_method/config.vsh.yaml b/src/methods/logistic_regression/config.vsh.yaml similarity index 66% rename from src/methods/my_method/config.vsh.yaml rename to src/methods/logistic_regression/config.vsh.yaml index 743101f..cb60d48 100644 --- a/src/methods/my_method/config.vsh.yaml +++ b/src/methods/logistic_regression/config.vsh.yaml @@ -8,27 +8,30 @@ __merge__: ../../api/comp_method.yaml # A unique identifier for your component (required). # Can contain only lowercase letters or underscores. -name: my_method +name: logistic_regression # Metadata for your component info: # A relatively short label, used when rendering visualisations (required) - label: My Method + label: Logistic Regression # A one sentence summary of how this method works (required). Used when # rendering summary tables. - summary: "FILL IN: A one sentence summary of this method." + summary: "Logistic Regression with 100-dimensional PCA coordinates estimates parameters for multivariate classification by minimizing cross entropy loss over cell type classes." # A multi-line description of how this component works (required). Used # when rendering reference documentation. description: | - FILL IN: A (multi-line) description of how this method works. + Logistic Regression estimates parameters of a logistic function for + multivariate classification tasks. Here, we use 100-dimensional whitened PCA + coordinates as independent variables, and the model minimises the cross + entropy loss over all cell type classes. # Which normalisation method this component prefers to use (required). preferred_normalization: log_cp10k # A reference key from the bibtex library at src/common/library.bib (required). - reference: bibtex_reference_key + reference: "hosmer2013applied" # URL to the documentation for this method (required). - documentation_url: https://url.to/the/documentation + documentation_url: "https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html" # URL to the code repository for this method (required). - repository_url: https://github.com/organisation/repository + repository_url: https://github.com/scikit-learn/scikit-learn # Component-specific parameters (optional) # arguments: @@ -49,12 +52,12 @@ resources: engines: # Specifications for the Docker image for this component. - type: docker - image: ghcr.io/openproblems-bio/base_python:1.0.4 + image: ghcr.io/openproblems-bio/base_images/python:1.1.0 # Add custom dependencies here (optional). For more information, see # https://viash.io/reference/config/engines/docker/#setup . - # setup: - # - type: python - # packages: scib==1.1.5 + setup: + - type: python + packages: scikit-learn runners: # This platform allows running the component natively @@ -62,4 +65,4 @@ runners: # Allows turning the component into a Nextflow module / pipeline. - type: nextflow directives: - label: [midtime,midmem,midcpu] + label: [midtime,midmem,lowcpu] diff --git a/src/methods/logistic_regression/script.py b/src/methods/logistic_regression/script.py new file mode 100644 index 0000000..cc851f8 --- /dev/null +++ b/src/methods/logistic_regression/script.py @@ -0,0 +1,46 @@ +import anndata as ad +import sklearn.linear_model + +## VIASH START +# Note: this section is auto-generated by viash at runtime. To edit it, make changes +# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. +par = { + 'input_train': 'resources_test/task_template/pancreas/train.h5ad', + 'input_test': 'resources_test/task_template/pancreas/test.h5ad', + 'output': 'output.h5ad' +} +meta = { + 'name': 'logistic_regression' +} +## VIASH END + +print('Reading input files', flush=True) +input_train = ad.read_h5ad(par['input_train']) +input_test = ad.read_h5ad(par['input_test']) + +print('Preprocess data', flush=True) +# ... preprocessing ... + +print('Train model', flush=True) +# ... train model ... +classifier = sklearn.linear_model.LogisticRegression() +classifier.fit(input_train.obsm["X_pca"], input_train.obs["label"].astype(str)) + +print('Generate predictions', flush=True) +# ... generate predictions ... +obs_label_pred = classifier.predict(input_test.obsm["X_pca"]) + +print("Write output AnnData to file", flush=True) +output = ad.AnnData( + uns={ + 'dataset_id': input_train.uns['dataset_id'], + 'normalization_id': input_train.uns['normalization_id'], + 'method_id': meta['name'] + }, + obs={ + 'label_pred': obs_label_pred + } +) +output.obs_names = input_test.obs_names + +output.write_h5ad(par['output'], compression='gzip') diff --git a/src/methods/my_method/script.py b/src/methods/my_method/script.py deleted file mode 100644 index b0ed7f1..0000000 --- a/src/methods/my_method/script.py +++ /dev/null @@ -1,37 +0,0 @@ -import anndata as ad - -## VIASH START -# Note: this section is auto-generated by viash at runtime. To edit it, make changes -# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. -par = { - 'train_h5ad': 'resources_test/task_template/pancreas/train_h5ad.h5ad', - 'output': 'output.h5ad' -} -meta = { - 'name': 'my_method' -} -## VIASH END - -print('Reading input files', flush=True) -train_h5ad = ad.read_h5ad(par['train_h5ad']) - -print('Preprocess data', flush=True) -# ... preprocessing ... - -print('Train model', flush=True) -# ... train model ... - -print('Generate predictions', flush=True) -# ... generate predictions ... - -print("Write output AnnData to file", flush=True) -output = ad.AnnData( - uns={ - 'dataset_id': train_h5ad.uns['dataset_id'], - 'method_id': meta['name'] - }, - layers={ - 'prediction': layers_prediction - } -) -output.write_h5ad(par['output'], compression='gzip') diff --git a/src/metrics/my_metric/config.vsh.yaml b/src/metrics/accuracy/config.vsh.yaml similarity index 51% rename from src/metrics/my_metric/config.vsh.yaml rename to src/metrics/accuracy/config.vsh.yaml index d998cf0..f336d1c 100644 --- a/src/metrics/my_metric/config.vsh.yaml +++ b/src/metrics/accuracy/config.vsh.yaml @@ -8,35 +8,31 @@ __merge__: ../../api/comp_metric.yaml # A unique identifier for your component (required). # Can contain only lowercase letters or underscores. -name: my_metric +name: accuracy # Metadata for your component info: metrics: # A unique identifier for your metric (required). # Can contain only lowercase letters or underscores. - name: my_metric - # A relatively short label, used when rendering visualisarions (required) - label: My Metric - # A one sentence summary of how this metric works (required). Used when - # rendering summary tables. - summary: "FILL IN: A one sentence summary of this metric." - # A multi-line description of how this component works (required). Used - # when rendering reference documentation. - description: | - FILL IN: A (multi-line) description of how this metric works. - # A reference key from the bibtex library at src/common/library.bib (required). - reference: bibtex_reference_key - # URL to the documentation for this metric (required). - documentation_url: https://url.to/the/documentation - # URL to the code repository for this metric (required). - repository_url: https://github.com/organisation/repository - # The minimum possible value for this metric (required) - min: 0 - # The maximum possible value for this metric (required) - max: 1 - # Whether a higher value represents a 'better' solution (required) - maximize: true + - name: accuracy + # A relatively short label, used when rendering visualisarions (required) + label: Accuracy + # A one sentence summary of how this metric works (required). Used when + # rendering summary tables. + summary: "The percentage of correctly predicted labels." + # A multi-line description of how this component works (required). Used + # when rendering reference documentation. + description: | + The percentage of correctly predicted labels. + # A reference key from the bibtex library at src/common/library.bib (required). + reference: grandini2020metrics + # The minimum possible value for this metric (required) + min: 0 + # The maximum possible value for this metric (required) + max: 1 + # Whether a higher value represents a 'better' solution (required) + maximize: true # Component-specific parameters (optional) # arguments: @@ -57,12 +53,12 @@ resources: engines: # Specifications for the Docker image for this component. - type: docker - image: ghcr.io/openproblems-bio/base_python:1.0.4 + image: ghcr.io/openproblems-bio/base_images/python:1.1.0 # Add custom dependencies here (optional). For more information, see # https://viash.io/reference/config/engines/docker/#setup . - # setup: - # - type: python - # packages: scib==1.1.5 + setup: + - type: python + packages: scikit-learn runners: # This platform allows running the component natively diff --git a/src/metrics/accuracy/script.py b/src/metrics/accuracy/script.py new file mode 100644 index 0000000..72dcb1e --- /dev/null +++ b/src/metrics/accuracy/script.py @@ -0,0 +1,47 @@ +import anndata as ad +import numpy as np +import sklearn.preprocessing + +## VIASH START +# Note: this section is auto-generated by viash at runtime. To edit it, make changes +# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. +par = { + 'input_solution': 'resources_test/task_template/pancreas/solution.h5ad', + 'input_prediction': 'resources_test/task_template/pancreas/prediction.h5ad', + 'output': 'output.h5ad' +} +meta = { + 'name': 'accuracy' +} +## VIASH END + +print('Reading input files', flush=True) +input_solution = ad.read_h5ad(par['input_solution']) +input_prediction = ad.read_h5ad(par['input_prediction']) + +assert (input_prediction.obs_names == input_solution.obs_names).all(), "obs_names not the same in prediction and solution inputs" + +print("Encode labels", flush=True) +cats = list(input_solution.obs["label"].dtype.categories) + list(input_prediction.obs["label_pred"].dtype.categories) +encoder = sklearn.preprocessing.LabelEncoder().fit(cats) +input_solution.obs["label"] = encoder.transform(input_solution.obs["label"]) +input_prediction.obs["label_pred"] = encoder.transform(input_prediction.obs["label_pred"]) + + +print('Compute metrics', flush=True) +# metric_ids and metric_values can have length > 1 +# but should be of equal length +uns_metric_ids = [ 'accuracy' ] +uns_metric_values = np.mean(input_solution.obs["label"] == input_prediction.obs["label_pred"]) + +print("Write output AnnData to file", flush=True) +output = ad.AnnData( + uns={ + 'dataset_id': input_prediction.uns['dataset_id'], + 'normalization_id': input_prediction.uns['normalization_id'], + 'method_id': input_prediction.uns['method_id'], + 'metric_ids': uns_metric_ids, + 'metric_values': uns_metric_values + } +) +output.write_h5ad(par['output'], compression='gzip') diff --git a/src/metrics/my_metric/script.py b/src/metrics/my_metric/script.py deleted file mode 100644 index 08dc74d..0000000 --- a/src/metrics/my_metric/script.py +++ /dev/null @@ -1,35 +0,0 @@ -import anndata as ad - -## VIASH START -# Note: this section is auto-generated by viash at runtime. To edit it, make changes -# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. -par = { - 'input_test': 'resources_test/task_template/pancreas/test.h5ad', - 'input_prediction': 'resources_test/task_template/pancreas/prediction.h5ad', - 'output': 'output.h5ad' -} -meta = { - 'name': 'my_metric' -} -## VIASH END - -print('Reading input files', flush=True) -input_test = ad.read_h5ad(par['input_test']) -input_prediction = ad.read_h5ad(par['input_prediction']) - -print('Compute metrics', flush=True) -# metric_ids and metric_values can have length > 1 -# but should be of equal length -uns_metric_ids = [ 'my_metric' ] -uns_metric_values = [ 0.5 ] - -print("Write output AnnData to file", flush=True) -output = ad.AnnData( - uns={ - 'dataset_id': input_prediction.uns['dataset_id'], - 'method_id': input_prediction.uns['method_id'], - 'metric_ids': uns_metric_ids, - 'metric_values': uns_metric_values - } -) -output.write_h5ad(par['output'], compression='gzip') diff --git a/src/workflows/process_datasets/config.vsh.yaml b/src/workflows/process_datasets/config.vsh.yaml new file mode 100644 index 0000000..ed41e20 --- /dev/null +++ b/src/workflows/process_datasets/config.vsh.yaml @@ -0,0 +1,41 @@ +name: process_datasets +namespace: workflows +argument_groups: + - name: Inputs + arguments: + - name: "--input" + __merge__: /src/api/file_common_dataset.yaml + required: true + direction: input + - name: Outputs + arguments: + - name: "--output_train" + __merge__: /src/api/file_train_h5ad.yaml + required: true + direction: output + - name: "--output_test" + __merge__: /src/api/file_test_h5ad.yaml + required: true + direction: output + - name: "--output_solution" + __merge__: /src/api/file_solution.yaml + required: true + direction: output +resources: + - type: nextflow_script + path: main.nf + entrypoint: run_wf + - path: /common/nextflow_helpers/helper.nf +repositories: + - name: openproblems-v2 + type: github + repo: openproblems-bio/openproblems-v2 + tag: main_build +dependencies: + - name: common/check_dataset_schema + repository: openproblems-v2 + - name: common/extract_metadata + repository: openproblems-v2 + - name: data_processors/process_dataset +runners: + - type: nextflow \ No newline at end of file diff --git a/src/workflows/process_datasets/main.nf b/src/workflows/process_datasets/main.nf new file mode 100644 index 0000000..eae19f7 --- /dev/null +++ b/src/workflows/process_datasets/main.nf @@ -0,0 +1,173 @@ +include { findArgumentSchema } from "${meta.resources_dir}/helper.nf" + +workflow auto { + findStatesTemp(params, meta.config) + | meta.workflow.run( + auto: [publish: "state"] + ) +} + +workflow run_wf { + take: + input_ch + + main: + output_ch = input_ch + + | check_dataset_schema.run( + fromState: { id, state -> + def schema = findArgumentSchema(meta.config, "input") + def schemaYaml = tempFile("schema.yaml") + writeYaml(schema, schemaYaml) + [ + "input": state.input, + "schema": schemaYaml + ] + }, + toState: { id, output, state -> + // read the output to see if dataset passed the qc + def checks = readYaml(output.output) + state + [ + "dataset": checks["exit_code"] == 0 ? state.input : null, + ] + } + ) + + // remove datasets which didn't pass the schema check + | filter { id, state -> + state.dataset != null + } + + | process_dataset.run( + fromState: [ input: "dataset" ], + toState: [ + output_train: "output_train", + output_test: "output_test", + output_solution: "output_solution" + ] + ) + + // only output the files for which an output file was specified + | setState(["output_train", "output_test", "output_solution"]) + + emit: + output_ch +} + + +// temp fix for rename_keys typo + +def findStatesTemp(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesTempWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey;newKey:oldKey'" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesTempWf +} \ No newline at end of file diff --git a/src/workflows/process_datasets/test.sh b/src/workflows/process_datasets/test.sh new file mode 100755 index 0000000..d918102 --- /dev/null +++ b/src/workflows/process_datasets/test.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Run this prior to executing this script: +# bin/viash_build -q 'batch_integration' + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +DATASETS_DIR="resources_test/common" +OUTPUT_DIR="output/process_datasets_test" + +if [ ! -d "$OUTPUT_DIR" ]; then + mkdir -p "$OUTPUT_DIR" +fi + +export NXF_VER=24.04.3 + +nextflow run . \ + -main-script target/nextflow/workflows/process_datasets/main.nf \ + -profile docker \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + --id run_test \ + --input_states "$DATASETS_DIR/**/state.yaml" \ + --rename_keys 'input:output_dataset' \ + --settings '{"output_train": "train.h5ad", "output_test": "test.h5ad"}' \ + --publish_dir "$OUTPUT_DIR" \ + --output_state "state.yaml" \ No newline at end of file diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml new file mode 100644 index 0000000..f298617 --- /dev/null +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -0,0 +1,79 @@ +name: run_benchmark +namespace: workflows + +argument_groups: + - name: Inputs + arguments: + - name: "--input_train" + __merge__: /src/api/file_train_h5ad.yaml + type: file + direction: input + required: true + - name: "--input_test" + __merge__: /src/api/file_test_h5ad.yaml + type: file + direction: input + required: true + - name: "--input_solution" + __merge__: /src/api/file_solution.yaml + type: file + direction: input + required: true + - name: Outputs + arguments: + - name: "--output_scores" + type: file + required: true + direction: output + description: A yaml file containing the scores of each of the methods + default: score_uns.yaml + - name: "--output_method_configs" + type: file + required: true + direction: output + default: method_configs.yaml + - name: "--output_metric_configs" + type: file + required: true + direction: output + default: metric_configs.yaml + - name: "--output_dataset_info" + type: file + required: true + direction: output + default: dataset_uns.yaml + - name: "--output_task_info" + type: file + required: true + direction: output + default: task_info.yaml + - name: Methods + arguments: + - name: "--method_ids" + type: string + multiple: true + description: A list of method ids to run. If not specified, all methods will be run. + +resources: + - type: nextflow_script + path: main.nf + entrypoint: run_wf + - type: file + path: /_viash.yaml + +repositories: + - name: openproblems-v2 + type: github + repo: openproblems-bio/openproblems-v2 + tag: main_build +dependencies: + - name: common/check_dataset_schema + repository: openproblems-v2 + - name: common/extract_metadata + repository: openproblems-v2 + - name: control_methods/true_labels + - name: methods/logistic_regression + - name: metrics/accuracy + +runners: + - type: nextflow diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf new file mode 100644 index 0000000..68e5ecd --- /dev/null +++ b/src/workflows/run_benchmark/main.nf @@ -0,0 +1,311 @@ +workflow auto { + findStatesTemp(params, meta.config) + | meta.workflow.run( + auto: [publish: "state"] + ) +} + +workflow run_wf { + take: + input_ch + + main: + + // construct list of methods + methods = [ + true_labels, + logistic_regression + ] + + // construct list of metrics + metrics = [ + accuracy + ] + + /**************************** + * EXTRACT DATASET METADATA * + ****************************/ + dataset_ch = input_ch + // store join id + | map{ id, state -> + [id, state + ["_meta": [join_id: id]]] + } + + // extract the dataset metadata + | extract_metadata.run( + fromState: [input: "input_solution"], + toState: { id, output, state -> + state + [ + dataset_uns: readYaml(output.output).uns + ] + } + ) + + /*************************** + * RUN METHODS AND METRICS * + ***************************/ + score_ch = dataset_ch + + // run all methods + | runEach( + components: methods, + + // use the 'filter' argument to only run a method on the normalisation the component is asking for + filter: { id, state, comp -> + def norm = state.dataset_uns.normalization_id + def pref = comp.config.info.preferred_normalization + // if the preferred normalisation is none at all, + // we can pass whichever dataset we want + def norm_check = (norm == "log_cp10k" && pref == "counts") || norm == pref + def method_check = !state.method_ids || state.method_ids.contains(comp.config.name) + + method_check && norm_check + }, + + // define a new 'id' by appending the method name to the dataset id + id: { id, state, comp -> + id + "." + comp.config.name + }, + + // use 'fromState' to fetch the arguments the component requires from the overall state + fromState: { id, state, comp -> + def new_args = [ + input_train: state.input_train, + input_test: state.input_test + ] + if (comp.config.info.type == "control_method") { + new_args.input_solution = state.input_solution + } + new_args + }, + + // use 'toState' to publish that component's outputs to the overall state + toState: { id, output, state, comp -> + state + [ + method_id: comp.config.name, + method_output: output.output + ] + } + ) + + // run all metrics + | runEach( + components: metrics, + id: { id, state, comp -> + id + "." + comp.config.name + }, + // use 'fromState' to fetch the arguments the component requires from the overall state + fromState: [ + input_solution: "input_solution", + input_prediction: "method_output" + ], + // use 'toState' to publish that component's outputs to the overall state + toState: { id, output, state, comp -> + state + [ + metric_id: comp.config.name, + metric_output: output.output + ] + } + ) + + + /****************************** + * GENERATE OUTPUT YAML FILES * + ******************************/ + // TODO: can we store everything below in a separate helper function? + + // extract the dataset metadata + dataset_meta_ch = dataset_ch + // only keep one of the normalization methods + | filter{ id, state -> + state.dataset_uns.normalization_id == "log_cp10k" + } + | joinStates { ids, states -> + // store the dataset metadata in a file + def dataset_uns = states.collect{state -> + def uns = state.dataset_uns.clone() + uns.remove("normalization_id") + uns + } + def dataset_uns_yaml_blob = toYamlBlob(dataset_uns) + def dataset_uns_file = tempFile("dataset_uns.yaml") + dataset_uns_file.write(dataset_uns_yaml_blob) + + ["output", [output_dataset_info: dataset_uns_file]] + } + + output_ch = score_ch + + // extract the scores + | extract_metadata.run( + key: "extract_scores", + fromState: [input: "metric_output"], + toState: { id, output, state -> + state + [ + score_uns: readYaml(output.output).uns + ] + } + ) + + | joinStates { ids, states -> + // store the method configs in a file + def method_configs = methods.collect{it.config} + def method_configs_yaml_blob = toYamlBlob(method_configs) + def method_configs_file = tempFile("method_configs.yaml") + method_configs_file.write(method_configs_yaml_blob) + + // store the metric configs in a file + def metric_configs = metrics.collect{it.config} + def metric_configs_yaml_blob = toYamlBlob(metric_configs) + def metric_configs_file = tempFile("metric_configs.yaml") + metric_configs_file.write(metric_configs_yaml_blob) + + def viash_file = meta.resources_dir.resolve("_viash.yaml") + def viash_file_content = toYamlBlob(readYaml(viash_file).info) + def task_info_file = tempFile("task_info.yaml") + task_info_file.write(viash_file_content) + + // store the scores in a file + def score_uns = states.collect{it.score_uns} + def score_uns_yaml_blob = toYamlBlob(score_uns) + def score_uns_file = tempFile("score_uns.yaml") + score_uns_file.write(score_uns_yaml_blob) + + def new_state = [ + output_method_configs: method_configs_file, + output_metric_configs: metric_configs_file, + output_task_info: task_info_file, + output_scores: score_uns_file, + _meta: states[0]._meta + ] + + ["output", new_state] + } + + // merge all of the output data + | mix(dataset_meta_ch) + | joinStates{ ids, states -> + def mergedStates = states.inject([:]) { acc, m -> acc + m } + [ids[0], mergedStates] + } + + emit: + output_ch +} + +// temp fix for rename_keys typo + +def findStatesTemp(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesTempWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey;newKey:oldKey'" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesTempWf +} \ No newline at end of file diff --git a/src/workflows/run_benchmark/test.sh b/src/workflows/run_benchmark/test.sh new file mode 100755 index 0000000..b0dbc24 --- /dev/null +++ b/src/workflows/run_benchmark/test.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +# export TOWER_WORKSPACE_ID=53907369739130 + +DATASETS_DIR="resources_test/task_template" +OUTPUT_DIR="output/temp" + +if [ ! -d "$OUTPUT_DIR" ]; then + mkdir -p "$OUTPUT_DIR" +fi + +export NXF_VER=24.04.3 +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + --input_states "$DATASETS_DIR/**/state.yaml" \ + --rename_keys 'input_train:output_train;input_test:output_test;input_solution:output_solution' \ + --settings '{"output_scores": "scores.yaml", "output_dataset_info": "dataset_info.yaml", "output_method_configs": "method_configs.yaml", "output_metric_configs": "metric_configs.yaml", "output_task_info": "task_info.yaml"}' \ + --publish_dir "$OUTPUT_DIR" \ + --output_state "state.yaml"