Backport PR #852 and #851 to 1.0.x (#853)

openpipelines-bio · Aug 9, 2024 · 12b273e · 12b273e
1 parent 2963adb
commit 12b273e
Show file tree

Hide file tree

Showing 678 changed files with 10,195 additions and 2,653 deletions.
diff --git a/.github/workflows/create-documentation-pr.yml b/.github/workflows/create-documentation-pr.yml
@@ -13,55 +13,8 @@ on:
       - '*'
 
 jobs:
-  list:
-    runs-on: ubuntu-latest
-
-    outputs:
-      component_matrix: ${{ steps.set_matrix.outputs.components }}
-      workflow_matrix: ${{ steps.set_matrix.outputs.workflows }}
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: viash-io/viash-actions/setup@v5
-
-    - id: ns_list_components
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        platform: docker
-        format: json
-        query_namespace: ^(?!workflows)
-
-    - id: ns_list_workflows
-      uses: viash-io/viash-actions/ns-list@v5
-      with:
-        format: json
-        query_namespace: ^workflows
-
-    - id: set_matrix
-      run: |
-        echo "components=$(jq -c '[ .[] | 
-          { 
-            "name": .functionality.name,
-            "namespace_separator": .platforms | map(select(.type == "docker"))[0].namespace_separator,
-            "namespace": .functionality.namespace,
-            "config": .info.config,
-            "dir": .info.config | capture("^(?<dir>.*\/)").dir
-          }
-        ]' ${{ steps.ns_list_components.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
-        echo "workflows=$(jq -c '[ .[] | 
-          { 
-            "name": $config.functionality.name,
-            "namespace": $config.functionality.namespace,
-            "main_script": ((.info.config | capture("^(?<dir>.*\/)").dir) + "/" + (.functionality.test_resources | map(select(.type == "nextflow_script")) | unique | .[].path )),
-            "config": .info.config,
-            "entry": .functionality.test_resources | map(select(.type == "nextflow_script")) | .[].entrypoint
-          }
-        ]' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT
-
   push_documentation:
     runs-on: ubuntu-latest
-    needs: [ list ]
 
     steps:
       - uses: actions/checkout@v4
@@ -74,7 +27,7 @@ jobs:
           path: website
           token: ${{ secrets.GTHB_PAT }}
 
-      - uses: viash-io/viash-actions/setup@v5
+      - uses: viash-io/viash-actions/setup@v6
 
       - name: Get tag to use
         id: get_tag
@@ -83,15 +36,30 @@ jobs:
           TAG_OR_BRANCH_NAME=${INPUT_TAG:-"${{ github.ref_name }}"}
           echo "tag=$TAG_OR_BRANCH_NAME" >> $GITHUB_OUTPUT
 
-      - uses: viash-io/viash-actions/pro/generate-documentation-qmd@v5
+      - uses: viash-io/viash-actions/pro/generate-documentation-qmd@v6
+        with:
+          project_directory: openpipelines
+          src: src
+          query: '^(?!workflows|test_workflows)'
+          output_dir: ../website/components/modules/
+          dest_path: "{namespace}/{name}.qmd"
+          viash_pro_token: ${{ secrets.GTHB_PAT }}
+          tools_version: main_build
+          ref: "${{ steps.get_tag.outputs.tag }}"
+          clean: true
+
+
+      - uses: viash-io/viash-actions/pro/generate-documentation-qmd@v6
         with:
           project_directory: openpipelines
-          src: ./
+          src: src
+          query: '^workflows'
           output_dir: ../website/components/
-          dest_path: "{type}s/{namespace}/{name}.qmd"
+          dest_path: "{namespace}/{name}.qmd"
           viash_pro_token: ${{ secrets.GTHB_PAT }}
           tools_version: main_build
           ref: "${{ steps.get_tag.outputs.tag }}"
+          clean: true
 
       - name: Create Pull Request
         env:

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -35,7 +35,7 @@ jobs:
       cache_key: ${{ steps.cache.outputs.cache_key }}
 
     steps:
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
       id: cache 
       with:
         s3_bucket: $s3_bucket
@@ -65,7 +65,7 @@ jobs:
       with:
         ref: 'integration_build'
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - uses: nf-core/[email protected]
 
@@ -100,6 +100,17 @@ jobs:
         find ./resources_test/ "${to_not_remove[@]}" -type f -exec rm {} +
         tree ./resources_test/
 
+    - name: Build test dependencies
+      uses: viash-io/viash-actions/ns-build@v6
+      with:
+        config_mod: |
+          .functionality.version := 'integration_build'
+          .platforms[.type == 'docker'].target_tag := 'integration_build'
+        platform: 'docker|nextflow'
+        src: ${{ matrix.component.dir }}
+        query: "^test_workflows"
+        setup: build
+
     - name: Run integration test
       timeout-minutes: 60
       run: |

diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml
@@ -131,41 +131,39 @@ jobs:
         git fetch origin ${{github.ref_name}} 
         git checkout -f --no-overlay origin/${{github.ref_name}} -- '.'
     
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - name: Remove target folder from .gitignore
       run: |
         # allow publishing the target folder
         sed -i '/^\/target\/$/d' .gitignore
 
-    - uses: viash-io/viash-actions/ns-build@v5
+    - uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: |
           .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
           .platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}'
         parallel: true
-        query: ^(?!workflows)
+        query: ^(?!workflows|test_workflows)
 
-    - uses: viash-io/viash-actions/ns-build@v5
+    - uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
         parallel: true
         query: ^workflows
 
     - name: Build nextflow schemas
-      uses: viash-io/viash-actions/pro/build-nextflow-schemas@v5
+      uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6
       with:
-        components: src
-        workflows: src
+        target_dir: target/nextflow
         viash_pro_token: ${{ secrets.GTHB_PAT }}
         tools_version: 'main_build'
         enable_dataset_input: true
 
     - name: Build parameter files
-      uses: viash-io/viash-actions/pro/build-nextflow-params@v5
+      uses: viash-io/viash-actions/pro/build-nextflow-params@v6
       with:
-        workflows: src
-        components: src
+        target_dir: target/nextflow
         viash_pro_token: ${{ secrets.GTHB_PAT }}
         tools_version: 'main_build'
 
@@ -178,16 +176,16 @@ jobs:
         branch:  ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch  }}
     - name: "List components"
       id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
+      uses: viash-io/viash-actions/ns-list@v6
       with:
         platform: docker
         src: src
         format: json
-        query_namespace: ^(?!workflows)
+        query_namespace: ^(?!workflows|test_workflows)
 
     - name: "List workflows"
       id: ns_list_workflows
-      uses: viash-io/viash-actions/ns-list@v5
+      uses: viash-io/viash-actions/ns-list@v6
       with:
         src: src
         format: json
@@ -235,10 +233,10 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - name: Build container
-      uses: viash-io/viash-actions/ns-build@v5
+      uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: |
           .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
@@ -255,7 +253,7 @@ jobs:
         password: ${{ secrets.GTHB_PAT }}
 
     - name: Push container
-      uses: viash-io/viash-actions/ns-build@v5
+      uses: viash-io/viash-actions/ns-build@v6
       with:
         config_mod: .functionality.version := "${{ inputs.version || format('{0}_build', github.ref_name) }}"
         platform: docker

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
@@ -28,7 +28,7 @@ jobs:
       cache_key: ${{ steps.cache.outputs.cache_key }}
 
     steps:
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
       id: cache 
       with:
         s3_bucket: $s3_bucket
@@ -58,7 +58,7 @@ jobs:
       with:
         ref: 'integration_build'
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     - uses: nf-core/[email protected]
 
@@ -123,7 +123,7 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     # use cache
     - name: Cache resources data

diff --git a/.github/workflows/viash-test.yml b/.github/workflows/viash-test.yml
@@ -41,28 +41,32 @@ jobs:
       with:
         fetch-depth: 0 
 
-    - uses: viash-io/viash-actions/setup@v5
+    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
+    - uses: data-intuitive/reclaim-the-bytes@v2
+
+    - uses: viash-io/viash-actions/setup@v6
 
     - name: Check if all config can be parsed if there is no unicode support
       run: |
         LANG=C viash ns list > /dev/null
+        JAVA_TOOL_OPTIONS=-Dfile.encoding=ascii viash ns build --parallel --setup meh > /dev/null
 
-    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v5
+    - uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
       id: cache
       with:
         s3_bucket: $s3_bucket
         dest_path: resources_test
         cache_key_prefix: resources_test__
 
     - id: ns_list
-      uses: viash-io/viash-actions/ns-list@v5
+      uses: viash-io/viash-actions/ns-list@v6
       with:
         platform: docker
         format: json
-        query_namespace: ^(?!workflows)
+        query_namespace: ^(?!workflows|test_workflows)
 
     - id: ns_list_filtered
-      uses: viash-io/viash-actions/project/detect-changed-components@v5
+      uses: viash-io/viash-actions/project/detect-changed-components@v6
       with:
         input_file: "${{ steps.ns_list.outputs.output_file }}"
 
@@ -95,7 +99,7 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - uses: viash-io/viash-actions/setup@v5
+    - uses: viash-io/viash-actions/setup@v6
 
     # use cache
     - name: Cache resources data
@@ -163,10 +167,10 @@ jobs:
         password: ${{ env.NVIDIA_PASSWORD }}
 
     - name: Run test
-      timeout-minutes: 30
+      timeout-minutes: 40
       run: |
         viash test \
           "${{ matrix.component.config }}" \
           --cpus 4 \
-          --memory "12gb"
+          --memory "14gb"
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+# openpipelines 1.0.3
+
+## BUG FIXES
+
+* `qc/calculate_qc_metrics`: increase total counts accuracy with low precision floating dtypes as input layer (PR # , backported from PR #852).
+
 # openpipelines 1.0.2
 
 ## BUG FIXES

diff --git a/resources_test_scripts/pbmc_1k_protein_v3.sh b/resources_test_scripts/pbmc_1k_protein_v3.sh
@@ -45,11 +45,6 @@ target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \
   --input "${OUT}_filtered_feature_bc_matrix.h5" \
   --input_metrics_summary "${OUT}_metrics_summary.csv" \
   --output "${OUT}_filtered_feature_bc_matrix.h5mu"
-
-target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \
-  --input "${OUT}_raw_feature_bc_matrix.h5" \
-  --input_metrics_summary "${OUT}_metrics_summary.csv" \
-  --output "${OUT}_raw_feature_bc_matrix.h5mu"
 
 # run single sample
 NXF_VER=21.10.6 nextflow \

diff --git a/src/correction/cellbender_remove_background/config.vsh.yaml b/src/correction/cellbender_remove_background/config.vsh.yaml
@@ -274,7 +274,7 @@ functionality:
     - type: python_script
       path: test.py
     # normally cellbender should only be run on unfiltered data, but for the purposes of the unit test the filtered data will do.
-    - path: /resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5mu
+    - path: /resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
 platforms:
   - type: docker
     image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04
@@ -296,6 +296,7 @@ platforms:
         packages:
           # Use this version of mudata because it is the last one that supports python 3.7
           - mudata~=0.2.1
+          - lxml[html_clean]
           - cellbender~=0.3.0
   - type: nextflow
     directives:

diff --git a/src/correction/cellbender_remove_background/script.py b/src/correction/cellbender_remove_background/script.py
@@ -7,7 +7,7 @@
 from scipy.sparse import csr_matrix
 from cellbender.remove_background.downstream import anndata_from_h5
 ## VIASH START
-file_input = "./resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5mu"
+file_input = "./resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
 
 par = {
     # inputs

diff --git a/src/correction/cellbender_remove_background/test.py b/src/correction/cellbender_remove_background/test.py
@@ -9,7 +9,7 @@
 }
 ## VIASH END
 
-file_input = meta["resources_dir"] + "/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5mu"
+file_input = meta["resources_dir"] + "/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
 file_output = "output.h5mu"
 
 print("> Check whether cellbender works when it should be working")

diff --git a/src/qc/calculate_qc_metrics/config.vsh.yaml b/src/qc/calculate_qc_metrics/config.vsh.yaml
@@ -140,9 +140,10 @@ platforms:
         __merge__: [/src/base/requirements/anndata_mudata.yaml, .]
         packages:
           - scikit-learn~=1.2.0
+    __merge__: [ /src/base/requirements/python_test_setup.yaml, .]
     test_setup:
       - type: python
-        __merge__: [ /src/base/requirements/viashpy.yaml, /src/base/requirements/scanpy.yaml, .]
+        __merge__: [ /src/base/requirements/scanpy.yaml, .]
   - type: nextflow
     directives:
       label: [singlecpu, midmem]